+26 -3

appview/pipelines/pipelines.go

··· 158 158 l.Error("websocket upgrade failed", "err", err) 159 159 return 160 160 } 161 161 - defer clientConn.Close() 161 161 + defer func() { 162 162 + _ = clientConn.WriteControl( 163 163 + websocket.CloseMessage, 164 164 + websocket.FormatCloseMessage(websocket.CloseNormalClosure, "log stream complete"), 165 165 + time.Now().Add(time.Second), 166 166 + ) 167 167 + clientConn.Close() 168 168 + }() 162 169 163 170 ctx, cancel := context.WithCancel(r.Context()) 164 171 defer cancel() ··· 236 243 // start a goroutine to read from spindle 237 244 go func() { 238 245 defer close(msgChan) 246 246 + defer close(errChan) 247 247 + 239 248 for { 240 249 _, msg, err := spindleConn.ReadMessage() 241 250 if err != nil { 242 242 - errChan <- err 251 251 + if websocket.IsCloseError(err, 252 252 + websocket.CloseNormalClosure, 253 253 + websocket.CloseGoingAway, 254 254 + websocket.CloseAbnormalClosure) { 255 255 + errChan <- nil // signal graceful end 256 256 + } else { 257 257 + errChan <- err 258 258 + } 243 259 return 244 260 } 245 261 msgChan <- msg ··· 252 268 l.Info("client disconnected") 253 269 return 254 270 case err := <-errChan: 255 255 - l.Error("error reading from spindle", "err", err) 271 271 + if err != nil { 272 272 + l.Error("error reading from spindle", "err", err) 273 273 + } 274 274 + 275 275 + if err == nil { 276 276 + l.Info("log tail complete") 277 277 + } 278 278 + 256 279 return 257 280 case msg := <-msgChan: 258 281 var logLine spindlemodel.LogLine

+34

spindle/db/events.go

··· 120 120 121 121 } 122 122 123 123 + func (d *DB) GetStatus(workflowId models.WorkflowId) (*tangled.PipelineStatus, error) { 124 124 + pipelineAtUri := workflowId.PipelineId.AtUri() 125 125 + 126 126 + var eventJson string 127 127 + err := d.QueryRow( 128 128 + ` 129 129 + select 130 130 + event from events 131 131 + where 132 132 + nsid = ? 133 133 + and json_extract(event, '$.pipeline') = ? 134 134 + and json_extract(event, '$.workflow') = ? 135 135 + order by 136 136 + created desc 137 137 + limit 138 138 + 1 139 139 + `, 140 140 + tangled.PipelineStatusNSID, 141 141 + string(pipelineAtUri), 142 142 + workflowId.Name, 143 143 + ).Scan(&eventJson) 144 144 + 145 145 + if err != nil { 146 146 + return nil, err 147 147 + } 148 148 + 149 149 + var status tangled.PipelineStatus 150 150 + if err := json.Unmarshal([]byte(eventJson), &status); err != nil { 151 151 + return nil, err 152 152 + } 153 153 + 154 154 + return &status, nil 155 155 + } 156 156 + 123 157 func (d *DB) StatusPending(workflowId models.WorkflowId, n *notifier.Notifier) error { 124 158 return d.createStatusEvent(workflowId, models.StatusKindPending, nil, nil, n) 125 159 }

+5 -1

spindle/engine/engine.go

··· 326 326 } 327 327 defer wfLogger.Close() 328 328 329 329 - _, err = stdcopy.StdCopy(wfLogger.Stdout(), wfLogger.Stderr(), logs) 329 329 + _, err = stdcopy.StdCopy( 330 330 + wfLogger.Writer("stdout", stepIdx), 331 331 + wfLogger.Writer("stderr", stepIdx), 332 332 + logs, 333 333 + ) 330 334 if err != nil && err != io.EOF && !errors.Is(err, context.DeadlineExceeded) { 331 335 return fmt.Errorf("failed to copy logs: %w", err) 332 336 }

+5 -23

spindle/engine/logger.go

··· 17 17 } 18 18 19 19 func NewWorkflowLogger(baseDir string, wid models.WorkflowId) (*WorkflowLogger, error) { 20 20 - dir := filepath.Join(baseDir, wid.String()) 21 21 - if err := os.MkdirAll(dir, 0755); err != nil { 22 22 - return nil, fmt.Errorf("creating log dir: %w", err) 23 23 - } 24 24 - 25 20 path := LogFilePath(baseDir, wid) 26 21 27 27 - file, err := os.Create(path) 22 22 + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) 28 23 if err != nil { 29 24 return nil, fmt.Errorf("creating log file: %w", err) 30 25 } ··· 43 38 return l.file.Close() 44 39 } 45 40 46 46 - func OpenLogFile(baseDir string, workflowID models.WorkflowId) (*os.File, error) { 47 47 - logPath := LogFilePath(baseDir, workflowID) 48 48 - 49 49 - file, err := os.Open(logPath) 50 50 - if err != nil { 51 51 - return nil, fmt.Errorf("error opening log file: %w", err) 52 52 - } 53 53 - 54 54 - return file, nil 55 55 - } 56 56 - 57 41 func LogFilePath(baseDir string, workflowID models.WorkflowId) string { 58 42 logFilePath := filepath.Join(baseDir, fmt.Sprintf("%s.log", workflowID.String())) 59 43 return logFilePath 60 44 } 61 45 62 62 - func (l *WorkflowLogger) Stdout() io.Writer { 63 63 - return &jsonWriter{logger: l, stream: "stdout"} 64 64 - } 65 65 - 66 66 - func (l *WorkflowLogger) Stderr() io.Writer { 67 67 - return &jsonWriter{logger: l, stream: "stderr"} 46 46 + func (l *WorkflowLogger) Writer(stream string, stepId int) io.Writer { 47 47 + return &jsonWriter{logger: l, stream: stream, stepId: stepId} 68 48 } 69 49 70 50 type jsonWriter struct { 71 51 logger *WorkflowLogger 72 52 stream string 53 53 + stepId int 73 54 } 74 55 75 56 func (w *jsonWriter) Write(p []byte) (int, error) { ··· 78 59 entry := models.LogLine{ 79 60 Stream: w.stream, 80 61 Data: line, 62 62 + StepId: w.stepId, 81 63 } 82 64 83 65 if err := w.logger.encoder.Encode(entry); err != nil {

+1

spindle/models/models.go

··· 74 74 type LogLine struct { 75 75 Stream string `json:"s"` 76 76 Data string `json:"d"` 77 77 + StepId int `json:"i"` 77 78 }

+29 -7

spindle/stream.go

··· 4 4 "context" 5 5 "encoding/json" 6 6 "fmt" 7 7 + "io" 7 8 "net/http" 8 9 "strconv" 9 10 "time" ··· 105 106 http.Error(w, "failed to upgrade", http.StatusInternalServerError) 106 107 return 107 108 } 108 108 - defer conn.Close() 109 109 + defer func() { 110 110 + _ = conn.WriteControl( 111 111 + websocket.CloseMessage, 112 112 + websocket.FormatCloseMessage(websocket.CloseNormalClosure, "log stream complete"), 113 113 + time.Now().Add(time.Second), 114 114 + ) 115 115 + conn.Close() 116 116 + }() 109 117 l.Debug("upgraded http to wss") 110 118 111 119 ctx, cancel := context.WithCancel(r.Context()) ··· 122 130 }() 123 131 124 132 if err := s.streamLogsFromDisk(ctx, conn, wid); err != nil { 125 125 - l.Error("log stream failed", "err", err) 133 133 + l.Info("log stream ended", "err", err) 126 134 } 127 127 - l.Debug("logs connection closed") 135 135 + 136 136 + l.Info("logs connection closed") 128 137 } 129 138 130 139 func (s *Spindle) streamLogsFromDisk(ctx context.Context, conn *websocket.Conn, wid models.WorkflowId) error { 140 140 + status, err := s.db.GetStatus(wid) 141 141 + if err != nil { 142 142 + return err 143 143 + } 144 144 + isFinished := models.StatusKind(status.Status).IsFinish() 145 145 + 131 146 filePath := engine.LogFilePath(s.cfg.Pipelines.LogDir, wid) 132 147 133 148 config := tail.Config{ 134 134 - Follow: true, 135 135 - ReOpen: true, 149 149 + Follow: !isFinished, 150 150 + ReOpen: !isFinished, 136 151 MustExist: false, 137 137 - Location: &tail.SeekInfo{Offset: 0, Whence: 0}, 138 138 - Logger: tail.DiscardingLogger, 152 152 + Location: &tail.SeekInfo{ 153 153 + Offset: 0, 154 154 + Whence: io.SeekStart, 155 155 + }, 156 156 + // Logger: tail.DiscardingLogger, 139 157 } 140 158 141 159 t, err := tail.TailFile(filePath, config) ··· 149 167 case <-ctx.Done(): 150 168 return ctx.Err() 151 169 case line := <-t.Lines: 170 170 + if line == nil && isFinished { 171 171 + return fmt.Errorf("tail completed") 172 172 + } 173 173 + 152 174 if line == nil { 153 175 return fmt.Errorf("tail channel closed unexpectedly") 154 176 }