spindle: add artifact store #3

api/tangled/cbor_gen.go

Reviewed

This file has not been changed.

api/tangled/tangledpipeline.go

Reviewed

This file has not been changed.

buf.gen.yaml

Reviewed

This file has not been changed.

buf.yaml

Reviewed

This file has not been changed.

cmd/spindle/main.go

Reviewed

This file has not been changed.

+10 -1

docker-compose.mill.yml

Reviewed

··· 25 25 SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev 26 26 SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images 27 27 SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays 28 28 - SPINDLE_S3_LOG_BUCKET: "" 28 28 + SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts 29 29 + SPINDLE_MILL_ARTIFACT_STORE: disk 29 30 SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" 30 31 SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 31 32 SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= ··· 69 70 spindle: 70 71 environment: 71 72 SPINDLE_ROLE: mill 73 73 + SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts 74 74 + SPINDLE_MILL_ARTIFACT_STORE: disk 75 75 + volumes: 76 76 + - spindle-artifacts:/var/lib/spindle/artifacts 72 77 73 78 mill-tokens: 74 79 profiles: ["linux"] ··· 110 115 SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11241" 111 116 volumes: 112 117 - spindle-executor-a-data:/var/lib/spindle 118 118 + - spindle-artifacts:/var/lib/spindle/artifacts 113 119 - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro 114 120 - init-state:/shared:ro 115 121 - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro ··· 124 130 SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11242" 125 131 volumes: 126 132 - spindle-executor-b-data:/var/lib/spindle 133 133 + - spindle-artifacts:/var/lib/spindle/artifacts 127 134 - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro 128 135 - init-state:/shared:ro 129 136 - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro ··· 138 145 SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11243" 139 146 volumes: 140 147 - spindle-executor-c-data:/var/lib/spindle 148 148 + - spindle-artifacts:/var/lib/spindle/artifacts 141 149 - ./out/localinfra-spindle-images-alpine:/var/lib/spindle/images:ro 142 150 - init-state:/shared:ro 143 151 - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro ··· 146 154 spindle-executor-a-data: 147 155 spindle-executor-b-data: 148 156 spindle-executor-c-data: 157 157 + spindle-artifacts:

eventstream/eventstream_test.go

Reviewed

This file has not been changed.

eventstream/store.go

Reviewed

This file has not been changed.

lexicons/pipeline/pipeline.json

Reviewed

This file has not been changed.

localinfra/readme.md

Reviewed

This file has not been changed.

localinfra/scripts/prepare-spindle-images.sh

Reviewed

This file has not been changed.

localinfra/spindle.Dockerfile

Reviewed

This file has not been changed.

netutil/ssrf.go

Reviewed

This file has not been changed.

+19 -3

spindle/config/config.go

Reviewed

··· 58 58 MaxConcurrentWorkflows int `env:"MAX_CONCURRENT_WORKFLOWS, default=8"` // max number of workflow containers running at once (memory cap) 59 59 } 60 60 61 61 - type S3 struct { 61 61 + type ArtifactStoreDisk struct { 62 62 + Dir string `env:"DIR"` 63 63 + } 64 64 + 65 65 + type ArtifactStoreS3 struct { 66 66 + Bucket string `env:"BUCKET"` 67 67 + Region string `env:"REGION, default=us-east-1"` 68 68 + } 69 69 + 70 70 + type ArtifactStores struct { 71 71 + Disk ArtifactStoreDisk `env:",prefix=DISK_"` 72 72 + S3 ArtifactStoreS3 `env:",prefix=S3_"` 73 73 + } 74 74 + 75 75 + type LegacyS3 struct { 62 76 LogBucket string `env:"LOG_BUCKET"` 63 77 } 64 78 65 79 type MicroVMPipelines struct { 66 80 ImageDir string `env:"IMAGE_DIR"` 67 67 - OverlayDir string `env:"OVERLAY_DIR, default="` // where microVM temporary disks will live 81 81 + OverlayDir string `env:"OVERLAY_DIR"` // where microVM temporary disks will live 68 82 DefaultImage string `env:"DEFAULT_IMAGE, default=nixos-x86_64"` 69 83 AgentPort uint32 `env:"AGENT_PORT, default=10240"` 70 84 EnableKVM bool `env:"ENABLE_KVM, default=true"` ··· 111 125 ReconnectGrace time.Duration `env:"RECONNECT_GRACE, default=45s"` // reconnect window before leases are failed 112 126 Seats int `env:"SEATS, default=4"` // executor seats advertised to the mill 113 127 Labels []string `env:"LABELS"` // executor capability labels 128 128 + ArtifactStore string `env:"ARTIFACT_STORE"` // store shared by mill and its executors 114 129 } 115 130 116 131 type Config struct { ··· 119 134 NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` 120 135 MicroVMPipelines MicroVMPipelines `env:",prefix=SPINDLE_MICROVM_PIPELINES_"` 121 136 NixCache NixCache `env:",prefix=SPINDLE_NIX_CACHE_"` 122 122 - S3 S3 `env:",prefix=SPINDLE_S3_"` 137 137 + ArtifactStores ArtifactStores `env:",prefix=SPINDLE_ARTIFACT_STORES_"` 138 138 + LegacyS3 LegacyS3 `env:",prefix=SPINDLE_S3_"` 123 139 Mill Mill `env:",prefix=SPINDLE_MILL_"` 124 140 } 125 141

+17 -9

spindle/db/db.go

Reviewed

··· 135 135 knot text not null, 136 136 rkey text not null, 137 137 workflow text not null, 138 138 - state text not null, 139 139 - created_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) 138 138 + state text not null 140 139 ); 141 140 142 141 create table if not exists mill_executor_cursors ( ··· 162 161 foreign key (epoch) references mill_outbox_state(epoch) on delete cascade 163 162 ); 164 163 165 165 - create table if not exists mill_canonical_logs ( 166 166 - node_id text not null, 167 167 - epoch text not null, 168 168 - seqno integer not null, 169 169 - workflow text not null, 170 170 - payload blob not null, 171 171 - primary key (node_id, epoch, seqno) 164 164 + create table if not exists mill_artifacts ( 165 165 + id integer primary key autoincrement, 166 166 + lease_id text not null, 167 167 + workflow text not null, 168 168 + ref text not null, 169 169 + hash text not null 170 170 + ); 171 171 + 172 172 + create table if not exists executor_pending_artifacts ( 173 173 + lease_id text primary key, 174 174 + workflow text not null, 175 175 + status text not null, 176 176 + error text not null default '', 177 177 + exit_code integer not null default 0, 178 178 + ref text not null, 179 179 + hash text not null 172 180 ); 173 181 174 182 create table if not exists migrations (

spindle/db/events.go

Reviewed

This file has not been changed.

+86 -128

spindle/db/mill_state.go

Reviewed

··· 1 1 package db 2 2 3 3 import ( 4 4 - "context" 5 4 "database/sql" 6 5 "fmt" 7 7 - "time" 8 6 9 7 "tangled.org/core/eventstream" 10 8 "tangled.org/core/notifier" ··· 36 34 Control bool 37 35 } 38 36 type OutboxDeletion struct { 39 39 - Rows int64 40 40 - LogBytes int64 41 41 - ControlBytes int64 37 37 + Rows int64 38 38 + Bytes int64 42 39 } 43 40 44 44 - type CanonicalLog struct { 45 45 - NodeID string 46 46 - Epoch string 47 47 - Seqno uint64 48 48 - Workflow string 49 49 - Payload []byte 50 50 - } 51 51 - 52 41 func (d *DB) SaveMillLease(l MillLease) error { 53 42 _, err := d.Exec( 54 43 `insert into mill_leases ( ··· 88 77 return leases, rows.Err() 89 78 } 90 79 91 91 - func (d *DB) SetExecutorCursor(nodeID, epoch string, seqno uint64) error { 92 92 - _, err := d.Exec( 93 93 - `insert into mill_executor_cursors (node_id, epoch, acked_seqno) values (?, ?, ?) 94 94 - on conflict(node_id, epoch) do update set acked_seqno = excluded.acked_seqno`, 95 95 - nodeID, epoch, seqno, 96 96 - ) 97 97 - return err 98 98 - } 99 99 - 100 100 - func (d *DB) DeleteExecutorCursor(nodeID, epoch string) error { 101 101 - _, err := d.Exec(`delete from mill_executor_cursors where node_id = ? and epoch = ?`, nodeID, epoch) 102 102 - return err 103 103 - } 104 104 - 105 80 func (d *DB) ListExecutorCursors() ([]ExecutorCursor, error) { 106 81 rows, err := d.Query(`select node_id, epoch, acked_seqno from mill_executor_cursors`) 107 82 if err != nil { ··· 200 175 var deleted OutboxDeletion 201 176 if err := tx.QueryRow(` 202 177 select count(*), 203 203 - coalesce(sum(case when control = 0 then byte_size else 0 end), 0), 204 204 - coalesce(sum(case when control != 0 then byte_size else 0 end), 0) 178 178 + coalesce(sum(byte_size), 0) 205 179 from mill_outbox_rows 206 180 where epoch = ? and seqno <= ? 207 207 - `, epoch, ackedSeqno).Scan(&deleted.Rows, &deleted.LogBytes, &deleted.ControlBytes); err != nil { 181 181 + `, epoch, ackedSeqno).Scan(&deleted.Rows, &deleted.Bytes); err != nil { 208 182 return OutboxDeletion{}, err 209 183 } 210 184 if _, err := tx.Exec( ··· 275 249 return epoch, nextSeqno, err 276 250 } 277 251 278 278 - func (d *DB) listCanonicalLogs(where string, args ...any) ([]CanonicalLog, error) { 279 279 - rows, err := d.Query(` 280 280 - select node_id, epoch, seqno, workflow, payload 281 281 - from mill_canonical_logs 282 282 - `+where+` 283 283 - order by workflow, node_id, epoch, seqno 284 284 - `, args...) 285 285 - if err != nil { 286 286 - return nil, err 287 287 - } 288 288 - defer rows.Close() 289 289 - 290 290 - var logs []CanonicalLog 291 291 - for rows.Next() { 292 292 - var log CanonicalLog 293 293 - if err := rows.Scan(&log.NodeID, &log.Epoch, &log.Seqno, &log.Workflow, &log.Payload); err != nil { 294 294 - return nil, err 295 295 - } 296 296 - logs = append(logs, log) 297 297 - } 298 298 - return logs, rows.Err() 252 252 + type FinishedLog struct { 253 253 + LeaseID string 254 254 + Workflow string 255 255 + Ref string 256 256 + Hash string 299 257 } 300 258 301 301 - func (d *DB) ListCanonicalLogs() ([]CanonicalLog, error) { 302 302 - return d.listCanonicalLogs("") 303 303 - } 304 304 - 305 305 - func (d *DB) ListCanonicalLogsFor(workflow string) ([]CanonicalLog, error) { 306 306 - return d.listCanonicalLogs(`where workflow = ?`, workflow) 307 307 - } 308 308 - 309 309 - // rows after a (node_id, epoch, seqno) position, so a follower can resume 310 310 - // without replaying 311 311 - func (d *DB) ListCanonicalLogsAfter(workflow, nodeID, epoch string, seqno uint64, limit int) ([]CanonicalLog, error) { 312 312 - rows, err := d.Query(` 313 313 - select node_id, epoch, seqno, workflow, payload 314 314 - from mill_canonical_logs 315 315 - where workflow = ? and (node_id, epoch, seqno) > (?, ?, ?) 316 316 - order by node_id, epoch, seqno 317 317 - limit ? 318 318 - `, workflow, nodeID, epoch, seqno, limit) 259 259 + func (d *DB) GetFinishedLog(workflow string) (*FinishedLog, error) { 260 260 + var fl FinishedLog 261 261 + err := d.QueryRow( 262 262 + `select lease_id, workflow, ref, hash 263 263 + from mill_artifacts 264 264 + where workflow = ? 265 265 + order by id desc limit 1`, 266 266 + workflow, 267 267 + ).Scan(&fl.LeaseID, &fl.Workflow, &fl.Ref, &fl.Hash) 319 268 if err != nil { 320 269 return nil, err 321 270 } 322 322 - defer rows.Close() 323 323 - 324 324 - var logs []CanonicalLog 325 325 - for rows.Next() { 326 326 - var log CanonicalLog 327 327 - if err := rows.Scan(&log.NodeID, &log.Epoch, &log.Seqno, &log.Workflow, &log.Payload); err != nil { 328 328 - return nil, err 329 329 - } 330 330 - logs = append(logs, log) 331 331 - } 332 332 - return logs, rows.Err() 271 271 + return &fl, nil 333 272 } 334 273 335 335 - // streams a workflow's canonical payloads as they arrive, until ctx 336 336 - // ends. query errors retry on the next tick, the table is the live log 337 337 - // while a job runs 338 338 - func (d *DB) FollowCanonicalLog(ctx context.Context, workflow string) <-chan []byte { 339 339 - out := make(chan []byte, 64) 340 340 - go func() { 341 341 - defer close(out) 342 342 - var nodeID, epoch string 343 343 - var seqno uint64 344 344 - for { 345 345 - logs, err := d.ListCanonicalLogsAfter(workflow, nodeID, epoch, seqno, 256) 346 346 - if err == nil { 347 347 - for _, log := range logs { 348 348 - select { 349 349 - case out <- log.Payload: 350 350 - nodeID, epoch, seqno = log.NodeID, log.Epoch, log.Seqno 351 351 - case <-ctx.Done(): 352 352 - return 353 353 - } 354 354 - } 355 355 - if len(logs) == 256 { 356 356 - continue 357 357 - } 358 358 - } 359 359 - select { 360 360 - case <-time.After(500 * time.Millisecond): 361 361 - case <-ctx.Done(): 362 362 - return 363 363 - } 364 364 - } 365 365 - }() 366 366 - return out 367 367 - } 368 368 - 369 369 - func (d *DB) DeleteCanonicalLogs(workflow string) error { 370 370 - _, err := d.Exec(`delete from mill_canonical_logs where workflow = ?`, workflow) 371 371 - return err 372 372 - } 373 373 - 374 274 type EventBatchTx struct { 375 275 tx *sql.Tx 376 276 db *DB ··· 384 284 return eventstream.Insert(tx.tx, event, nil) 385 285 } 386 286 387 387 - func (tx *EventBatchTx) AddCanonicalLog(nodeID, epoch string, seqno uint64, workflow string, payload []byte) error { 388 388 - _, err := tx.tx.Exec( 389 389 - `insert into mill_canonical_logs (node_id, epoch, seqno, workflow, payload) values (?, ?, ?, ?, ?) 390 390 - on conflict(node_id, epoch, seqno) do nothing`, 391 391 - nodeID, epoch, seqno, workflow, payload, 392 392 - ) 393 393 - return err 394 394 - } 395 395 - 396 287 func (tx *EventBatchTx) DeleteLease(leaseID string) error { 397 288 _, err := tx.tx.Exec(`delete from mill_leases where lease_id = ?`, leaseID) 398 289 return err ··· 405 296 nodeID, epoch, seqno, 406 297 ) 407 298 return err 299 299 + } 300 300 + 301 301 + func (tx *EventBatchTx) InsertArtifactRef(leaseID, workflow, ref, hash string) error { 302 302 + _, err := tx.tx.Exec( 303 303 + `insert into mill_artifacts (lease_id, workflow, ref, hash) 304 304 + values (?, ?, ?, ?)`, 305 305 + leaseID, workflow, ref, hash, 306 306 + ) 307 307 + return err 308 308 + } 309 309 + 310 310 + func (d *DB) SaveArtifactRef(leaseID, workflow, ref, hash string) error { 311 311 + _, err := d.Exec( 312 312 + `insert into mill_artifacts (lease_id, workflow, ref, hash) 313 313 + values (?, ?, ?, ?)`, 314 314 + leaseID, workflow, ref, hash, 315 315 + ) 316 316 + return err 317 317 + } 318 318 + 319 319 + type PendingArtifact struct { 320 320 + LeaseID string 321 321 + Workflow string 322 322 + Status string 323 323 + Error string 324 324 + ExitCode int64 325 325 + Ref string 326 326 + Hash string 327 327 + } 328 328 + 329 329 + func (d *DB) SavePendingArtifact(leaseID, workflow, status, errStr string, exitCode int64, ref, hash string) error { 330 330 + _, err := d.Exec( 331 331 + `insert into executor_pending_artifacts (lease_id, workflow, status, error, exit_code, ref, hash) 332 332 + values (?, ?, ?, ?, ?, ?, ?) 333 333 + on conflict(lease_id) do update set 334 334 + workflow = excluded.workflow, 335 335 + status = excluded.status, 336 336 + error = excluded.error, 337 337 + exit_code = excluded.exit_code, 338 338 + ref = excluded.ref, 339 339 + hash = excluded.hash`, 340 340 + leaseID, workflow, status, errStr, exitCode, ref, hash, 341 341 + ) 342 342 + return err 343 343 + } 344 344 + 345 345 + func (d *DB) RemovePendingArtifact(leaseID string) error { 346 346 + _, err := d.Exec(`delete from executor_pending_artifacts where lease_id = ?`, leaseID) 347 347 + return err 348 348 + } 349 349 + 350 350 + func (d *DB) ListPendingArtifacts() ([]PendingArtifact, error) { 351 351 + rows, err := d.Query(`select lease_id, workflow, status, error, exit_code, ref, hash from executor_pending_artifacts`) 352 352 + if err != nil { 353 353 + return nil, err 354 354 + } 355 355 + defer rows.Close() 356 356 + 357 357 + var res []PendingArtifact 358 358 + for rows.Next() { 359 359 + var p PendingArtifact 360 360 + if err := rows.Scan(&p.LeaseID, &p.Workflow, &p.Status, &p.Error, &p.ExitCode, &p.Ref, &p.Hash); err != nil { 361 361 + return nil, err 362 362 + } 363 363 + res = append(res, p) 364 364 + } 365 365 + return res, rows.Err() 408 366 } 409 367 410 368 func (d *DB) ApplyEventBatch(n *notifier.Notifier, fn func(tx *EventBatchTx) error) error {

+16 -70

spindle/db/mill_state_test.go

Reviewed

··· 53 53 func TestExecutorCursors(t *testing.T) { 54 54 d := newTestDB(t) 55 55 56 56 - if err := d.SetExecutorCursor("node-1", "inc-1", 5); err != nil { 57 57 - t.Fatalf("SetExecutorCursor: %v", err) 56 56 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 5) }); err != nil { 57 57 + t.Fatalf("AdvanceCursor: %v", err) 58 58 } 59 59 - if err := d.SetExecutorCursor("node-1", "inc-1", 9); err != nil { 60 60 - t.Fatalf("SetExecutorCursor(advance): %v", err) 59 59 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 9) }); err != nil { 60 60 + t.Fatalf("AdvanceCursor(advance): %v", err) 61 61 } 62 62 - if err := d.SetExecutorCursor("node-2", "inc-1", 1); err != nil { 63 63 - t.Fatalf("SetExecutorCursor(node-2): %v", err) 62 62 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-2", "inc-1", 1) }); err != nil { 63 63 + t.Fatalf("AdvanceCursor(node-2): %v", err) 64 64 } 65 65 66 66 cursors, err := d.ListExecutorCursors() ··· 80 80 t.Fatalf("unexpected cursors: %v", cursorMap) 81 81 } 82 82 83 83 - if err := d.DeleteExecutorCursor("node-1", "inc-1"); err != nil { 84 84 - t.Fatalf("DeleteExecutorCursor: %v", err) 85 85 - } 86 86 - cursors, _ = d.ListExecutorCursors() 87 87 - if len(cursors) != 1 || cursors[0].NodeID != "node-2" { 88 88 - t.Fatalf("ListExecutorCursors after delete = %v, want only node-2", cursors) 89 89 - } 90 83 } 91 84 92 85 func TestCompleteMillLeaseIsAtomic(t *testing.T) { ··· 185 178 t.Fatalf("SaveMillLease: %v", err) 186 179 } 187 180 188 188 - if err := d.SetExecutorCursor("node-p", "inc-p", 42); err != nil { 189 189 - t.Fatalf("SetExecutorCursor: %v", err) 181 181 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-p", "inc-p", 42) }); err != nil { 182 182 + t.Fatalf("AdvanceCursor: %v", err) 190 183 } 191 184 192 185 if err := d.SetOutboxEpoch("inc-p"); err != nil { ··· 196 189 t.Fatalf("AppendOutboxRow: %v", err) 197 190 } 198 191 199 199 - if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { 200 200 - return tx.AddCanonicalLog("node-p", "inc-p", 1, "w", []byte("log content")) 201 201 - }); err != nil { 202 202 - t.Fatalf("AddCanonicalLog: %v", err) 203 203 - } 204 204 - 205 192 if err := d.Close(); err != nil { 206 193 t.Fatalf("Close: %v", err) 207 194 } ··· 242 229 if len(rows) != 1 || string(rows[0].Payload) != "hello world" || !rows[0].Control { 243 230 t.Fatalf("unexpected outbox rows: %+v", rows) 244 231 } 245 245 - 246 246 - logs, err := d2.ListCanonicalLogs() 247 247 - if err != nil { 248 248 - t.Fatalf("ListCanonicalLogs: %v", err) 249 249 - } 250 250 - if len(logs) != 1 || string(logs[0].Payload) != "log content" { 251 251 - t.Fatalf("unexpected canonical logs: %+v", logs) 252 252 - } 253 232 } 254 233 255 234 func TestOutboxPrefixAck(t *testing.T) { ··· 276 255 if err != nil { 277 256 t.Fatalf("DeleteOutboxPrefix: %v", err) 278 257 } 279 279 - if n.Rows != 2 || n.LogBytes != 8 || n.ControlBytes != 0 { 280 280 - t.Fatalf("deleted prefix = %+v, want 2 rows and 8 log bytes", n) 258 258 + if n.Rows != 2 || n.Bytes != 8 { 259 259 + t.Fatalf("deleted prefix = %+v, want 2 rows and 8 bytes", n) 281 260 } 282 261 283 262 rows, err := d.ListOutboxRows() ··· 292 271 func TestCompositeCursors(t *testing.T) { 293 272 d := newTestDB(t) 294 273 295 295 - if err := d.SetExecutorCursor("node-1", "inc-1", 10); err != nil { 296 296 - t.Fatalf("SetExecutorCursor: %v", err) 274 274 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 10) }); err != nil { 275 275 + t.Fatalf("AdvanceCursor: %v", err) 297 276 } 298 298 - if err := d.SetExecutorCursor("node-1", "inc-2", 20); err != nil { 299 299 - t.Fatalf("SetExecutorCursor: %v", err) 277 277 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-2", 20) }); err != nil { 278 278 + t.Fatalf("AdvanceCursor: %v", err) 300 279 } 301 301 - if err := d.SetExecutorCursor("node-2", "inc-1", 5); err != nil { 302 302 - t.Fatalf("SetExecutorCursor: %v", err) 280 280 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-2", "inc-1", 5) }); err != nil { 281 281 + t.Fatalf("AdvanceCursor: %v", err) 303 282 } 304 283 305 284 cursors, err := d.ListExecutorCursors() ··· 320 299 t.Fatalf("unexpected cursor values: %v", cursorMap) 321 300 } 322 301 323 323 - if err := d.DeleteExecutorCursor("node-1", "inc-1"); err != nil { 324 324 - t.Fatalf("DeleteExecutorCursor: %v", err) 325 325 - } 326 326 - 327 327 - cursors, err = d.ListExecutorCursors() 328 328 - if err != nil { 329 329 - t.Fatalf("ListExecutorCursors: %v", err) 330 330 - } 331 331 - if len(cursors) != 2 { 332 332 - t.Fatalf("expected 2 cursors, got %d", len(cursors)) 333 333 - } 334 302 } 335 303 336 304 func TestBatchRollback(t *testing.T) { ··· 422 390 case <-notifications: 423 391 default: 424 392 t.Fatal("notifier was not fired after batch commit") 425 425 - } 426 426 - } 427 427 - 428 428 - func TestCanonicalLogDedup(t *testing.T) { 429 429 - d := newTestDB(t) 430 430 - add := func(payload string) error { 431 431 - return d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { 432 432 - return tx.AddCanonicalLog("node-1", "inc-1", 10, "workflow-1", []byte(payload)) 433 433 - }) 434 434 - } 435 435 - if err := add("first content"); err != nil { 436 436 - t.Fatalf("first AddCanonicalLog: %v", err) 437 437 - } 438 438 - if err := add("second content"); err != nil { 439 439 - t.Fatalf("second AddCanonicalLog: %v", err) 440 440 - } 441 441 - logs, err := d.ListCanonicalLogs() 442 442 - if err != nil { 443 443 - t.Fatalf("ListCanonicalLogs: %v", err) 444 444 - } 445 445 - if len(logs) != 1 || string(logs[0].Payload) != "first content" { 446 446 - t.Fatalf("canonical logs = %+v, want original payload", logs) 447 393 } 448 394 }

spindle/db/mill_tokens.go

Reviewed

This file has not been changed.

spindle/db/mill_tokens_test.go

Reviewed

This file has not been changed.

+43 -18

spindle/engine/engine.go

Reviewed

··· 2 2 3 3 import ( 4 4 "context" 5 5 + "crypto/sha256" 6 6 + "encoding/hex" 5 7 "errors" 6 6 - "fmt" 8 8 + "io" 7 9 "log/slog" 8 8 - "path/filepath" 10 10 + "os" 9 11 "sync" 12 12 + "time" 10 13 11 14 "tangled.org/core/notifier" 15 15 + "tangled.org/core/spindle/artifactstore" 12 16 "tangled.org/core/spindle/config" 13 17 "tangled.org/core/spindle/db" 14 18 "tangled.org/core/spindle/models" ··· 51 55 } 52 56 } 53 57 54 54 - func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { 58 58 + func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, stores *artifactstore.Stores, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { 55 59 l.Info("starting all workflows in parallel", "pipeline", pipelineId) 56 60 57 61 var allSecrets []secrets.UnlockedSecret ··· 69 73 secretValues[i] = s.Value 70 74 } 71 75 72 72 - s3, err := NewS3(cfg.S3.LogBucket) 73 73 - if err != nil { 74 74 - l.Error("error creating s3 client", "err", err) 75 75 - } 76 76 - 77 76 var wg sync.WaitGroup 78 77 for eng, wfs := range pipeline.Workflows { 79 78 workflowTimeout := eng.WorkflowTimeout() ··· 86 85 Name: w.Name, 87 86 } 88 87 89 89 - defer func() { 90 90 - if s3 != nil { 91 91 - logFile := filepath.Join(cfg.Server.LogDir, fmt.Sprintf("%s.log", wid.String())) 92 92 - if err := s3.WriteFile(ctx, logFile); err != nil { 93 93 - l.Error("error uploading logs", "err", err) 94 94 - } 95 95 - } 96 96 - }() 97 97 - 88 88 + var err error 98 89 var wfLogger models.WorkflowLogger 99 90 if p, ok := eng.(workflowLoggerProvider); ok { 100 91 wfLogger = p.WorkflowLogger(wid) ··· 104 95 } else { 105 96 l.Info("setup step logger; logs will be persisted", "logDir", cfg.Server.LogDir, "wid", wid) 106 97 wfLogger = fileLogger 98 98 + defer archiveWorkflowLog(l, stores, db, cfg.Server.LogDir, wid) 107 99 defer fileLogger.Close() 108 100 } 109 101 ··· 112 104 _, remoteStatus := eng.(RemoteStatusEngine) 113 105 114 106 if s, ok := eng.(WorkflowSlotter); ok { 115 115 - var err error 116 107 slot, err = s.AcquireWorkflowSlot(ctx, wid, &w, Wait) 117 108 if err != nil { 118 109 l.Error("failed to acquire slot", "wid", wid, "err", err) ··· 188 179 189 180 wg.Wait() 190 181 l.Info("all workflows completed") 182 182 + } 183 183 + 184 184 + func archiveWorkflowLog(l *slog.Logger, stores *artifactstore.Stores, database *db.DB, logDir string, wid models.WorkflowId) { 185 185 + if stores == nil { 186 186 + return 187 187 + } 188 188 + logPath := models.LogFilePath(logDir, wid) 189 189 + file, err := os.Open(logPath) 190 190 + if err != nil { 191 191 + l.Error("open workflow log for archival", "wid", wid, "err", err) 192 192 + return 193 193 + } 194 194 + hash := sha256.New() 195 195 + if _, err := io.Copy(hash, file); err != nil { 196 196 + _ = file.Close() 197 197 + l.Error("hash workflow log", "wid", wid, "err", err) 198 198 + return 199 199 + } 200 200 + _ = file.Close() 201 201 + 202 202 + ref := wid.String() + ".log" 203 203 + uploadCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) 204 204 + defer cancel() 205 205 + errs := stores.PutFile(uploadCtx, ref, logPath) 206 206 + for _, err := range errs { 207 207 + l.Error("archive workflow log", "wid", wid, "err", err) 208 208 + } 209 209 + if len(errs) == len(stores.Names()) { 210 210 + return 211 211 + } 212 212 + digest := "sha256:" + hex.EncodeToString(hash.Sum(nil)) 213 213 + if err := database.SaveArtifactRef(wid.String(), wid.Name, ref, digest); err != nil { 214 214 + l.Error("save workflow log artifact", "wid", wid, "err", err) 215 215 + } 191 216 }

+28 -7

spindle/engine/engine_test.go

Reviewed

··· 13 13 "github.com/bluesky-social/indigo/atproto/syntax" 14 14 "tangled.org/core/api/tangled" 15 15 "tangled.org/core/notifier" 16 16 + "tangled.org/core/spindle/artifactstore" 16 17 "tangled.org/core/spindle/config" 17 18 "tangled.org/core/spindle/db" 18 19 "tangled.org/core/spindle/models" ··· 147 148 148 149 t.Run("success path", func(t *testing.T) { 149 150 eng := &mockEngine{} 151 151 + store, err := artifactstore.NewStores(config.ArtifactStores{}, cfg.Server.LogDir, "") 152 152 + if err != nil { 153 153 + t.Fatal(err) 154 154 + } 150 155 pipeline := &models.Pipeline{ 151 156 RepoDid: syntax.DID("did:plc:testrepo"), 152 157 Workflows: map[models.Engine][]models.Workflow{ ··· 160 165 TrustedSource: true, 161 166 } 162 167 163 163 - StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId) 168 168 + StartWorkflows(l, vault, cfg, store, database, &n, context.Background(), pipeline, pipelineId) 164 169 165 170 wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci"} 166 171 statuses := getWorkflowStatuses(t, database, wid) ··· 173 178 t.Errorf("status[%d] = %s, want %s", i, s, want[i]) 174 179 } 175 180 } 181 181 + 182 182 + finished, err := database.GetFinishedLog(wid.Name) 183 183 + if err != nil { 184 184 + t.Fatalf("GetFinishedLog: %v", err) 185 185 + } 186 186 + if finished.Ref != wid.String()+".log" { 187 187 + t.Fatalf("artifact ref = %q", finished.Ref) 188 188 + } 189 189 + log, err := store.Open(context.Background(), finished.Ref) 190 190 + if err != nil { 191 191 + t.Fatalf("open archived log: %v", err) 192 192 + } 193 193 + defer log.Close() 194 194 + if contents, err := io.ReadAll(log); err != nil || len(contents) == 0 { 195 195 + t.Fatalf("archived log is empty: %v", err) 196 196 + } 176 197 }) 177 198 178 199 t.Run("setup failure", func(t *testing.T) { ··· 190 211 TrustedSource: true, 191 212 } 192 213 193 193 - StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId) 214 214 + StartWorkflows(l, vault, cfg, nil, database, &n, context.Background(), pipeline, pipelineId) 194 215 195 216 wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-setup-fail"} 196 217 statuses := getWorkflowStatuses(t, database, wid) ··· 220 241 TrustedSource: true, 221 242 } 222 243 223 223 - StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId) 244 244 + StartWorkflows(l, vault, cfg, nil, database, &n, context.Background(), pipeline, pipelineId) 224 245 225 246 wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-run-fail"} 226 247 statuses := getWorkflowStatuses(t, database, wid) ··· 262 283 TrustedSource: true, 263 284 } 264 285 265 265 - StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId) 286 286 + StartWorkflows(l, vault, cfg, nil, database, &n, context.Background(), pipeline, pipelineId) 266 287 267 288 wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-remote-acquire-fail"} 268 289 statuses := getWorkflowStatuses(t, database, wid) ··· 286 307 TrustedSource: true, 287 308 } 288 309 289 289 - StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId) 310 310 + StartWorkflows(l, vault, cfg, nil, database, &n, context.Background(), pipeline, pipelineId) 290 311 291 312 wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-remote-success"} 292 313 statuses := getWorkflowStatuses(t, database, wid) ··· 310 331 TrustedSource: true, 311 332 } 312 333 313 313 - StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId) 334 334 + StartWorkflows(l, vault, cfg, nil, database, &n, context.Background(), pipeline, pipelineId) 314 335 315 336 wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-remote-setup-fail"} 316 337 statuses := getWorkflowStatuses(t, database, wid) ··· 334 355 TrustedSource: true, 335 356 } 336 357 337 337 - StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId) 358 358 + StartWorkflows(l, vault, cfg, nil, database, &n, context.Background(), pipeline, pipelineId) 338 359 339 360 wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-remote-run-fail"} 340 361 statuses := getWorkflowStatuses(t, database, wid)

spindle/engine/manifest_test.go

Reviewed

This file has not been changed.

spindle/engine/placement.go

Reviewed

This file has not been changed.

spindle/engine/scheduler.go

Reviewed

This file has not been changed.

spindle/engine/scheduler_test.go

Reviewed

This file has not been changed.

spindle/engine/slot.go

Reviewed

This file has not been changed.

spindle/engine/slot_test.go

Reviewed

This file has not been changed.

spindle/engines/dummy/engine.go

Reviewed

This file has not been changed.

spindle/engines/microvm/budget.go

Reviewed

This file has not been changed.

spindle/engines/microvm/engine.go

Reviewed

This file has not been changed.

spindle/engines/microvm/image.go

Reviewed

This file has not been changed.

spindle/engines/microvm/image_test.go

Reviewed

This file has not been changed.

spindle/engines/microvm/placement/placement.go

Reviewed

This file has not been changed.

spindle/engines/microvm/placement/placement_test.go

Reviewed

This file has not been changed.

spindle/engines/microvm/placement_linux.go

Reviewed

This file has not been changed.

spindle/engines/microvm/qemu.go

Reviewed

This file has not been changed.

spindle/engines/microvm/vm.go

Reviewed

This file has not been changed.

spindle/engines/nixery/engine.go

Reviewed

This file has not been changed.

+26 -16

spindle/logview/logview.go

Reviewed

··· 1 1 package logview 2 2 3 3 import ( 4 4 + "bufio" 4 5 "context" 5 6 "io" 6 7 "strings" 7 8 8 9 "github.com/hpcloud/tail" 10 10 + "tangled.org/core/spindle/artifactstore" 9 11 "tangled.org/core/spindle/db" 10 12 "tangled.org/core/spindle/models" 11 13 ) 12 14 13 13 - // streams a wf's log lines. a live mill job reads from the canonical 14 14 - // table, everything else (finished jobs, standalone spindles) tails the 15 15 - // cold log file. stop ends a live follow early, the channel closes when 16 16 - // the source drains or ctx ends 17 17 - func Follow(ctx context.Context, d *db.DB, logDir string, wid models.WorkflowId, mill bool, finished bool) (<-chan *tail.Line, func(), error) { 18 18 - if mill && !finished { 19 19 - followCtx, cancel := context.WithCancel(ctx) 20 20 - path := models.LogFilePath(logDir, wid) 21 21 - payloads := d.FollowCanonicalLog(followCtx, path) 22 22 - lines := make(chan *tail.Line, 64) 23 23 - go func() { 24 24 - defer close(lines) 25 25 - for p := range payloads { 26 26 - lines <- &tail.Line{Text: strings.TrimSuffix(string(p), "\n")} 15 15 + // streams a workflow's log lines: tails the log file while running, reads 16 16 + // the uploaded artifact once finished, same for every role. stop ends a 17 17 + // live follow early, the channel closes when the source drains or ctx ends 18 18 + func Follow(ctx context.Context, d *db.DB, reader artifactstore.Reader, logDir string, wid models.WorkflowId, finished bool) (<-chan *tail.Line, func(), error) { 19 19 + if finished && reader != nil && d != nil { 20 20 + if fl, err := d.GetFinishedLog(wid.Name); err == nil && fl.Ref != "" { 21 21 + rc, err := reader.Open(ctx, fl.Ref) 22 22 + if err == nil { 23 23 + ch := make(chan *tail.Line, 64) 24 24 + followCtx, cancel := context.WithCancel(ctx) 25 25 + go func() { 26 26 + defer close(ch) 27 27 + defer rc.Close() 28 28 + scanner := bufio.NewScanner(rc) 29 29 + for scanner.Scan() { 30 30 + select { 31 31 + case <-followCtx.Done(): 32 32 + return 33 33 + case ch <- &tail.Line{Text: strings.TrimSuffix(scanner.Text(), "\r")}: 34 34 + } 35 35 + } 36 36 + }() 37 37 + return ch, cancel, nil 27 38 } 28 28 - }() 29 29 - return lines, cancel, nil 39 39 + } 30 40 } 31 41 32 42 t, err := tail.TailFile(models.LogFilePath(logDir, wid), tail.Config{

spindle/mill/README.md

Reviewed

This file has not been changed.

-74

spindle/mill/auth_test.go

Reviewed

··· 234 234 } 235 235 } 236 236 237 237 - func TestOnLogEventOwnership(t *testing.T) { 238 238 - ctx := context.Background() 239 239 - l := discardLogger() 240 240 - dir := t.TempDir() 241 241 - bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 242 242 - if err != nil { 243 243 - t.Fatalf("db.Make: %v", err) 244 244 - } 245 245 - t.Cleanup(func() { bdb.Close() }) 246 246 - n := notifier.New() 247 247 - 248 248 - m := New(l, Config{LogDir: dir, ReconnectGrace: time.Minute}) 249 249 - m.Attach(bdb, &n) 250 250 - 251 251 - foreign := newLease("lease-foreign", "node-x", "inc-x", "dummy") 252 252 - foreign.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "foreign"}, Name: "build"} 253 253 - owned := newLease("lease-owned", "node-z", "inc-z", "dummy") 254 254 - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "owned"}, Name: "build"} 255 255 - m.mu.Lock() 256 256 - m.leases[foreign.id] = foreign 257 257 - m.leases[owned.id] = owned 258 258 - m.mu.Unlock() 259 259 - 260 260 - sessY := newSession("node-y", "inc-y", nil, nopEncoder(), l) 261 261 - m.attachSession(sessY) 262 262 - sessZ := newSession("node-z", "inc-z", nil, nopEncoder(), l) 263 263 - m.attachSession(sessZ) 264 264 - 265 265 - _ = m.onEventBatch(sessY, &millv1.EventBatch{ 266 266 - Epoch: sessY.epoch, 267 267 - Events: []*millv1.Event{ 268 268 - { 269 269 - Seqno: 1, 270 270 - LeaseId: foreign.id, 271 271 - Payload: &millv1.Event_LogLine{ 272 272 - LogLine: &millv1.LogLine{RawJson: []byte(`{"line":"forged"}`)}, 273 273 - }, 274 274 - }, 275 275 - }, 276 276 - }) 277 277 - // a forged foreign-lease line must not reach the table at all 278 278 - foreignLogs, err := bdb.ListCanonicalLogsFor(models.LogFilePath(dir, foreign.wid)) 279 279 - if err != nil { 280 280 - t.Fatal(err) 281 281 - } 282 282 - if len(foreignLogs) != 0 { 283 283 - t.Fatalf("log stream for a foreign lease wrote canonical rows; an executor forged another node's logs") 284 284 - } 285 285 - 286 286 - line := []byte(`{"line":"legit"}`) 287 287 - _ = m.onEventBatch(sessZ, &millv1.EventBatch{ 288 288 - Epoch: sessZ.epoch, 289 289 - Events: []*millv1.Event{ 290 290 - { 291 291 - Seqno: 1, 292 292 - LeaseId: owned.id, 293 293 - Payload: &millv1.Event_LogLine{ 294 294 - LogLine: &millv1.LogLine{RawJson: line}, 295 295 - }, 296 296 - }, 297 297 - }, 298 298 - }) 299 299 - ownedLogs, err := bdb.ListCanonicalLogsFor(models.LogFilePath(dir, owned.wid)) 300 300 - if err != nil { 301 301 - t.Fatalf("owned log stream did not reach the canonical table: %v", err) 302 302 - } 303 303 - if len(ownedLogs) != 1 { 304 304 - t.Fatalf("canonical rows = %d, want 1", len(ownedLogs)) 305 305 - } 306 306 - if want := string(line) + "\n"; string(ownedLogs[0].Payload) != want { 307 307 - t.Fatalf("owned log line = %q, want %q", string(ownedLogs[0].Payload), want) 308 308 - } 309 309 - } 310 310 - 311 237 func TestOnStatusEventOwnership(t *testing.T) { 312 238 ctx := context.Background() 313 239 l := discardLogger()

-23

spindle/mill/capability_test.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "testing" 5 5 - 6 6 - "tangled.org/core/spindle/models" 7 7 - ) 8 8 - 9 9 - func TestRankCandidatesIntersectsLabels(t *testing.T) { 10 10 - m := New(discardLogger(), Config{}) 11 11 - addCandidateSession(t, m, "wrong-region", []string{"region/us"}, 0.10, nil) 12 12 - addCandidateSession(t, m, "no-gpu", []string{"region/eu"}, 0.05, nil) 13 13 - addCandidateSession(t, m, "eligible", []string{"region/eu", "gpu"}, 0.70, nil) 14 14 - 15 15 - got := m.rankCandidates("dummy", []string{"region/eu", "gpu"}) 16 16 - assertRankedNodes(t, got, []string{"eligible"}) 17 17 - } 18 18 - 19 19 - func TestRequiredLabelsWithoutMillStateAreEmpty(t *testing.T) { 20 20 - if got := requiredLabels(&models.Workflow{}); got != nil { 21 21 - t.Fatalf("requiredLabels() = %v, want nil", got) 22 22 - } 23 23 - }

+5 -7

spindle/mill/engine.go

Reviewed

··· 13 13 14 14 // raw pipeline/workflow carried forward, executor runs the real InitWorkflow 15 15 type millWorkflowState struct { 16 16 - TargetEngine string 17 17 - RawWorkflow tangled.Pipeline_Workflow 18 18 - RawPipeline tangled.Pipeline 19 19 - Lease *RemoteLease 16 16 + RawWorkflow tangled.Pipeline_Workflow 17 17 + RawPipeline tangled.Pipeline 18 18 + Lease *RemoteLease 20 19 } 21 20 22 21 // stand-in for a real engine, registered under the real names ··· 42 41 Environment: map[string]string{}, 43 42 Steps: []models.Step{remoteStep{}}, 44 43 Data: &millWorkflowState{ 45 45 - TargetEngine: e.name, 46 46 - RawWorkflow: twf, 47 47 - RawPipeline: tpl, 44 44 + RawWorkflow: twf, 45 45 + RawPipeline: tpl, 48 46 }, 49 47 }, nil 50 48 }

spindle/mill/executor/capability_test.go

Reviewed

This file has not been changed.

+93 -111

spindle/mill/executor/executor.go

Reviewed

··· 5 5 "encoding/json" 6 6 "errors" 7 7 "fmt" 8 8 - "log/slog" 9 8 "maps" 10 9 "net/http" 11 10 "runtime" 12 12 - "strings" 13 11 "sync" 14 12 "time" 15 13 16 14 "github.com/bluesky-social/indigo/atproto/syntax" 15 15 + "log/slog" 16 16 + "strings" 17 17 18 18 "tangled.org/core/api/tangled" 19 19 + "tangled.org/core/netutil" 19 20 "tangled.org/core/notifier" 21 21 + "tangled.org/core/spindle/artifactstore" 20 22 "tangled.org/core/spindle/config" 21 23 "tangled.org/core/spindle/db" 22 24 "tangled.org/core/spindle/engine" 23 23 - "tangled.org/core/spindle/models" 24 24 - "tangled.org/core/util/netutil" 25 25 - 26 25 millproto "tangled.org/core/spindle/mill/proto" 27 26 millv1 "tangled.org/core/spindle/mill/proto/gen" 27 27 + "tangled.org/core/spindle/models" 28 28 ) 29 29 30 30 const ( ··· 46 46 n *notifier.Notifier 47 47 cfg *config.Config 48 48 l *slog.Logger 49 49 + writer artifactstore.Writer 49 50 50 50 - epoch string 51 51 - outboxLogBytes int64 52 52 - outboxControlBytes int64 53 53 - droppedLogs uint64 54 54 - maxLogBytes int64 55 55 - maxControlBytes int64 51 51 + epoch string 52 52 + outboxBytes int64 53 53 + maxOutboxBytes int64 // 10 MiB default outbox cap 56 54 57 57 - eventMu sync.Mutex // protects outbox operations 58 58 - sendMu sync.Mutex // serializes all writes to enc and sentSeqno 55 55 + eventMu sync.Mutex 56 56 + sendMu sync.Mutex 59 57 flushMu sync.Mutex 60 58 61 61 - sentSeqno uint64 62 62 - flushTimer *time.Timer 59 59 + sentSeqno uint64 63 60 64 61 connMu sync.Mutex 65 62 enc messageEncoder ··· 83 80 slot engine.WorkflowSlot 84 81 wf *models.Workflow 85 82 repoDid syntax.DID 83 83 + vault *memVault 86 84 87 85 committed bool 88 86 cancelled bool ··· 95 93 Encode(*millproto.Message) error 96 94 } 97 95 98 98 - func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *notifier.Notifier, l *slog.Logger) (*Executor, error) { 99 99 - seats := cfg.Mill.Seats 100 100 - if seats <= 0 { 101 101 - seats = defaultSeats 96 96 + func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *notifier.Notifier, l *slog.Logger, writers ...artifactstore.Writer) (*Executor, error) { 97 97 + seats := defaultSeats 98 98 + millURL := "" 99 99 + token := "" 100 100 + nodeID := "" 101 101 + var labels []string 102 102 + if cfg != nil { 103 103 + if cfg.Mill.Seats > 0 { 104 104 + seats = cfg.Mill.Seats 105 105 + } 106 106 + labels = normalizeLabels(cfg.Mill.Labels) 107 107 + millURL = cfg.Mill.URL 108 108 + token = cfg.Mill.SharedSecret 109 109 + nodeID = cfg.Server.Hostname 102 110 } 103 103 - labels := normalizeLabels(cfg.Mill.Labels) 104 111 if d == nil || n == nil { 105 112 return nil, fmt.Errorf("executor requires a database and notifier") 106 113 } 114 114 + var writer artifactstore.Writer 115 115 + if len(writers) > 0 { 116 116 + writer = writers[0] 117 117 + } 107 118 e := &Executor{ 108 108 - millURL: cfg.Mill.URL, 109 109 - token: cfg.Mill.SharedSecret, 110 110 - nodeID: cfg.Server.Hostname, 111 111 - seats: seats, 112 112 - labels: labels, 113 113 - engines: engines, 114 114 - db: d, 115 115 - n: n, 116 116 - cfg: cfg, 117 117 - l: l.With("component", "mill.executor"), 118 118 - active: make(map[string]*reservation), 119 119 - maxLogBytes: 100 * 1024 * 1024, // 100 MiB generous default log cap 120 120 - maxControlBytes: 10 * 1024 * 1024, // 10 MiB default control cap 119 119 + millURL: millURL, 120 120 + token: token, 121 121 + nodeID: nodeID, 122 122 + seats: seats, 123 123 + labels: labels, 124 124 + engines: engines, 125 125 + db: d, 126 126 + n: n, 127 127 + cfg: cfg, 128 128 + l: l.With("component", "mill.executor"), 129 129 + writer: writer, 130 130 + active: make(map[string]*reservation), 131 131 + maxOutboxBytes: 10 * 1024 * 1024, 121 132 } 122 133 if err := e.initOutbox(); err != nil { 123 134 return nil, fmt.Errorf("initialize executor outbox: %w", err) ··· 168 179 } 169 180 170 181 func (e *Executor) runSession(ctx context.Context) error { 171 171 - dev := e.cfg.Server.Dev 182 182 + dev := e.cfg == nil || e.cfg.Server.Dev 172 183 if _, err := netutil.EnforceWSSURL(e.millURL, dev); err != nil { 173 184 return fmt.Errorf("mill url: %w", err) 174 185 } ··· 209 220 if resume == nil { 210 221 return fmt.Errorf("expected resume, got something else") 211 222 } 212 212 - 213 223 if resume.GetEpoch() != e.epoch { 214 224 return fmt.Errorf("resume epoch mismatch: got %q, want %q", resume.GetEpoch(), e.epoch) 215 225 } ··· 248 258 go e.snapshotLoop(sessionCtx, enc) 249 259 return <-readErr 250 260 } 251 251 - 252 261 func (e *Executor) send(msg *millproto.Message) { 253 262 e.connMu.Lock() 254 263 enc := e.enc ··· 280 289 case msg.GetAck() != nil: 281 290 e.handleAck(msg.GetAck()) 282 291 default: 283 283 - e.l.Warn("executor received unexpected message") 292 292 + e.l.Warn("unhandled incoming message", "type", fmt.Sprintf("%T", msg)) 284 293 } 285 294 } 286 295 ··· 302 311 } 303 312 304 313 func (e *Executor) releaseReservation(cleanup func()) { 305 305 - cleanup() 314 314 + if cleanup != nil { 315 315 + cleanup() 316 316 + } 306 317 e.pushSnapshot() 307 318 } 319 319 + 308 320 func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { 309 321 reject := func(reason string, class millv1.RejectClass) { 310 322 e.sendReject(rs.GetLeaseId(), reason, class) ··· 339 351 reject("bad pipeline json", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 340 352 return 341 353 } 342 342 - // engines deref TriggerMetadata blindly. a pipeline without it gets a 343 343 - // reject, not a crashed executor 344 354 if tpl.TriggerMetadata == nil { 345 355 reject("pipeline missing trigger metadata", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 346 356 return ··· 360 370 return 361 371 } 362 372 } 363 363 - // the job skipped processPipeline, so inject the TANGLED_* env vars here 364 373 if wf.Environment == nil { 365 374 wf.Environment = make(map[string]string) 366 375 } 367 376 maps.Copy(wf.Environment, models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId)) 368 377 369 369 - // nowait because the executor doesn't queue locally, the mill owns the backlog 370 378 slot, err := slotter.AcquireWorkflowSlot(ctx, wid, wf, engine.NoWait) 371 379 if err != nil { 372 380 class := millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE ··· 394 402 e.snapshotMu.Lock() 395 403 e.mu.Lock() 396 404 e.active[res.leaseID] = res 397 397 - // start timer under lock to avoid publish races 398 405 res.ttlTimer = time.AfterFunc(ttlDuration(rs.GetTtlSeconds()), func() { e.expireReservation(res.leaseID) }) 399 406 e.mu.Unlock() 400 407 ··· 424 431 res.ttlTimer.Stop() 425 432 } 426 433 427 427 - // rooted in the lifecycle ctx, not the session, so the job survives reconnects 428 434 jobCtx, cancel := context.WithCancel(e.lifecycleCtx) 429 435 res.cancel = cancel 430 436 e.mu.Unlock() ··· 442 448 e.jobsWG.Add(1) 443 449 go func() { 444 450 defer e.jobsWG.Done() 445 445 - engine.StartWorkflows(e.l, vault, e.cfg, e.db, e.n, jobCtx, pipeline, res.wid.PipelineId) 451 451 + engine.StartWorkflows(e.l, vault, e.cfg, nil, e.db, e.n, jobCtx, pipeline, res.wid.PipelineId) 446 452 }() 447 453 448 454 e.sendCommitted(cl.GetLeaseId()) ··· 499 505 if !ok { 500 506 return 501 507 } 502 502 - e.l.Warn("reservation expired before commit", "lease", leaseID) 503 508 e.releaseReservation(cleanup) 504 509 } 505 510 ··· 508 513 defer e.mu.Unlock() 509 514 res := e.active[leaseID] 510 515 if res == nil || res.committed { 511 511 - return func() {}, false 516 516 + return nil, false 512 517 } 518 518 + if res.ttlTimer != nil { 519 519 + res.ttlTimer.Stop() 520 520 + } 513 521 return e.removeReservationLocked(res, releaseSlot), true 514 522 } 515 523 516 524 func (e *Executor) removeReservationLocked(res *reservation, releaseSlot bool) func() { 517 525 delete(e.active, res.leaseID) 518 518 - if res.ttlTimer != nil { 519 519 - res.ttlTimer.Stop() 520 520 - res.ttlTimer = nil 521 521 - } 522 522 - stopTail := res.stopTail 523 523 - res.stopTail = nil 524 526 slot := res.slot 525 525 - if releaseSlot { 526 526 - res.slot = nil 527 527 - } 528 527 return func() { 529 529 - if stopTail != nil { 530 530 - stopTail() 531 531 - } 532 528 if releaseSlot && slot != nil { 533 529 slot.Release() 534 530 } 535 531 } 536 532 } 537 533 538 538 - func normalizeLabels(labels []string) []string { 539 539 - seen := make(map[string]struct{}, len(labels)) 540 540 - out := make([]string, 0, len(labels)) 541 541 - for _, label := range labels { 542 542 - label = strings.TrimSpace(label) 543 543 - if label == "" { 544 544 - continue 545 545 - } 546 546 - if _, ok := seen[label]; ok { 547 547 - continue 548 548 - } 549 549 - seen[label] = struct{}{} 550 550 - out = append(out, label) 551 551 - } 552 552 - return out 553 553 - } 554 554 - 555 534 func (e *Executor) snapshotLoop(ctx context.Context, enc *millproto.Encoder) { 556 556 - t := time.NewTicker(snapshotEvery) 557 557 - defer t.Stop() 535 535 + ticker := time.NewTicker(snapshotEvery) 536 536 + defer ticker.Stop() 537 537 + 558 538 for { 559 539 select { 560 540 case <-ctx.Done(): 561 541 return 562 562 - case <-t.C: 563 563 - // stop once connection is replaced 564 564 - e.connMu.Lock() 565 565 - cur := e.enc 566 566 - e.connMu.Unlock() 567 567 - if cur != enc { 568 568 - return 569 569 - } 542 542 + case <-ticker.C: 570 543 e.pushSnapshot() 571 544 } 572 545 } ··· 579 552 } 580 553 581 554 func (e *Executor) pushSnapshotLocked() { 582 582 - 583 555 e.mu.Lock() 584 584 - draining := e.draining 585 585 - active := len(e.active) 586 586 - leaseIDs := make([]string, 0, len(e.active)) 587 587 - for id := range e.active { 588 588 - leaseIDs = append(leaseIDs, id) 556 556 + activeLeases := make([]string, 0, len(e.active)) 557 557 + for leaseID := range e.active { 558 558 + activeLeases = append(activeLeases, leaseID) 589 559 } 590 560 e.mu.Unlock() 591 561 592 592 - load := 0.0 593 593 - if e.seats > 0 { 594 594 - load = float64(active) / float64(e.seats) 595 595 - } 596 596 - available := !draining && active < e.seats 597 597 - loadMap := map[string]float64{"slots": load} 598 598 - 599 599 - engines := make(map[string]*millv1.EngineAvailability, len(e.engines)) 600 600 - for name := range e.engines { 601 601 - engines[name] = &millv1.EngineAvailability{ 602 602 - Available: available, 603 603 - Load: loadMap, 562 562 + avail := make(map[string]*millv1.EngineAvailability) 563 563 + for name, eng := range e.engines { 564 564 + a := &millv1.EngineAvailability{Available: true} 565 565 + if getter, ok := eng.(interface{ Load() map[string]float64 }); ok { 566 566 + a.Load = getter.Load() 604 567 } 568 568 + avail[name] = a 605 569 } 606 570 607 571 e.nextSeqno++ 608 608 - seq := e.nextSeqno 609 609 - 610 610 - e.send(&millproto.Message{NodeSnapshot: &millv1.NodeSnapshot{ 611 611 - Seqno: seq, 612 612 - Engines: engines, 613 613 - ActiveLeaseIds: leaseIDs, 614 614 - }}) 572 572 + snap := &millproto.Message{ 573 573 + NodeSnapshot: &millv1.NodeSnapshot{ 574 574 + Seqno: e.nextSeqno, 575 575 + Engines: avail, 576 576 + ActiveLeaseIds: activeLeases, 577 577 + }, 578 578 + } 579 579 + e.send(snap) 615 580 } 616 581 617 582 func (e *Executor) Drain() { ··· 629 594 } 630 595 631 596 const defaultReservationTTL = 60 * time.Second 597 597 + 598 598 + func normalizeLabels(labels []string) []string { 599 599 + seen := make(map[string]struct{}, len(labels)) 600 600 + out := make([]string, 0, len(labels)) 601 601 + for _, label := range labels { 602 602 + label = strings.TrimSpace(label) 603 603 + if label == "" { 604 604 + continue 605 605 + } 606 606 + if _, ok := seen[label]; ok { 607 607 + continue 608 608 + } 609 609 + seen[label] = struct{}{} 610 610 + out = append(out, label) 611 611 + } 612 612 + return out 613 613 + }

+120 -47

spindle/mill/executor/observe.go

Reviewed

··· 2 2 3 3 import ( 4 4 "context" 5 5 + "crypto/sha256" 6 6 + "encoding/hex" 7 7 + "fmt" 5 8 "io" 9 9 + "os" 10 10 + "strings" 6 11 "sync" 7 12 "time" 8 13 9 14 "github.com/hpcloud/tail" 10 10 - 11 15 "tangled.org/core/api/tangled" 16 16 + millproto "tangled.org/core/spindle/mill/proto" 12 17 millv1 "tangled.org/core/spindle/mill/proto/gen" 13 18 "tangled.org/core/spindle/models" 14 19 "tangled.org/core/spindle/secrets" 15 20 ) 16 21 17 17 - // streams status rows of active leases. jobs write status exactly like a 18 18 - // standalone spindle, we just forward them 19 22 func (e *Executor) observeLoop(ctx context.Context, sub <-chan struct{}, cursor int64) { 20 23 ticker := time.NewTicker(5 * time.Second) 21 24 defer ticker.Stop() ··· 32 35 } 33 36 34 37 func (e *Executor) drainEvents(cursor *int64) { 35 35 - for { 36 36 - evs, err := e.db.GetEvents(*cursor, 100) 37 37 - if err != nil { 38 38 - e.l.Error("observe GetEvents failed", "err", err) 39 39 - return 40 40 - } 41 41 - for _, ev := range evs { 42 42 - if ev.Nsid == tangled.PipelineStatusNSID { 43 43 - st, ok := parseStatus(ev.EventJson) 44 44 - if ok { 45 45 - if err := e.onStatusRow(st); err != nil { 46 46 - e.l.Error("persist streamed status", "err", err) 47 47 - return 48 48 - } 49 49 - } 50 50 - } 38 38 + events, err := e.db.GetEvents(*cursor, 128) 39 39 + if err != nil { 40 40 + e.l.Error("drain status events failed", "err", err) 41 41 + return 42 42 + } 43 43 + for _, ev := range events { 44 44 + if ev.Created > *cursor { 51 45 *cursor = ev.Created 52 46 } 53 53 - if len(evs) < 100 { 54 54 - return 47 47 + st, ok := parseStatus(ev.EventJson) 48 48 + if !ok { 49 49 + continue 55 50 } 51 51 + if err := e.onStatusRow(st); err != nil { 52 52 + e.l.Error("process status row failed", "err", err) 53 53 + } 56 54 } 57 55 } 58 56 ··· 62 60 return nil 63 61 } 64 62 65 65 - kind := models.StatusKind(st.Status) 66 66 - switch { 67 67 - case kind.IsFinish(): 63 63 + if models.StatusKind(st.Status).IsFinish() { 68 64 return e.finishJob(res, st) 69 69 - case kind == models.StatusKindRunning: 70 70 - return e.appendStatus(res.leaseID, st) 71 71 - default: 72 72 - return nil 73 65 } 66 66 + 67 67 + return e.appendStatus(res.leaseID, st) 74 68 } 75 69 76 76 - // flush the tail first so all log seqnos land before the terminal's 77 70 func (e *Executor) finishJob(res *reservation, st *tangled.PipelineStatus) error { 78 71 e.mu.Lock() 79 72 if e.active[res.leaseID] != res { ··· 81 74 return nil 82 75 } 83 76 cancelled := res.cancelled 84 84 - cleanup := e.removeReservationLocked(res, false) 85 77 e.mu.Unlock() 86 78 87 87 - cleanup() 79 79 + // 1. Finalize log tail first so all log lines precede terminal event 80 80 + if res.stopTail != nil { 81 81 + res.stopTail() 82 82 + } 88 83 89 84 terminalStatus := st.Status 90 85 if cancelled { 91 86 terminalStatus = string(models.StatusKindCancelled) 92 87 } 93 93 - if err := e.appendTerminal(res.leaseID, terminalStatus, st); err != nil { 88 88 + var logDir string 89 89 + if e.cfg != nil { 90 90 + logDir = e.cfg.Server.LogDir 91 91 + } 92 92 + logPath := models.LogFilePath(logDir, res.wid) 93 93 + 94 94 + var errStr string 95 95 + if st != nil && st.Error != nil { 96 96 + errStr = *st.Error 97 97 + } 98 98 + var exitCode int64 99 99 + if st != nil && st.ExitCode != nil { 100 100 + exitCode = *st.ExitCode 101 101 + } 102 102 + 103 103 + // Calculate SHA256 of log file 104 104 + hash := "" 105 105 + if f, err := os.Open(logPath); err == nil { 106 106 + h := sha256.New() 107 107 + if _, err := io.Copy(h, f); err == nil { 108 108 + hash = "sha256:" + hex.EncodeToString(h.Sum(nil)) 109 109 + } 110 110 + _ = f.Close() 111 111 + } 112 112 + 113 113 + // refs are opaque keys interpreted by the configured artifact store 114 114 + ref := "logs/" + res.leaseID + ".log" 115 115 + 116 116 + // 2. Persist restart-retryable pending artifact state 117 117 + if e.db != nil { 118 118 + _ = e.db.SavePendingArtifact(res.leaseID, res.wid.Name, terminalStatus, errStr, exitCode, ref, hash) 119 119 + } 120 120 + 121 121 + // 3. Upload artifact with non-cancelled cleanup context 122 122 + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(context.Background()), 2*time.Minute) 123 123 + defer cancel() 124 124 + 125 125 + if e.writer != nil { 126 126 + if f, err := os.Open(logPath); err == nil { 127 127 + defer f.Close() 128 128 + if uploadErr := e.writer.Put(cleanupCtx, ref, f); uploadErr != nil { 129 129 + e.l.Error("artifact upload failed", "lease", res.leaseID, "err", uploadErr) 130 130 + return fmt.Errorf("artifact upload: %w", uploadErr) 131 131 + } 132 132 + } 133 133 + } 134 134 + 135 135 + // 4. Append terminal event (with LogArtifact) to outbox 136 136 + if err := e.appendTerminalWithArtifact(res.leaseID, terminalStatus, st, ref, hash); err != nil { 94 137 return err 95 138 } 139 139 + 140 140 + // 5. Remove pending artifact state after terminal outbox append succeeds 141 141 + if e.db != nil { 142 142 + _ = e.db.RemovePendingArtifact(res.leaseID) 143 143 + } 144 144 + e.mu.Lock() 145 145 + cleanup := e.removeReservationLocked(res, false) 146 146 + e.mu.Unlock() 147 147 + 148 148 + cleanup() 149 149 + 96 150 e.pushSnapshot() 97 151 return nil 98 152 } 99 153 100 100 - // reservations are few (bounded by seats), so a scan beats a parallel index 101 154 func (e *Executor) reservationFor(pipelineAturi, workflow string) *reservation { 102 155 e.mu.Lock() 103 156 defer e.mu.Unlock() ··· 109 162 return nil 110 163 } 111 164 112 112 - func (e *Executor) streamLogLine(leaseID, text string) { 113 113 - e.appendLog(leaseID, []byte(text)) 165 165 + func (e *Executor) maskSecrets(res *reservation, text string) string { 166 166 + if res == nil || res.vault == nil { 167 167 + return text 168 168 + } 169 169 + for _, s := range res.vault.secrets { 170 170 + if s.Value != "" { 171 171 + text = strings.ReplaceAll(text, s.Value, "***") 172 172 + } 173 173 + } 174 174 + return text 114 175 } 115 176 116 116 - // log lines come pre-encoded as models.LogLine JSON, forward them verbatim 177 177 + func (e *Executor) SendLiveLog(leaseID string, raw []byte) error { 178 178 + e.connMu.Lock() 179 179 + enc := e.enc 180 180 + e.connMu.Unlock() 181 181 + if enc == nil { 182 182 + return nil 183 183 + } 184 184 + return enc.Encode(&millproto.Message{ 185 185 + LiveLog: &millv1.LiveLog{ 186 186 + LeaseId: leaseID, 187 187 + RawJson: raw, 188 188 + }, 189 189 + }) 190 190 + } 191 191 + 117 192 func (e *Executor) startTail(res *reservation) { 118 193 path := models.LogFilePath(e.cfg.Server.LogDir, res.wid) 119 194 t, err := tail.TailFile(path, tail.Config{ ··· 135 210 if line == nil || line.Err != nil { 136 211 continue 137 212 } 138 138 - e.streamLogLine(res.leaseID, line.Text) 213 213 + masked := e.maskSecrets(res, line.Text) 214 214 + _ = e.SendLiveLog(res.leaseID, []byte(masked+"\n")) 139 215 } 140 216 }() 141 217 142 218 var once sync.Once 143 219 res.stopTail = func() { 144 220 once.Do(func() { 145 145 - // Stop() blocks until tailing ends, then the consumer drains 146 146 - // what's buffered and closes done 147 147 - // waiting for done means every log line streams before finishJob's 148 148 - // terminal. a slow drain would let the terminal overtake a log line 149 149 - // and the mill drops those 150 221 _ = t.Stop() 151 222 <-done 152 223 }) 153 224 } 154 225 } 155 226 156 156 - // in-memory secrets manager holding what the mill handed over at commit 157 227 type memVault struct { 158 228 secrets []secrets.UnlockedSecret 159 229 } 160 230 161 231 func newMemVault(pb []*millv1.Secret) *memVault { 162 162 - s := make([]secrets.UnlockedSecret, len(pb)) 163 163 - for i, x := range pb { 164 164 - s[i] = secrets.UnlockedSecret{Key: x.GetKey(), Value: x.GetValue()} 232 232 + v := &memVault{secrets: make([]secrets.UnlockedSecret, 0, len(pb))} 233 233 + for _, s := range pb { 234 234 + v.secrets = append(v.secrets, secrets.UnlockedSecret{ 235 235 + Key: s.Key, 236 236 + Value: s.Value, 237 237 + }) 165 238 } 166 166 - return &memVault{secrets: s} 239 239 + return v 167 240 } 168 241 169 242 func (v *memVault) GetSecretsUnlocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.UnlockedSecret, error) {

+128 -177

spindle/mill/executor/outbox.go

Reviewed

··· 1 1 package executor 2 2 3 3 import ( 4 4 + "context" 4 5 "crypto/rand" 5 6 "encoding/hex" 6 7 "encoding/json" 7 8 "fmt" 9 9 + "os" 8 10 "time" 9 9 - "unicode/utf8" 10 11 11 12 "google.golang.org/protobuf/proto" 12 13 "tangled.org/core/api/tangled" ··· 17 18 ) 18 19 19 20 const ( 20 20 - maxBatchEntries = 1000 21 21 - maxBatchBytes = 1 * 1024 * 1024 22 22 - maxEventErrorBytes = 64 * 1024 23 23 - eventFlushDelay = 10 * time.Millisecond 21 21 + maxBatchBytes = 4 * 1024 * 1024 22 22 + maxBatchEvents = 128 24 23 ) 25 24 26 25 func generateEpoch() string { 27 27 - b := make([]byte, 16) 28 28 - if _, err := rand.Read(b); err != nil { 29 29 - panic(err) 30 30 - } 31 31 - return hex.EncodeToString(b) 26 26 + var b [8]byte 27 27 + _, _ = rand.Read(b[:]) 28 28 + return hex.EncodeToString(b[:]) 32 29 } 33 30 34 31 func (e *Executor) initOutbox() error { 35 32 e.eventMu.Lock() 36 33 defer e.eventMu.Unlock() 37 34 38 38 - inc, nextSeqno, err := e.db.GetOutboxState() 35 35 + epoch, _, err := e.db.GetOutboxState() 39 36 if err != nil { 40 37 return err 41 38 } 42 42 - if inc == "" { 43 43 - inc = generateEpoch() 44 44 - if err := e.db.SetOutboxEpoch(inc); err != nil { 45 45 - return err 39 39 + if epoch != "" { 40 40 + e.epoch = epoch 41 41 + } else { 42 42 + e.epoch = generateEpoch() 43 43 + if err := e.db.SetOutboxEpoch(e.epoch); err != nil { 44 44 + return fmt.Errorf("set outbox epoch: %w", err) 46 45 } 47 46 } 48 48 - e.epoch = inc 49 47 50 48 rows, err := e.db.ListOutboxRows() 51 49 if err != nil { 52 52 - return err 50 50 + return fmt.Errorf("list outbox rows: %w", err) 53 51 } 54 54 - e.outboxLogBytes = 0 55 55 - e.outboxControlBytes = 0 56 56 - var previousSeqno uint64 52 52 + 57 53 for _, r := range rows { 58 58 - if r.Epoch != inc || r.Seqno == 0 || r.Seqno >= nextSeqno { 59 59 - return fmt.Errorf("invalid outbox row epoch=%q seqno=%d next=%d", r.Epoch, r.Seqno, nextSeqno) 60 60 - } 61 61 - if previousSeqno != 0 && r.Seqno != previousSeqno+1 { 62 62 - return fmt.Errorf("outbox seqno gap after %d: got %d", previousSeqno, r.Seqno) 63 63 - } 64 64 - entry, err := decodeEvent(r.Payload, r.Seqno) 65 65 - if err != nil { 66 66 - return err 67 67 - } 68 68 - if entry.GetLeaseId() == "" || entry.GetPayload() == nil { 69 69 - return fmt.Errorf("invalid outbox payload at seqno %d", r.Seqno) 70 70 - } 71 71 - previousSeqno = r.Seqno 72 72 - if r.Control { 73 73 - e.outboxControlBytes += r.ByteSize 74 74 - } else { 75 75 - e.outboxLogBytes += r.ByteSize 76 76 - } 54 54 + e.outboxBytes += r.ByteSize 77 55 } 56 56 + _ = e.recoverPendingArtifacts() 78 57 return nil 79 58 } 80 59 ··· 83 62 switch payload := payload.(type) { 84 63 case *millv1.Event_StatusEvent: 85 64 entry.Payload = payload 86 86 - case *millv1.Event_LogLine: 87 87 - entry.Payload = payload 88 65 case *millv1.Event_AttemptResult: 89 66 entry.Payload = payload 90 67 default: ··· 101 78 }}) 102 79 entry.Seqno = 0 103 80 if wireSize > maxBatchBytes { 104 104 - if !control { 105 105 - e.eventMu.Lock() 106 106 - e.droppedLogs++ 107 107 - dropped := e.droppedLogs 108 108 - e.eventMu.Unlock() 109 109 - e.l.Warn("stream log entry exceeds wire limit; dropping line", "bytes", wireSize, "limit", maxBatchBytes, "droppedCount", dropped) 110 110 - return nil 111 111 - } 112 81 return fmt.Errorf("control stream entry exceeds wire limit: %d > %d", wireSize, maxBatchBytes) 113 82 } 114 83 ··· 118 87 } 119 88 120 89 e.eventMu.Lock() 121 121 - if !control && e.outboxLogBytes+int64(len(encoded)) > e.maxLogBytes { 122 122 - e.droppedLogs++ 123 123 - e.l.Warn("outbox log cap exceeded; dropping log line", "droppedCount", e.droppedLogs, "cap", e.maxLogBytes) 90 90 + if control && !isTerminal && e.maxOutboxBytes > 0 && e.outboxBytes+int64(len(encoded)) > e.maxOutboxBytes { 91 91 + e.l.Warn("outbox reserve exhausted; dropping nonterminal status", "cap", e.maxOutboxBytes) 124 92 e.eventMu.Unlock() 125 93 return nil 126 94 } 127 127 - if control && !isTerminal && e.maxControlBytes > 0 && e.outboxControlBytes+int64(len(encoded)) > e.maxControlBytes { 128 128 - e.l.Warn("control outbox reserve exhausted; dropping nonterminal status", "cap", e.maxControlBytes) 129 129 - e.eventMu.Unlock() 130 130 - return nil 131 131 - } 132 95 if _, err := e.db.AppendOutboxRow(encoded, control); err != nil { 133 96 e.eventMu.Unlock() 134 97 return fmt.Errorf("append outbox row: %w", err) 135 98 } 136 136 - if control { 137 137 - e.outboxControlBytes += int64(len(encoded)) 138 138 - } else { 139 139 - e.outboxLogBytes += int64(len(encoded)) 140 140 - } 99 99 + e.outboxBytes += int64(len(encoded)) 141 100 e.eventMu.Unlock() 142 101 143 143 - if control { 144 144 - e.sendPending() 145 145 - } else { 146 146 - e.schedulePending() 147 147 - } 102 102 + e.sendPending() 148 103 return nil 149 104 } 150 105 ··· 161 116 return e.appendAndSend(leaseID, payload, true) 162 117 } 163 118 164 164 - func (e *Executor) appendLog(leaseID string, raw []byte) { 165 165 - logLine := &millv1.LogLine{ 166 166 - RawJson: raw, 167 167 - } 168 168 - payload := &millv1.Event_LogLine{LogLine: logLine} 169 169 - if err := e.appendAndSend(leaseID, payload, false); err != nil { 170 170 - e.l.Error("failed to persist streamed log", "lease", leaseID, "err", err) 171 171 - } 119 119 + func (e *Executor) appendTerminal(leaseID, status string, st *tangled.PipelineStatus) error { 120 120 + return e.appendTerminalWithArtifact(leaseID, status, st, "", "") 172 121 } 173 122 174 174 - func (e *Executor) appendTerminal(leaseID, status string, st *tangled.PipelineStatus) error { 123 123 + func (e *Executor) appendTerminalWithArtifact(leaseID, status string, st *tangled.PipelineStatus, ref, hash string) error { 175 124 var terminalStatus millv1.TerminalStatus 176 125 switch status { 177 126 case string(models.StatusKindSuccess): ··· 187 136 } 188 137 189 138 exit, errStr := parseStatusExitAndError(st) 139 139 + var logArtifact *millv1.LogArtifact 140 140 + if ref != "" { 141 141 + logArtifact = &millv1.LogArtifact{ 142 142 + Ref: ref, 143 143 + Hash: hash, 144 144 + } 145 145 + } 190 146 payload := &millv1.Event_AttemptResult{AttemptResult: &millv1.AttemptResult{ 191 191 - Status: terminalStatus, 192 192 - Error: errStr, 193 193 - ExitCode: exit, 147 147 + Status: terminalStatus, 148 148 + Error: errStr, 149 149 + ExitCode: exit, 150 150 + LogArtifact: logArtifact, 194 151 }} 195 152 return e.appendAndSend(leaseID, payload, true) 196 153 } 197 154 198 198 - func truncateEventString(value string) string { 199 199 - if len(value) <= maxEventErrorBytes { 200 200 - return value 155 155 + func (e *Executor) recoverPendingArtifacts() error { 156 156 + if e.db == nil { 157 157 + return nil 201 158 } 202 202 - end := maxEventErrorBytes 203 203 - for end > 0 && !utf8.RuneStart(value[end]) { 204 204 - end-- 159 159 + pending, err := e.db.ListPendingArtifacts() 160 160 + if err != nil || len(pending) == 0 { 161 161 + return err 205 162 } 206 206 - return value[:end] 163 163 + for _, p := range pending { 164 164 + if p.Ref != "" { 165 165 + if e.writer == nil { 166 166 + continue 167 167 + } 168 168 + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) 169 169 + var logDir string 170 170 + if e.cfg != nil { 171 171 + logDir = e.cfg.Server.LogDir 172 172 + } 173 173 + logPath := models.LogFilePath(logDir, models.WorkflowId{Name: p.Workflow}) 174 174 + f, openErr := os.Open(logPath) 175 175 + if openErr != nil { 176 176 + cancel() 177 177 + continue 178 178 + } 179 179 + uploadErr := e.writer.Put(ctx, p.Ref, f) 180 180 + _ = f.Close() 181 181 + cancel() 182 182 + if uploadErr != nil { 183 183 + continue 184 184 + } 185 185 + } 186 186 + 187 187 + st := &tangled.PipelineStatus{ 188 188 + Status: p.Status, 189 189 + Error: &p.Error, 190 190 + ExitCode: &p.ExitCode, 191 191 + } 192 192 + if err := e.appendTerminalWithArtifact(p.LeaseID, p.Status, st, p.Ref, p.Hash); err == nil { 193 193 + _ = e.db.RemovePendingArtifact(p.LeaseID) 194 194 + } 195 195 + } 196 196 + return nil 207 197 } 208 198 209 209 - func (e *Executor) schedulePending() { 210 210 - e.flushMu.Lock() 211 211 - defer e.flushMu.Unlock() 212 212 - if e.flushTimer != nil { 213 213 - return 199 199 + func truncateEventString(value string) string { 200 200 + if len(value) > 65536 { 201 201 + return value[:65536] 214 202 } 215 215 - e.flushTimer = time.AfterFunc(eventFlushDelay, func() { 216 216 - e.flushMu.Lock() 217 217 - e.flushTimer = nil 218 218 - e.flushMu.Unlock() 219 219 - e.sendPending() 220 220 - }) 203 203 + return value 221 204 } 222 205 223 206 func (e *Executor) sendPending() { 224 224 - e.sendMu.Lock() 225 225 - defer e.sendMu.Unlock() 226 226 - 227 207 e.connMu.Lock() 228 208 enc := e.enc 229 229 - cancel := e.sessionCancel 230 209 e.connMu.Unlock() 210 210 + 231 211 if enc == nil { 232 212 return 233 213 } 214 214 + 215 215 + e.flushMu.Lock() 216 216 + defer e.flushMu.Unlock() 217 217 + 234 218 if err := e.sendPendingLocked(enc); err != nil { 235 235 - e.l.Error("failed to send stream batch, ending session", "err", err) 236 236 - if cancel != nil { 237 237 - cancel() 238 238 - } 219 219 + e.l.Error("send pending events failed", "err", err) 239 220 } 240 221 } 241 222 242 223 func (e *Executor) sendPendingLocked(enc messageEncoder) error { 243 224 for { 244 244 - e.eventMu.Lock() 245 245 - rows, err := e.db.ListOutboxRowsAfter(e.sentSeqno, maxBatchEntries) 246 246 - e.eventMu.Unlock() 225 225 + rows, err := e.db.ListOutboxRowsAfter(e.sentSeqno, maxBatchEvents) 247 226 if err != nil { 248 248 - return err 227 227 + return fmt.Errorf("list outbox rows after %d: %w", e.sentSeqno, err) 249 228 } 250 229 if len(rows) == 0 { 251 230 return nil ··· 253 232 if err := e.sendRows(enc, rows); err != nil { 254 233 return err 255 234 } 256 256 - if len(rows) < maxBatchEntries { 257 257 - return nil 258 258 - } 259 235 } 260 236 } 261 237 262 238 func (e *Executor) sendRows(enc messageEncoder, rows []db.OutboxRow) error { 263 263 - var ( 264 264 - batch []*millv1.Event 265 265 - batchSize int64 266 266 - last uint64 267 267 - ) 268 268 - sendBatch := func() error { 269 269 - if len(batch) == 0 { 270 270 - return nil 271 271 - } 272 272 - if err := enc.Encode(&millproto.Message{EventBatch: &millv1.EventBatch{ 273 273 - Epoch: e.epoch, 274 274 - Events: batch, 275 275 - }}); err != nil { 276 276 - return err 277 277 - } 278 278 - e.sentSeqno = last 279 279 - batch = nil 280 280 - batchSize = 0 281 281 - return nil 282 282 - } 239 239 + events := make([]*millv1.Event, 0, len(rows)) 240 240 + var lastSeqno uint64 283 241 284 284 - for _, row := range rows { 285 285 - if row.Seqno <= e.sentSeqno { 286 286 - continue 287 287 - } 288 288 - entry, err := decodeEvent(row.Payload, row.Seqno) 242 242 + for _, r := range rows { 243 243 + ev, err := decodeEvent(r.Payload, r.Seqno) 289 244 if err != nil { 290 290 - return err 245 245 + return fmt.Errorf("decode event at seqno %d: %w", r.Seqno, err) 291 246 } 292 292 - entry.Seqno = row.Seqno 293 293 - if len(batch) >= maxBatchEntries || batchSize+row.ByteSize > maxBatchBytes { 294 294 - if err := sendBatch(); err != nil { 295 295 - return err 296 296 - } 297 297 - } 298 298 - batch = append(batch, entry) 299 299 - batchSize += row.ByteSize 300 300 - last = row.Seqno 247 247 + events = append(events, ev) 248 248 + lastSeqno = r.Seqno 301 249 } 302 302 - return sendBatch() 303 303 - } 304 250 305 305 - func (e *Executor) replay(ackSeqno uint64) error { 251 251 + batch := &millv1.EventBatch{ 252 252 + Epoch: e.epoch, 253 253 + Events: events, 254 254 + } 255 255 + 306 256 e.sendMu.Lock() 307 257 defer e.sendMu.Unlock() 308 258 309 309 - _, nextSeqno, err := e.db.GetOutboxState() 310 310 - if err != nil { 311 311 - return err 259 259 + if err := enc.Encode(&millproto.Message{EventBatch: batch}); err != nil { 260 260 + return fmt.Errorf("encode event batch (seqno %d..%d): %w", rows[0].Seqno, lastSeqno, err) 312 261 } 313 313 - if ackSeqno >= nextSeqno { 314 314 - return fmt.Errorf("mill acknowledged unsent seqno %d (next seqno %d)", ackSeqno, nextSeqno) 315 315 - } 262 262 + e.sentSeqno = lastSeqno 263 263 + return nil 264 264 + } 316 265 317 317 - if err := e.deleteOutboxPrefix(ackSeqno); err != nil { 318 318 - return err 319 319 - } 320 320 - 321 321 - e.sentSeqno = ackSeqno 266 266 + func (e *Executor) replay(ackSeqno uint64) error { 322 267 e.connMu.Lock() 323 268 enc := e.enc 324 269 e.connMu.Unlock() 325 270 if enc == nil { 326 271 return nil 327 272 } 273 273 + 274 274 + e.flushMu.Lock() 275 275 + defer e.flushMu.Unlock() 276 276 + 277 277 + e.sendMu.Lock() 278 278 + e.sentSeqno = ackSeqno 279 279 + e.sendMu.Unlock() 280 280 + 328 281 return e.sendPendingLocked(enc) 329 282 } 330 283 331 284 func (e *Executor) handleAck(ack *millv1.Ack) { 332 332 - if ack.GetEpoch() != e.epoch { 333 333 - e.l.Warn("received Ack for mismatched epoch", "got", ack.GetEpoch(), "want", e.epoch) 285 285 + if ack == nil || ack.GetEpoch() != e.epoch { 334 286 return 335 287 } 336 288 337 337 - e.sendMu.Lock() 338 338 - if ack.GetUpToSeqno() > e.sentSeqno { 339 339 - e.sendMu.Unlock() 340 340 - e.l.Warn("received Ack for unsent stream seqno", "got", ack.GetUpToSeqno(), "sent", e.sentSeqno) 289 289 + upTo := ack.GetUpToSeqno() 290 290 + if upTo == 0 { 341 291 return 342 292 } 343 343 - err := e.deleteOutboxPrefix(ack.GetUpToSeqno()) 344 344 - e.sendMu.Unlock() 345 345 - if err != nil { 346 346 - e.l.Error("failed to delete outbox prefix on Ack", "upTo", ack.GetUpToSeqno(), "err", err) 293 293 + 294 294 + if err := e.deleteOutboxPrefix(upTo); err != nil { 295 295 + e.l.Error("delete outbox prefix failed", "upTo", upTo, "err", err) 347 296 } 348 297 } 349 298 350 299 func (e *Executor) subtractOutboxBytes(deleted db.OutboxDeletion) { 351 351 - e.outboxLogBytes = max(0, e.outboxLogBytes-deleted.LogBytes) 352 352 - e.outboxControlBytes = max(0, e.outboxControlBytes-deleted.ControlBytes) 300 300 + e.outboxBytes = max(0, e.outboxBytes-deleted.Bytes) 353 301 } 354 302 355 303 func parseStatus(raw json.RawMessage) (*tangled.PipelineStatus, bool) { ··· 359 307 } 360 308 return &st, true 361 309 } 310 310 + 362 311 func (e *Executor) deleteOutboxPrefix(upTo uint64) error { 363 312 e.eventMu.Lock() 364 313 defer e.eventMu.Unlock() 314 314 + 365 315 deleted, err := e.db.DeleteOutboxPrefix(upTo) 366 316 if err == nil { 367 317 e.subtractOutboxBytes(deleted) ··· 385 335 } 386 336 387 337 func decodeEvent(payload []byte, seqno uint64) (*millv1.Event, error) { 388 388 - var entry millv1.Event 389 389 - if err := proto.Unmarshal(payload, &entry); err != nil { 390 390 - return nil, fmt.Errorf("decode outbox seqno %d: %w", seqno, err) 338 338 + var ev millv1.Event 339 339 + if err := proto.Unmarshal(payload, &ev); err != nil { 340 340 + return nil, err 391 341 } 392 392 - return &entry, nil 342 342 + ev.Seqno = seqno 343 343 + return &ev, nil 393 344 }

spindle/mill/executor/reserved.go

Reviewed

This file has not been changed.

+46 -367

spindle/mill/executor/reserved_test.go

Reviewed

··· 3 3 import ( 4 4 "context" 5 5 "encoding/json" 6 6 - "fmt" 7 6 "github.com/bluesky-social/indigo/atproto/syntax" 8 7 "github.com/gorilla/websocket" 9 8 "google.golang.org/protobuf/proto" ··· 119 118 } 120 119 } 121 120 122 122 - func TestReservedEngineForwardsLifecycle(t *testing.T) { 123 123 - inner := &fakeEngine{} 124 124 - re := newReservedEngine(inner, &fakeSlot{}) 125 125 - 126 126 - if d := re.WorkflowTimeout(); d != 7*time.Minute { 127 127 - t.Fatalf("WorkflowTimeout() = %v, want 7m", d) 128 128 - } 129 129 - if err := re.SetupWorkflow(context.Background(), models.WorkflowId{}, &models.Workflow{}, models.NullLogger{}); err != nil { 130 130 - t.Fatal(err) 131 131 - } 132 132 - if err := re.RunStep(context.Background(), models.WorkflowId{}, &models.Workflow{Steps: []models.Step{}}, 0, nil, models.NullLogger{}); err != nil { 133 133 - t.Fatal(err) 134 134 - } 135 135 - if err := re.DestroyWorkflow(context.Background(), models.WorkflowId{}); err != nil { 136 136 - t.Fatal(err) 137 137 - } 138 138 - if !inner.setupCalled || !inner.runCalled || !inner.destroyCalled { 139 139 - t.Fatalf("lifecycle not forwarded: %+v", inner) 140 140 - } 141 141 - } 142 142 - 143 121 func TestHandleCommitIsIdempotent(t *testing.T) { 144 122 enc := newCaptureEncoder() 145 123 e := testExecutor(t) ··· 207 185 n := notifier.New() 208 186 enc := newCaptureEncoder() 209 187 e := &Executor{ 210 210 - cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 211 211 - db: d, 212 212 - n: &n, 213 213 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 214 214 - active: make(map[string]*reservation), 215 215 - maxLogBytes: 100 * 1024 * 1024, 216 216 - maxControlBytes: 10 * 1024 * 1024, 217 217 - enc: enc, 188 188 + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 189 189 + db: d, 190 190 + n: &n, 191 191 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 192 192 + active: make(map[string]*reservation), 193 193 + maxOutboxBytes: 10 * 1024 * 1024, 194 194 + enc: enc, 218 195 } 219 196 if err := e.initOutbox(); err != nil { 220 197 t.Fatal(err) ··· 294 271 } 295 272 } 296 273 297 297 - func TestRunSessionConsumesAcksWhileReplaying(t *testing.T) { 298 298 - e := testSessionExecutor(t, "") 299 299 - e.appendLog("lease", []byte("first")) 300 300 - e.appendLog("lease", make([]byte, 7<<20)) 301 301 - 302 302 - serverErr := make(chan error, 1) 303 303 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 304 304 - conn, err := websocket.Upgrade(w, r, nil, 1024, 1024) 305 305 - if err != nil { 306 306 - serverErr <- err 307 307 - return 308 308 - } 309 309 - defer conn.Close() 310 310 - if tcp, ok := conn.UnderlyingConn().(interface{ SetReadBuffer(int) error }); ok { 311 311 - _ = tcp.SetReadBuffer(1024) 312 312 - } 313 313 - stream := millproto.NewWSStream(conn) 314 314 - enc := millproto.NewEncoder(stream) 315 315 - dec := millproto.NewDecoder(stream) 316 316 - if _, err := dec.Decode(); err != nil { 317 317 - serverErr <- err 318 318 - return 319 319 - } 320 320 - if err := enc.Encode(&millproto.Message{Resume: &millv1.Resume{Epoch: e.epoch}}); err != nil { 321 321 - serverErr <- err 322 322 - return 323 323 - } 324 324 - first, err := dec.Decode() 325 325 - if err != nil { 326 326 - serverErr <- err 327 327 - return 328 328 - } 329 329 - batch := first.GetEventBatch() 330 330 - if batch == nil || len(batch.GetEvents()) == 0 { 331 331 - serverErr <- fmt.Errorf("expected non-empty batch") 332 332 - return 333 333 - } 334 334 - got := batch.GetEvents()[0].GetSeqno() 335 335 - if got != 1 { 336 336 - serverErr <- fmt.Errorf("first replay seqno = %d, want 1", got) 337 337 - return 338 338 - } 339 339 - if err := enc.Encode(&millproto.Message{Ack: &millv1.Ack{Epoch: e.epoch, UpToSeqno: 1}}); err != nil { 340 340 - serverErr <- err 341 341 - return 342 342 - } 343 343 - 344 344 - deadline := time.Now().Add(10 * time.Second) 345 345 - for { 346 346 - rows, err := e.db.ListOutboxRows() 347 347 - if err == nil && len(rows) <= 1 { 348 348 - break 349 349 - } 350 350 - if time.Now().After(deadline) { 351 351 - serverErr <- context.DeadlineExceeded 352 352 - return 353 353 - } 354 354 - time.Sleep(time.Millisecond) 355 355 - } 356 356 - 357 357 - serverErr <- nil 358 358 - })) 359 359 - t.Cleanup(srv.Close) 360 360 - e.millURL = "ws" + strings.TrimPrefix(srv.URL, "http") 361 361 - 362 362 - done := make(chan error, 1) 363 363 - go func() { done <- e.runSession(context.Background()) }() 364 364 - select { 365 365 - case err := <-serverErr: 366 366 - if err != nil { 367 367 - t.Fatalf("server: %v", err) 368 368 - } 369 369 - case <-time.After(20 * time.Second): 370 370 - t.Fatal("replay stalled while the peer waited for its acknowledgement to be consumed") 371 371 - } 372 372 - select { 373 373 - case <-done: 374 374 - case <-time.After(5 * time.Second): 375 375 - t.Fatal("runSession did not stop after peer closed") 376 376 - } 377 377 - } 378 378 - 379 274 func testSessionExecutor(t *testing.T, url string) *Executor { 380 275 d := testDB(t) 381 276 e := &Executor{ 382 382 - millURL: url, 383 383 - seats: 1, 384 384 - engines: make(map[string]models.Engine), 385 385 - db: d, 386 386 - cfg: &config.Config{Server: config.Server{Dev: true}}, 387 387 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 388 388 - active: make(map[string]*reservation), 389 389 - maxLogBytes: 100 * 1024 * 1024, 390 390 - maxControlBytes: 10 * 1024 * 1024, 277 277 + millURL: url, 278 278 + seats: 1, 279 279 + engines: make(map[string]models.Engine), 280 280 + db: d, 281 281 + cfg: &config.Config{Server: config.Server{Dev: true}}, 282 282 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 283 283 + active: make(map[string]*reservation), 284 284 + maxOutboxBytes: 10 * 1024 * 1024, 391 285 } 392 286 if err := e.initOutbox(); err != nil { 393 287 t.Fatal(err) ··· 398 292 func testExecutor(t *testing.T) *Executor { 399 293 d := testDB(t) 400 294 e := &Executor{ 401 401 - db: d, 402 402 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 403 403 - active: make(map[string]*reservation), 404 404 - maxLogBytes: 100 * 1024 * 1024, 405 405 - maxControlBytes: 10 * 1024 * 1024, 295 295 + db: d, 296 296 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 297 297 + active: make(map[string]*reservation), 298 298 + maxOutboxBytes: 10 * 1024 * 1024, 406 299 } 407 300 if err := e.initOutbox(); err != nil { 408 301 t.Fatal(err) ··· 426 319 cancelled: true, 427 320 } 428 321 e := &Executor{ 429 429 - db: d, 430 430 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 431 431 - active: map[string]*reservation{res.leaseID: res}, 432 432 - maxLogBytes: 100 * 1024 * 1024, 433 433 - maxControlBytes: 10 * 1024 * 1024, 322 322 + db: d, 323 323 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 324 324 + active: map[string]*reservation{res.leaseID: res}, 325 325 + maxOutboxBytes: 10 * 1024 * 1024, 434 326 } 435 327 if err := e.initOutbox(); err != nil { 436 328 t.Fatal(err) ··· 460 352 } 461 353 } 462 354 463 463 - func TestRestartOutboxReplay(t *testing.T) { 464 464 - d := testDB(t) 465 465 - 466 466 - e1 := &Executor{ 467 467 - db: d, 468 468 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 469 469 - active: make(map[string]*reservation), 470 470 - maxLogBytes: 100 * 1024 * 1024, 471 471 - maxControlBytes: 10 * 1024 * 1024, 472 472 - } 473 473 - if err := e1.initOutbox(); err != nil { 474 474 - t.Fatal(err) 475 475 - } 476 476 - inc1 := e1.epoch 477 477 - 478 478 - e1.appendLog("lease-1", []byte("log 1")) 479 479 - 480 480 - e2 := &Executor{ 481 481 - db: d, 482 482 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 483 483 - active: make(map[string]*reservation), 484 484 - maxLogBytes: 100 * 1024 * 1024, 485 485 - maxControlBytes: 10 * 1024 * 1024, 486 486 - } 487 487 - if err := e2.initOutbox(); err != nil { 488 488 - t.Fatal(err) 489 489 - } 490 490 - 491 491 - if e2.epoch != inc1 { 492 492 - t.Fatalf("reconstructed executor got epoch %q, want %q", e2.epoch, inc1) 493 493 - } 494 494 - 495 495 - enc := newCaptureEncoder() 496 496 - e2.enc = enc 497 497 - 498 498 - if err := e2.replay(0); err != nil { 499 499 - t.Fatal(err) 500 500 - } 501 501 - 502 502 - select { 503 503 - case msg := <-enc.messages: 504 504 - batch := msg.GetEventBatch() 505 505 - if batch == nil { 506 506 - t.Fatal("expected EventBatch") 507 507 - } 508 508 - if batch.GetEpoch() != inc1 { 509 509 - t.Fatalf("batch epoch = %q, want %q", batch.GetEpoch(), inc1) 510 510 - } 511 511 - if len(batch.GetEvents()) != 1 { 512 512 - t.Fatalf("expected 1 entry, got %d", len(batch.GetEvents())) 513 513 - } 514 514 - entry := batch.GetEvents()[0] 515 515 - if string(entry.GetLogLine().GetRawJson()) != "log 1" { 516 516 - t.Fatalf("got log raw = %q, want log 1", entry.GetLogLine().GetRawJson()) 517 517 - } 518 518 - case <-time.After(2 * time.Second): 519 519 - t.Fatal("timeout waiting for replay message") 520 520 - } 521 521 - } 522 522 - 523 355 func TestReplayRejectsMalformedOutboxRow(t *testing.T) { 524 356 d := testDB(t) 525 357 e := &Executor{ 526 526 - db: d, 527 527 - enc: newCaptureEncoder(), 528 528 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 529 529 - maxLogBytes: 100 * 1024 * 1024, 358 358 + db: d, 359 359 + enc: newCaptureEncoder(), 360 360 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 530 361 } 531 362 if err := e.initOutbox(); err != nil { 532 363 t.Fatal(err) ··· 539 370 } 540 371 } 541 372 542 542 - func TestBatchAck(t *testing.T) { 543 543 - d := testDB(t) 544 544 - e := &Executor{ 545 545 - db: d, 546 546 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 547 547 - active: make(map[string]*reservation), 548 548 - maxLogBytes: 100 * 1024 * 1024, 549 549 - maxControlBytes: 10 * 1024 * 1024, 550 550 - } 551 551 - if err := e.initOutbox(); err != nil { 552 552 - t.Fatal(err) 553 553 - } 554 554 - 555 555 - e.appendLog("lease-1", []byte("log 1")) 556 556 - e.appendLog("lease-1", []byte("log 2")) 557 557 - e.sentSeqno = 2 558 558 - 559 559 - rows, err := d.ListOutboxRows() 560 560 - if err != nil || len(rows) != 2 { 561 561 - t.Fatalf("expected 2 rows, got %d", len(rows)) 562 562 - } 563 563 - 564 564 - e.handleAck(&millv1.Ack{ 565 565 - Epoch: e.epoch, 566 566 - UpToSeqno: 1, 567 567 - }) 568 568 - 569 569 - rows, err = d.ListOutboxRows() 570 570 - if err != nil || len(rows) != 1 { 571 571 - t.Fatalf("expected 1 row after Ack, got %d", len(rows)) 572 572 - } 573 573 - if rows[0].Seqno != 2 { 574 574 - t.Fatalf("expected remaining row to have seqno 2, got %d", rows[0].Seqno) 575 575 - } 576 576 - } 577 577 - 578 578 - func TestLiveEventBatchesLogsWithoutResendingUnackedRows(t *testing.T) { 579 579 - d := testDB(t) 580 580 - enc := newCaptureEncoder() 581 581 - e := &Executor{ 582 582 - db: d, 583 583 - enc: enc, 584 584 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 585 585 - active: make(map[string]*reservation), 586 586 - maxLogBytes: 100 * 1024 * 1024, 587 587 - } 588 588 - if err := e.initOutbox(); err != nil { 589 589 - t.Fatal(err) 590 590 - } 591 591 - 592 592 - e.appendLog("lease-1", []byte("one")) 593 593 - e.appendLog("lease-1", []byte("two")) 594 594 - select { 595 595 - case msg := <-enc.messages: 596 596 - entries := msg.GetEventBatch().GetEvents() 597 597 - if len(entries) != 2 || entries[0].GetSeqno() != 1 || entries[1].GetSeqno() != 2 { 598 598 - t.Fatalf("first batch entries = %+v, want seqnos 1 and 2", entries) 599 599 - } 600 600 - case <-time.After(time.Second): 601 601 - t.Fatal("timed out waiting for coalesced stream batch") 602 602 - } 603 603 - 604 604 - e.appendLog("lease-1", []byte("three")) 605 605 - select { 606 606 - case msg := <-enc.messages: 607 607 - entries := msg.GetEventBatch().GetEvents() 608 608 - if len(entries) != 1 || entries[0].GetSeqno() != 3 { 609 609 - t.Fatalf("second batch entries = %+v, want only seqno 3", entries) 610 610 - } 611 611 - case <-time.After(time.Second): 612 612 - t.Fatal("timed out waiting for second stream batch") 613 613 - } 614 614 - } 615 615 - 616 373 func TestSocketCancellationIndependence(t *testing.T) { 617 374 d := testDB(t) 618 375 n := notifier.New() 619 376 e := &Executor{ 620 620 - db: d, 621 621 - n: &n, 622 622 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 623 623 - active: make(map[string]*reservation), 624 624 - maxLogBytes: 100 * 1024 * 1024, 625 625 - maxControlBytes: 10 * 1024 * 1024, 626 626 - cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 377 377 + db: d, 378 378 + n: &n, 379 379 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 380 380 + active: make(map[string]*reservation), 381 381 + maxOutboxBytes: 10 * 1024 * 1024, 382 382 + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 627 383 } 628 384 if err := e.initOutbox(); err != nil { 629 385 t.Fatal(err) ··· 670 426 func TestMonotonicSnapshots(t *testing.T) { 671 427 enc := newCaptureEncoder() 672 428 e := &Executor{ 673 673 - enc: enc, 674 674 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 675 675 - active: make(map[string]*reservation), 676 676 - maxLogBytes: 100 * 1024 * 1024, 677 677 - maxControlBytes: 10 * 1024 * 1024, 429 429 + enc: enc, 430 430 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 431 431 + active: make(map[string]*reservation), 432 432 + maxOutboxBytes: 10 * 1024 * 1024, 678 433 } 679 434 680 435 e.pushSnapshot() ··· 695 450 func TestTimerRace(t *testing.T) { 696 451 d := testDB(t) 697 452 e := &Executor{ 698 698 - db: d, 699 699 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 700 700 - active: make(map[string]*reservation), 701 701 - maxLogBytes: 100 * 1024 * 1024, 702 702 - maxControlBytes: 10 * 1024 * 1024, 453 453 + db: d, 454 454 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 455 455 + active: make(map[string]*reservation), 456 456 + maxOutboxBytes: 10 * 1024 * 1024, 703 457 } 704 458 if err := e.initOutbox(); err != nil { 705 459 t.Fatal(err) ··· 742 496 } 743 497 } 744 498 745 745 - func TestOversizedLogBehavior(t *testing.T) { 746 746 - d := testDB(t) 747 747 - e := &Executor{ 748 748 - db: d, 749 749 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 750 750 - active: make(map[string]*reservation), 751 751 - maxLogBytes: 10, 752 752 - maxControlBytes: 10 * 1024 * 1024, 753 753 - } 754 754 - if err := e.initOutbox(); err != nil { 755 755 - t.Fatal(err) 756 756 - } 757 757 - 758 758 - e.appendLog("lease-1", []byte("12345678901")) 759 759 - 760 760 - rows, err := d.ListOutboxRows() 761 761 - if err != nil { 762 762 - t.Fatal(err) 763 763 - } 764 764 - if len(rows) != 0 { 765 765 - t.Fatalf("expected log to be dropped, but got %d outbox rows", len(rows)) 766 766 - } 767 767 - if e.droppedLogs != 1 { 768 768 - t.Fatalf("droppedLogs counter = %d, want 1", e.droppedLogs) 769 769 - } 770 770 - } 771 771 - 772 772 - func TestBoundedOutbox(t *testing.T) { 773 773 - d := testDB(t) 774 774 - e := &Executor{ 775 775 - db: d, 776 776 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 777 777 - active: make(map[string]*reservation), 778 778 - maxLogBytes: 100, 779 779 - maxControlBytes: 10 * 1024 * 1024, 780 780 - } 781 781 - if err := e.initOutbox(); err != nil { 782 782 - t.Fatal(err) 783 783 - } 784 784 - 785 785 - e.appendLog("lease-1", []byte("12345")) 786 786 - rows, err := d.ListOutboxRows() 787 787 - if err != nil || len(rows) != 1 { 788 788 - t.Fatalf("expected 1 row, got %d", len(rows)) 789 789 - } 790 790 - 791 791 - e.appendLog("lease-1", []byte("12345")) 792 792 - rows, err = d.ListOutboxRows() 793 793 - if err != nil || len(rows) != 2 { 794 794 - t.Fatalf("expected 2 rows, got %d", len(rows)) 795 795 - } 796 796 - 797 797 - e.appendLog("lease-1", make([]byte, 80)) 798 798 - rows, err = d.ListOutboxRows() 799 799 - if err != nil { 800 800 - t.Fatal(err) 801 801 - } 802 802 - if len(rows) != 2 { 803 803 - t.Fatalf("expected 3rd log to be dropped, got %d rows", len(rows)) 804 804 - } 805 805 - 806 806 - // control status is never dropped even over the cap 807 807 - if err := e.appendStatus("lease-1", &tangled.PipelineStatus{ 808 808 - Status: string(models.StatusKindRunning), 809 809 - }); err != nil { 810 810 - t.Fatal(err) 811 811 - } 812 812 - 813 813 - rows, err = d.ListOutboxRows() 814 814 - if err != nil || len(rows) != 3 { 815 815 - t.Fatalf("expected status to be appended regardless of log cap, got %d rows", len(rows)) 816 816 - } 817 817 - } 818 818 - 819 499 func TestStructuredShutdown(t *testing.T) { 820 500 d := testDB(t) 821 501 n := notifier.New() 822 502 e := &Executor{ 823 823 - db: d, 824 824 - n: &n, 825 825 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 826 826 - active: make(map[string]*reservation), 827 827 - maxLogBytes: 100 * 1024 * 1024, 828 828 - maxControlBytes: 10 * 1024 * 1024, 829 829 - cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 503 503 + db: d, 504 504 + n: &n, 505 505 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 506 506 + active: make(map[string]*reservation), 507 507 + maxOutboxBytes: 10 * 1024 * 1024, 508 508 + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 830 509 } 831 510 if err := e.initOutbox(); err != nil { 832 511 t.Fatal(err)

spindle/mill/handler.go

Reviewed

This file has not been changed.

+59

spindle/mill/integration_test.go

Reviewed

··· 7 7 "log/slog" 8 8 "net/http" 9 9 "net/http/httptest" 10 10 + "os" 10 11 "path/filepath" 11 12 "strings" 12 13 "testing" ··· 51 52 cfg := &config.Config{} 52 53 cfg.Server.Dev = true 53 54 cfg.Server.LogDir = execDir 55 55 + cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") 54 56 cfg.Server.Hostname = "exec-1" 55 57 cfg.Mill.URL = wsURL 56 58 cfg.Mill.Seats = 2 ··· 83 85 } 84 86 defer slot.Release() 85 87 88 88 + logPath := models.LogFilePath(millDir, wid) 89 89 + ch := bn.Subscribe() 90 90 + defer bn.Unsubscribe(ch) 91 91 + 92 92 + sawLogContent := make(chan bool, 1) 93 93 + go func() { 94 94 + for { 95 95 + select { 96 96 + case <-ch: 97 97 + data, err := os.ReadFile(logPath) 98 98 + if err == nil && strings.Contains(string(data), "echo hi") { 99 99 + sawLogContent <- true 100 100 + return 101 101 + } 102 102 + case <-time.After(5 * time.Second): 103 103 + sawLogContent <- false 104 104 + return 105 105 + } 106 106 + } 107 107 + }() 108 108 + 86 109 if err := mill.commitAndWait(placeCtx, wf, nil); err != nil { 87 110 t.Fatalf("commitAndWait: %v, want success", err) 88 111 } 89 112 113 113 + if !<-sawLogContent { 114 114 + t.Fatal("mill live log file never received expected content while job was running") 115 115 + } 116 116 + 117 117 + if !waitForFileRemoval(t, logPath) { 118 118 + t.Fatalf("mill live log file was not removed after terminal artifact was recorded") 119 119 + } 120 120 + 90 121 if !waitForStatus(t, bdb, wid, "running") { 91 122 events, _ := bdb.GetEvents(0, 1000) 92 123 t.Logf("mill events after completion: %+v", events) ··· 124 155 cfg := &config.Config{} 125 156 cfg.Server.Dev = true 126 157 cfg.Server.LogDir = execDir 158 158 + cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") 127 159 cfg.Server.Hostname = "exec-labels" 128 160 cfg.Mill.URL = wsURL 129 161 cfg.Mill.Seats = 2 ··· 177 209 cfg := &config.Config{} 178 210 cfg.Server.Dev = true 179 211 cfg.Server.LogDir = execDir 212 212 + cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") 180 213 cfg.Server.Hostname = name 181 214 cfg.Mill.URL = wsURL 182 215 cfg.Mill.Seats = 1 ··· 275 308 } 276 309 } 277 310 time.Sleep(50 * time.Millisecond) 311 311 + } 312 312 + return false 313 313 + } 314 314 + 315 315 + func waitForLogFileContent(t *testing.T, path, want string) bool { 316 316 + t.Helper() 317 317 + deadline := time.Now().Add(5 * time.Second) 318 318 + for time.Now().Before(deadline) { 319 319 + data, err := os.ReadFile(path) 320 320 + if err == nil && strings.Contains(string(data), want) { 321 321 + return true 322 322 + } 323 323 + time.Sleep(20 * time.Millisecond) 324 324 + } 325 325 + return false 326 326 + } 327 327 + 328 328 + func waitForFileRemoval(t *testing.T, path string) bool { 329 329 + t.Helper() 330 330 + deadline := time.Now().Add(5 * time.Second) 331 331 + for time.Now().Before(deadline) { 332 332 + _, err := os.Stat(path) 333 333 + if os.IsNotExist(err) { 334 334 + return true 335 335 + } 336 336 + time.Sleep(20 * time.Millisecond) 278 337 } 279 338 return false 280 339 }

-28

spindle/mill/lease.go

Reviewed

··· 52 52 reason string 53 53 released bool 54 54 terminal chan *millv1.AttemptResult // buffered(1), RunStep waits here 55 55 - dead chan struct{} // closed when the executor is lost past grace 56 56 - deadOnce sync.Once 57 55 58 56 finishMu sync.Mutex 59 59 - 60 60 - // mill-side byte accounting for streamed logs. the executor caps its own 61 61 - // outbox, but a malicious executor won't, so the mill enforces its own cap 62 62 - logMu sync.Mutex 63 63 - logBytes int64 64 64 - droppedLogs int64 65 57 } 66 58 67 59 func newLease(id, nodeID, epoch, engine string) *RemoteLease { ··· 72 64 engine: engine, 73 65 state: leaseReserved, 74 66 terminal: make(chan *millv1.AttemptResult, 1), 75 75 - dead: make(chan struct{}), 76 67 } 77 68 } 78 69 ··· 170 161 case l.terminal <- res: 171 162 default: 172 163 } 173 173 - } 174 174 - 175 175 - // executor running this lease is gone for good 176 176 - func (l *RemoteLease) markDead() { 177 177 - l.deadOnce.Do(func() { close(l.dead) }) 178 178 - } 179 179 - 180 180 - // counts log lines against the cap. past it, lines drop and count, so a 181 181 - // hostile executor can't fill the mill's disk. firstDrop lets the caller 182 182 - // warn once. resets on restart, restored leases get a fresh budget 183 183 - func (l *RemoteLease) accountLogLine(n, cap int64) (accepted, firstDrop bool) { 184 184 - l.logMu.Lock() 185 185 - defer l.logMu.Unlock() 186 186 - if l.logBytes+n > cap { 187 187 - l.droppedLogs++ 188 188 - return false, l.droppedLogs == 1 189 189 - } 190 190 - l.logBytes += n 191 191 - return true, false 192 164 } 193 165 194 166 // mill's synthetic single step. real steps run on the executor and the

+80 -47

spindle/mill/mill.go

Reviewed

··· 6 6 "errors" 7 7 "fmt" 8 8 "log/slog" 9 9 + "os" 10 10 + "path/filepath" 9 11 "slices" 10 12 "strings" 11 13 "sync" ··· 23 25 ) 24 26 25 27 const ( 26 26 - defaultReconnectGrace = 45 * time.Second 27 27 - defaultJobTimeout = 24 * time.Hour 28 28 - defaultBidTimeout = 5 * time.Second 29 29 - defaultTopK = 3 30 30 - defaultMaxPending = 100 31 31 - defaultMaxLeaseLogBytes = 50 * 1024 * 1024 28 28 + defaultReconnectGrace = 45 * time.Second 29 29 + defaultJobTimeout = 24 * time.Hour 30 30 + defaultBidTimeout = 5 * time.Second 31 31 + defaultTopK = 3 32 32 + defaultMaxPending = 100 32 33 ) 33 34 34 35 type Config struct { 35 35 - MaxPending int 36 36 - ReconnectGrace time.Duration 37 37 - JobTimeout time.Duration 38 38 - BidTimeout time.Duration 39 39 - TopK int 40 40 - LogDir string 41 41 - CancelTimeout time.Duration 42 42 - MaxLeaseLogBytes int64 36 36 + // mill appends live-tailed executor lines here so logview can follow running remote jobs. 37 37 + LogDir string 38 38 + MaxPending int 39 39 + ReconnectGrace time.Duration 40 40 + JobTimeout time.Duration 41 41 + BidTimeout time.Duration 42 42 + TopK int 43 43 + CancelTimeout time.Duration 43 44 } 44 45 45 46 type Mill struct { ··· 79 80 if cfg.CancelTimeout <= 0 { 80 81 cfg.CancelTimeout = 10 * time.Second 81 82 } 82 82 - if cfg.MaxLeaseLogBytes <= 0 { 83 83 - cfg.MaxLeaseLogBytes = defaultMaxLeaseLogBytes 84 84 - } 85 83 return &Mill{ 86 84 l: l, 87 85 cfg: cfg, ··· 623 621 select { 624 622 case res := <-lease.terminal: 625 623 return terminalError(res.Status) 626 626 - case <-lease.dead: 627 627 - m.l.Warn("executor lost while running job", "lease", lease.id, "node", lease.nodeID) 628 628 - return engine.ErrWorkflowFailed 629 624 case <-ctx.Done(): 630 625 if ctx.Err() == context.DeadlineExceeded { 631 626 return engine.ErrTimedOut ··· 644 639 select { 645 640 case res := <-lease.terminal: 646 641 return true, terminalError(res.Status) 647 647 - case <-lease.dead: 648 648 - return true, engine.ErrWorkflowFailed 649 642 case <-ctx.Done(): 650 643 if ctx.Err() == context.DeadlineExceeded { 651 644 return true, engine.ErrTimedOut ··· 728 721 return nil 729 722 } 730 723 if m.db != nil { 731 731 - logPath := models.LogFilePath(m.cfg.LogDir, lease.wid) 732 732 - if err := m.flushCanonicalLog(logPath); err != nil { 733 733 - m.scheduleCleanupLocked(lease) 734 734 - return err 735 735 - } 736 724 if err := m.db.DeleteMillLease(lease.id); err != nil { 737 725 m.scheduleCleanupLocked(lease) 738 726 return err ··· 851 839 ar *millv1.AttemptResult 852 840 } 853 841 var pendingTerminals []pendingTerminal 842 842 + var artifactLeases []*RemoteLease 854 843 finishedInBatch := make(map[string]struct{}) 855 844 856 845 var highestSeqno uint64 = current ··· 906 895 } 907 896 } 908 897 909 909 - case entry.GetLogLine() != nil: 910 910 - raw := entry.GetLogLine().GetRawJson() 911 911 - line := make([]byte, len(raw)+1) 912 912 - copy(line, raw) 913 913 - line[len(raw)] = '\n' 914 914 - accepted, firstDrop := lease.accountLogLine(int64(len(line)), m.cfg.MaxLeaseLogBytes) 915 915 - if !accepted { 916 916 - if firstDrop { 917 917 - m.l.Warn("mill-side log cap reached, dropping further lines", "lease", lease.id, "cap", m.cfg.MaxLeaseLogBytes) 918 918 - } 919 919 - highestSeqno = entry.Seqno 920 920 - continue 921 921 - } 922 922 - logPath := models.LogFilePath(m.cfg.LogDir, lease.wid) 923 923 - if tx != nil { 924 924 - if err := tx.AddCanonicalLog(sess.nodeID, sess.epoch, entry.Seqno, logPath, line); err != nil { 925 925 - return err 926 926 - } 927 927 - } 928 928 - 929 898 case entry.GetAttemptResult() != nil: 930 899 ar := entry.GetAttemptResult() 931 900 statusStr := "success" ··· 958 927 if err := tx.DeleteLease(lease.id); err != nil { 959 928 return err 960 929 } 930 930 + if a := ar.GetLogArtifact(); a != nil { 931 931 + if a.GetRef() == "" { 932 932 + return fmt.Errorf("protocol error: empty log artifact ref") 933 933 + } 934 934 + if !strings.HasPrefix(a.GetHash(), "sha256:") { 935 935 + return fmt.Errorf("protocol error: invalid log artifact hash %q", a.GetHash()) 936 936 + } 937 937 + if tx != nil { 938 938 + if err := tx.InsertArtifactRef(lease.id, lease.wid.Name, a.GetRef(), a.GetHash()); err != nil { 939 939 + return err 940 940 + } 941 941 + } 942 942 + artifactLeases = append(artifactLeases, lease) 943 943 + } 961 944 } 962 945 pendingTerminals = append(pendingTerminals, pendingTerminal{ 963 946 lease: lease, ··· 985 968 return err 986 969 } 987 970 971 971 + if m.cfg.LogDir != "" { 972 972 + for _, lease := range artifactLeases { 973 973 + path := models.LogFilePath(m.cfg.LogDir, lease.wid) 974 974 + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { 975 975 + m.l.Warn("failed to remove live log file after artifact recorded", "path", path, "err", err) 976 976 + } 977 977 + } 978 978 + } 979 979 + 988 980 m.mu.Lock() 989 981 if highestSeqno > m.nodeSeqno[currentKey] { 990 982 m.nodeSeqno[currentKey] = highestSeqno ··· 1015 1007 if err := sess.send(msg); err != nil { 1016 1008 return fmt.Errorf("send ack message: %w", err) 1017 1009 } 1010 1010 + return nil 1011 1011 + } 1012 1012 + func (m *Mill) onLiveLog(sess *millSession, ll *millv1.LiveLog) error { 1013 1013 + if ll == nil || ll.GetLeaseId() == "" { 1014 1014 + return nil 1015 1015 + } 1016 1016 + m.mu.Lock() 1017 1017 + lease := m.leases[ll.GetLeaseId()] 1018 1018 + m.mu.Unlock() 1019 1019 + if lease == nil || lease.nodeID != sess.nodeID { 1020 1020 + return nil 1021 1021 + } 1022 1022 + if m.n != nil { 1023 1023 + m.n.NotifyAll() 1024 1024 + } 1025 1025 + 1026 1026 + raw := ll.GetRawJson() 1027 1027 + if m.cfg.LogDir == "" || len(raw) == 0 { 1028 1028 + return nil 1029 1029 + } 1030 1030 + lease.mu.Lock() 1031 1031 + isDone := (lease.state == leaseDone) 1032 1032 + lease.mu.Unlock() 1033 1033 + if isDone { 1034 1034 + return nil 1035 1035 + } 1036 1036 + 1037 1037 + logPath := models.LogFilePath(m.cfg.LogDir, lease.wid) 1038 1038 + if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { 1039 1039 + m.l.Warn("failed to create log dir", "path", filepath.Dir(logPath), "err", err) 1040 1040 + return nil 1041 1041 + } 1042 1042 + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) 1043 1043 + if err != nil { 1044 1044 + m.l.Warn("failed to open log file", "path", logPath, "err", err) 1045 1045 + return nil 1046 1046 + } 1047 1047 + if _, err := f.Write(raw); err != nil { 1048 1048 + m.l.Warn("failed to write log file", "path", logPath, "err", err) 1049 1049 + } 1050 1050 + _ = f.Close() 1018 1051 return nil 1019 1052 } 1020 1053

+2 -3

spindle/mill/mill_test.go

Reviewed

··· 27 27 Environment: map[string]string{}, 28 28 Steps: []models.Step{remoteStep{}}, 29 29 Data: &millWorkflowState{ 30 30 - TargetEngine: "dummy", 31 31 - RawWorkflow: tangled.Pipeline_Workflow{Name: name}, 32 32 - RawPipeline: tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}, 30 30 + RawWorkflow: tangled.Pipeline_Workflow{Name: name}, 31 31 + RawPipeline: tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}, 33 32 }, 34 33 } 35 34 }

+143 -76

spindle/mill/proto/gen/mill.pb.go

Reviewed

··· 926 926 return 0 927 927 } 928 928 929 929 - // one already-encoded models.LogLine JSON line 930 930 - type LogLine struct { 929 929 + type LogArtifact struct { 931 930 state protoimpl.MessageState `protogen:"open.v1"` 932 932 - RawJson []byte `protobuf:"bytes,1,opt,name=raw_json,json=rawJson,proto3" json:"raw_json,omitempty"` 931 931 + Ref string `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` 932 932 + Hash string `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"` 933 933 unknownFields protoimpl.UnknownFields 934 934 sizeCache protoimpl.SizeCache 935 935 } 936 936 937 937 - func (x *LogLine) Reset() { 938 938 - *x = LogLine{} 937 937 + func (x *LogArtifact) Reset() { 938 938 + *x = LogArtifact{} 939 939 mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] 940 940 ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 941 941 ms.StoreMessageInfo(mi) 942 942 } 943 943 944 944 - func (x *LogLine) String() string { 944 944 + func (x *LogArtifact) String() string { 945 945 return protoimpl.X.MessageStringOf(x) 946 946 } 947 947 948 948 - func (*LogLine) ProtoMessage() {} 948 948 + func (*LogArtifact) ProtoMessage() {} 949 949 950 950 - func (x *LogLine) ProtoReflect() protoreflect.Message { 950 950 + func (x *LogArtifact) ProtoReflect() protoreflect.Message { 951 951 mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] 952 952 if x != nil { 953 953 ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ··· 959 959 return mi.MessageOf(x) 960 960 } 961 961 962 962 - // Deprecated: Use LogLine.ProtoReflect.Descriptor instead. 963 963 - func (*LogLine) Descriptor() ([]byte, []int) { 962 962 + // Deprecated: Use LogArtifact.ProtoReflect.Descriptor instead. 963 963 + func (*LogArtifact) Descriptor() ([]byte, []int) { 964 964 return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{13} 965 965 } 966 966 967 967 - func (x *LogLine) GetRawJson() []byte { 967 967 + func (x *LogArtifact) GetRef() string { 968 968 if x != nil { 969 969 - return x.RawJson 969 969 + return x.Ref 970 970 } 971 971 - return nil 971 971 + return "" 972 972 } 973 973 974 974 + func (x *LogArtifact) GetHash() string { 975 975 + if x != nil { 976 976 + return x.Hash 977 977 + } 978 978 + return "" 979 979 + } 980 980 + 974 981 // terminal outcome of an attempt 975 982 type AttemptResult struct { 976 983 state protoimpl.MessageState `protogen:"open.v1"` 977 984 Status TerminalStatus `protobuf:"varint,1,opt,name=status,proto3,enum=spindle.mill.v1.TerminalStatus" json:"status,omitempty"` 978 985 Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` 979 986 ExitCode int64 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` 987 987 + LogArtifact *LogArtifact `protobuf:"bytes,4,opt,name=log_artifact,json=logArtifact,proto3" json:"log_artifact,omitempty"` 980 988 unknownFields protoimpl.UnknownFields 981 989 sizeCache protoimpl.SizeCache 982 990 } ··· 1032 1040 return 0 1033 1041 } 1034 1042 1043 1043 + func (x *AttemptResult) GetLogArtifact() *LogArtifact { 1044 1044 + if x != nil { 1045 1045 + return x.LogArtifact 1046 1046 + } 1047 1047 + return nil 1048 1048 + } 1049 1049 + 1050 1050 + // live non-replay log frame 1051 1051 + type LiveLog struct { 1052 1052 + state protoimpl.MessageState `protogen:"open.v1"` 1053 1053 + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 1054 1054 + RawJson []byte `protobuf:"bytes,2,opt,name=raw_json,json=rawJson,proto3" json:"raw_json,omitempty"` 1055 1055 + unknownFields protoimpl.UnknownFields 1056 1056 + sizeCache protoimpl.SizeCache 1057 1057 + } 1058 1058 + 1059 1059 + func (x *LiveLog) Reset() { 1060 1060 + *x = LiveLog{} 1061 1061 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] 1062 1062 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1063 1063 + ms.StoreMessageInfo(mi) 1064 1064 + } 1065 1065 + 1066 1066 + func (x *LiveLog) String() string { 1067 1067 + return protoimpl.X.MessageStringOf(x) 1068 1068 + } 1069 1069 + 1070 1070 + func (*LiveLog) ProtoMessage() {} 1071 1071 + 1072 1072 + func (x *LiveLog) ProtoReflect() protoreflect.Message { 1073 1073 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] 1074 1074 + if x != nil { 1075 1075 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1076 1076 + if ms.LoadMessageInfo() == nil { 1077 1077 + ms.StoreMessageInfo(mi) 1078 1078 + } 1079 1079 + return ms 1080 1080 + } 1081 1081 + return mi.MessageOf(x) 1082 1082 + } 1083 1083 + 1084 1084 + // Deprecated: Use LiveLog.ProtoReflect.Descriptor instead. 1085 1085 + func (*LiveLog) Descriptor() ([]byte, []int) { 1086 1086 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{15} 1087 1087 + } 1088 1088 + 1089 1089 + func (x *LiveLog) GetLeaseId() string { 1090 1090 + if x != nil { 1091 1091 + return x.LeaseId 1092 1092 + } 1093 1093 + return "" 1094 1094 + } 1095 1095 + 1096 1096 + func (x *LiveLog) GetRawJson() []byte { 1097 1097 + if x != nil { 1098 1098 + return x.RawJson 1099 1099 + } 1100 1100 + return nil 1101 1101 + } 1102 1102 + 1035 1103 // one event in the executor's stream back to the mill 1036 1104 type Event struct { 1037 1105 state protoimpl.MessageState `protogen:"open.v1"` ··· 1040 1108 // Types that are valid to be assigned to Payload: 1041 1109 // 1042 1110 // *Event_StatusEvent 1043 1043 - // *Event_LogLine 1044 1111 // *Event_AttemptResult 1045 1112 Payload isEvent_Payload `protobuf_oneof:"payload"` 1046 1113 unknownFields protoimpl.UnknownFields ··· 1049 1116 1050 1117 func (x *Event) Reset() { 1051 1118 *x = Event{} 1052 1052 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] 1119 1119 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] 1053 1120 ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1054 1121 ms.StoreMessageInfo(mi) 1055 1122 } ··· 1061 1128 func (*Event) ProtoMessage() {} 1062 1129 1063 1130 func (x *Event) ProtoReflect() protoreflect.Message { 1064 1064 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] 1131 1131 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] 1065 1132 if x != nil { 1066 1133 ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1067 1134 if ms.LoadMessageInfo() == nil { ··· 1074 1141 1075 1142 // Deprecated: Use Event.ProtoReflect.Descriptor instead. 1076 1143 func (*Event) Descriptor() ([]byte, []int) { 1077 1077 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{15} 1144 1144 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16} 1078 1145 } 1079 1146 1080 1147 func (x *Event) GetSeqno() uint64 { ··· 1107 1174 return nil 1108 1175 } 1109 1176 1110 1110 - func (x *Event) GetLogLine() *LogLine { 1111 1111 - if x != nil { 1112 1112 - if x, ok := x.Payload.(*Event_LogLine); ok { 1113 1113 - return x.LogLine 1114 1114 - } 1115 1115 - } 1116 1116 - return nil 1117 1117 - } 1118 1118 - 1119 1177 func (x *Event) GetAttemptResult() *AttemptResult { 1120 1178 if x != nil { 1121 1179 if x, ok := x.Payload.(*Event_AttemptResult); ok { ··· 1133 1191 StatusEvent *StatusEvent `protobuf:"bytes,3,opt,name=status_event,json=statusEvent,proto3,oneof"` 1134 1192 } 1135 1193 1136 1136 - type Event_LogLine struct { 1137 1137 - LogLine *LogLine `protobuf:"bytes,4,opt,name=log_line,json=logLine,proto3,oneof"` 1138 1138 - } 1139 1139 - 1140 1194 type Event_AttemptResult struct { 1141 1141 - AttemptResult *AttemptResult `protobuf:"bytes,5,opt,name=attempt_result,json=attemptResult,proto3,oneof"` 1195 1195 + AttemptResult *AttemptResult `protobuf:"bytes,4,opt,name=attempt_result,json=attemptResult,proto3,oneof"` 1142 1196 } 1143 1197 1144 1198 func (*Event_StatusEvent) isEvent_Payload() {} 1145 1199 1146 1146 - func (*Event_LogLine) isEvent_Payload() {} 1147 1147 - 1148 1200 func (*Event_AttemptResult) isEvent_Payload() {} 1149 1201 1150 1202 // a flushed bundle of events ··· 1158 1210 1159 1211 func (x *EventBatch) Reset() { 1160 1212 *x = EventBatch{} 1161 1161 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] 1213 1213 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] 1162 1214 ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1163 1215 ms.StoreMessageInfo(mi) 1164 1216 } ··· 1170 1222 func (*EventBatch) ProtoMessage() {} 1171 1223 1172 1224 func (x *EventBatch) ProtoReflect() protoreflect.Message { 1173 1173 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] 1225 1225 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] 1174 1226 if x != nil { 1175 1227 ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1176 1228 if ms.LoadMessageInfo() == nil { ··· 1183 1235 1184 1236 // Deprecated: Use EventBatch.ProtoReflect.Descriptor instead. 1185 1237 func (*EventBatch) Descriptor() ([]byte, []int) { 1186 1186 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16} 1238 1238 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17} 1187 1239 } 1188 1240 1189 1241 func (x *EventBatch) GetEpoch() string { ··· 1211 1263 1212 1264 func (x *Ack) Reset() { 1213 1265 *x = Ack{} 1214 1214 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] 1266 1266 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] 1215 1267 ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1216 1268 ms.StoreMessageInfo(mi) 1217 1269 } ··· 1223 1275 func (*Ack) ProtoMessage() {} 1224 1276 1225 1277 func (x *Ack) ProtoReflect() protoreflect.Message { 1226 1226 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] 1278 1278 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] 1227 1279 if x != nil { 1228 1280 ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1229 1281 if ms.LoadMessageInfo() == nil { ··· 1236 1288 1237 1289 // Deprecated: Use Ack.ProtoReflect.Descriptor instead. 1238 1290 func (*Ack) Descriptor() ([]byte, []int) { 1239 1239 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17} 1291 1291 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18} 1240 1292 } 1241 1293 1242 1294 func (x *Ack) GetEpoch() string { ··· 1267 1319 CancelAck *CancelAck `protobuf:"bytes,10,opt,name=cancel_ack,json=cancelAck,proto3" json:"cancel_ack,omitempty"` 1268 1320 EventBatch *EventBatch `protobuf:"bytes,11,opt,name=event_batch,json=eventBatch,proto3" json:"event_batch,omitempty"` 1269 1321 Ack *Ack `protobuf:"bytes,12,opt,name=ack,proto3" json:"ack,omitempty"` 1322 1322 + LiveLog *LiveLog `protobuf:"bytes,13,opt,name=live_log,json=liveLog,proto3" json:"live_log,omitempty"` 1270 1323 unknownFields protoimpl.UnknownFields 1271 1324 sizeCache protoimpl.SizeCache 1272 1325 } 1273 1326 1274 1327 func (x *Message) Reset() { 1275 1328 *x = Message{} 1276 1276 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] 1329 1329 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] 1277 1330 ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1278 1331 ms.StoreMessageInfo(mi) 1279 1332 } ··· 1285 1338 func (*Message) ProtoMessage() {} 1286 1339 1287 1340 func (x *Message) ProtoReflect() protoreflect.Message { 1288 1288 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] 1341 1341 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] 1289 1342 if x != nil { 1290 1343 ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1291 1344 if ms.LoadMessageInfo() == nil { ··· 1298 1351 1299 1352 // Deprecated: Use Message.ProtoReflect.Descriptor instead. 1300 1353 func (*Message) Descriptor() ([]byte, []int) { 1301 1301 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18} 1354 1354 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{19} 1302 1355 } 1303 1356 1304 1357 func (x *Message) GetHello() *Hello { ··· 1385 1438 return nil 1386 1439 } 1387 1440 1441 1441 + func (x *Message) GetLiveLog() *LiveLog { 1442 1442 + if x != nil { 1443 1443 + return x.LiveLog 1444 1444 + } 1445 1445 + return nil 1446 1446 + } 1447 1447 + 1388 1448 var File_spindle_mill_v1_mill_proto protoreflect.FileDescriptor 1389 1449 1390 1450 const file_spindle_mill_v1_mill_proto_rawDesc = "" + ··· 1444 1504 "\x06status\x18\x01 \x01(\x0e2\".spindle.mill.v1.NonterminalStatusB\n" + 1445 1505 "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" + 1446 1506 "\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" + 1447 1447 - "\texit_code\x18\x03 \x01(\x03R\bexitCode\"-\n" + 1448 1448 - "\aLogLine\x12\"\n" + 1449 1449 - "\braw_json\x18\x01 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\x92\x01\n" + 1507 1507 + "\texit_code\x18\x03 \x01(\x03R\bexitCode\"E\n" + 1508 1508 + "\vLogArtifact\x12\x19\n" + 1509 1509 + "\x03ref\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x03ref\x12\x1b\n" + 1510 1510 + "\x04hash\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x04hash\"\xd3\x01\n" + 1450 1511 "\rAttemptResult\x12C\n" + 1451 1512 "\x06status\x18\x01 \x01(\x0e2\x1f.spindle.mill.v1.TerminalStatusB\n" + 1452 1513 "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" + 1453 1514 "\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" + 1454 1454 - "\texit_code\x18\x03 \x01(\x03R\bexitCode\"\x9f\x02\n" + 1515 1515 + "\texit_code\x18\x03 \x01(\x03R\bexitCode\x12?\n" + 1516 1516 + "\flog_artifact\x18\x04 \x01(\v2\x1c.spindle.mill.v1.LogArtifactR\vlogArtifact\"Q\n" + 1517 1517 + "\aLiveLog\x12\"\n" + 1518 1518 + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\"\n" + 1519 1519 + "\braw_json\x18\x02 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\xe8\x01\n" + 1455 1520 "\x05Event\x12\x1d\n" + 1456 1521 "\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12\"\n" + 1457 1522 "\blease_id\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12A\n" + 1458 1458 - "\fstatus_event\x18\x03 \x01(\v2\x1c.spindle.mill.v1.StatusEventH\x00R\vstatusEvent\x125\n" + 1459 1459 - "\blog_line\x18\x04 \x01(\v2\x18.spindle.mill.v1.LogLineH\x00R\alogLine\x12G\n" + 1460 1460 - "\x0eattempt_result\x18\x05 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResultB\x10\n" + 1523 1523 + "\fstatus_event\x18\x03 \x01(\v2\x1c.spindle.mill.v1.StatusEventH\x00R\vstatusEvent\x12G\n" + 1524 1524 + "\x0eattempt_result\x18\x04 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResultB\x10\n" + 1461 1525 "\apayload\x12\x05\xbaH\x02\b\x01\"e\n" + 1462 1526 "\n" + 1463 1527 "EventBatch\x12\x1d\n" + ··· 1465 1529 "\x06events\x18\x02 \x03(\v2\x16.spindle.mill.v1.EventB\b\xbaH\x05\x92\x01\x02\b\x01R\x06events\"D\n" + 1466 1530 "\x03Ack\x12\x1d\n" + 1467 1531 "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1e\n" + 1468 1468 - "\vup_to_seqno\x18\x02 \x01(\x04R\tupToSeqno\"\xf9\x06\n" + 1532 1532 + "\vup_to_seqno\x18\x02 \x01(\x04R\tupToSeqno\"\xb8\a\n" + 1469 1533 "\aMessage\x12,\n" + 1470 1534 "\x05hello\x18\x01 \x01(\v2\x16.spindle.mill.v1.HelloR\x05hello\x12/\n" + 1471 1535 "\x06resume\x18\x02 \x01(\v2\x17.spindle.mill.v1.ResumeR\x06resume\x12B\n" + ··· 1481 1545 " \x01(\v2\x1a.spindle.mill.v1.CancelAckR\tcancelAck\x12<\n" + 1482 1546 "\vevent_batch\x18\v \x01(\v2\x1b.spindle.mill.v1.EventBatchR\n" + 1483 1547 "eventBatch\x12&\n" + 1484 1484 - "\x03ack\x18\f \x01(\v2\x14.spindle.mill.v1.AckR\x03ack:\x9b\x01\xbaH\x97\x01\"\x94\x01\n" + 1548 1548 + "\x03ack\x18\f \x01(\v2\x14.spindle.mill.v1.AckR\x03ack\x123\n" + 1549 1549 + "\blive_log\x18\r \x01(\v2\x18.spindle.mill.v1.LiveLogR\aliveLog:\xa5\x01\xbaH\xa1\x01\"\x9e\x01\n" + 1485 1550 "\x05hello\n" + 1486 1551 "\x06resume\n" + 1487 1552 "\rnode_snapshot\n" + ··· 1494 1559 "\n" + 1495 1560 "cancel_ack\n" + 1496 1561 "\vevent_batch\n" + 1497 1497 - "\x03ack\x10\x01*f\n" + 1562 1562 + "\x03ack\n" + 1563 1563 + "\blive_log\x10\x01*f\n" + 1498 1564 "\vRejectClass\x12\x1c\n" + 1499 1565 "\x18REJECT_CLASS_UNSPECIFIED\x10\x00\x12\x1a\n" + 1500 1566 "\x16REJECT_CLASS_TRANSIENT\x10\x01\x12\x1d\n" + ··· 1523 1589 } 1524 1590 1525 1591 var file_spindle_mill_v1_mill_proto_enumTypes = make([]protoimpl.EnumInfo, 3) 1526 1526 - var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 21) 1592 1592 + var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 22) 1527 1593 var file_spindle_mill_v1_mill_proto_goTypes = []any{ 1528 1594 (RejectClass)(0), // 0: spindle.mill.v1.RejectClass 1529 1595 (NonterminalStatus)(0), // 1: spindle.mill.v1.NonterminalStatus ··· 1541 1607 (*CancelAttempt)(nil), // 13: spindle.mill.v1.CancelAttempt 1542 1608 (*CancelAck)(nil), // 14: spindle.mill.v1.CancelAck 1543 1609 (*StatusEvent)(nil), // 15: spindle.mill.v1.StatusEvent 1544 1544 - (*LogLine)(nil), // 16: spindle.mill.v1.LogLine 1610 1610 + (*LogArtifact)(nil), // 16: spindle.mill.v1.LogArtifact 1545 1611 (*AttemptResult)(nil), // 17: spindle.mill.v1.AttemptResult 1546 1546 - (*Event)(nil), // 18: spindle.mill.v1.Event 1547 1547 - (*EventBatch)(nil), // 19: spindle.mill.v1.EventBatch 1548 1548 - (*Ack)(nil), // 20: spindle.mill.v1.Ack 1549 1549 - (*Message)(nil), // 21: spindle.mill.v1.Message 1550 1550 - nil, // 22: spindle.mill.v1.EngineAvailability.LoadEntry 1551 1551 - nil, // 23: spindle.mill.v1.NodeSnapshot.EnginesEntry 1612 1612 + (*LiveLog)(nil), // 18: spindle.mill.v1.LiveLog 1613 1613 + (*Event)(nil), // 19: spindle.mill.v1.Event 1614 1614 + (*EventBatch)(nil), // 20: spindle.mill.v1.EventBatch 1615 1615 + (*Ack)(nil), // 21: spindle.mill.v1.Ack 1616 1616 + (*Message)(nil), // 22: spindle.mill.v1.Message 1617 1617 + nil, // 23: spindle.mill.v1.EngineAvailability.LoadEntry 1618 1618 + nil, // 24: spindle.mill.v1.NodeSnapshot.EnginesEntry 1552 1619 } 1553 1620 var file_spindle_mill_v1_mill_proto_depIdxs = []int32{ 1554 1554 - 22, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry 1555 1555 - 23, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry 1621 1621 + 23, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry 1622 1622 + 24, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry 1556 1623 0, // 2: spindle.mill.v1.ReserveResult.reject_class:type_name -> spindle.mill.v1.RejectClass 1557 1624 9, // 3: spindle.mill.v1.CommitLease.secrets:type_name -> spindle.mill.v1.Secret 1558 1625 1, // 4: spindle.mill.v1.StatusEvent.status:type_name -> spindle.mill.v1.NonterminalStatus 1559 1626 2, // 5: spindle.mill.v1.AttemptResult.status:type_name -> spindle.mill.v1.TerminalStatus 1560 1560 - 15, // 6: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent 1561 1561 - 16, // 7: spindle.mill.v1.Event.log_line:type_name -> spindle.mill.v1.LogLine 1627 1627 + 16, // 6: spindle.mill.v1.AttemptResult.log_artifact:type_name -> spindle.mill.v1.LogArtifact 1628 1628 + 15, // 7: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent 1562 1629 17, // 8: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult 1563 1563 - 18, // 9: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event 1630 1630 + 19, // 9: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event 1564 1631 3, // 10: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello 1565 1632 4, // 11: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume 1566 1633 6, // 12: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot ··· 1571 1638 12, // 17: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease 1572 1639 13, // 18: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt 1573 1640 14, // 19: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck 1574 1574 - 19, // 20: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch 1575 1575 - 20, // 21: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack 1576 1576 - 5, // 22: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability 1577 1577 - 23, // [23:23] is the sub-list for method output_type 1578 1578 - 23, // [23:23] is the sub-list for method input_type 1579 1579 - 23, // [23:23] is the sub-list for extension type_name 1580 1580 - 23, // [23:23] is the sub-list for extension extendee 1581 1581 - 0, // [0:23] is the sub-list for field type_name 1641 1641 + 20, // 20: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch 1642 1642 + 21, // 21: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack 1643 1643 + 18, // 22: spindle.mill.v1.Message.live_log:type_name -> spindle.mill.v1.LiveLog 1644 1644 + 5, // 23: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability 1645 1645 + 24, // [24:24] is the sub-list for method output_type 1646 1646 + 24, // [24:24] is the sub-list for method input_type 1647 1647 + 24, // [24:24] is the sub-list for extension type_name 1648 1648 + 24, // [24:24] is the sub-list for extension extendee 1649 1649 + 0, // [0:24] is the sub-list for field type_name 1582 1650 } 1583 1651 1584 1652 func init() { file_spindle_mill_v1_mill_proto_init() } ··· 1586 1654 if File_spindle_mill_v1_mill_proto != nil { 1587 1655 return 1588 1656 } 1589 1589 - file_spindle_mill_v1_mill_proto_msgTypes[15].OneofWrappers = []any{ 1657 1657 + file_spindle_mill_v1_mill_proto_msgTypes[16].OneofWrappers = []any{ 1590 1658 (*Event_StatusEvent)(nil), 1591 1591 - (*Event_LogLine)(nil), 1592 1659 (*Event_AttemptResult)(nil), 1593 1660 } 1594 1661 type x struct{} ··· 1597 1664 GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 1598 1665 RawDescriptor: unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc)), 1599 1666 NumEnums: 3, 1600 1600 - NumMessages: 21, 1667 1667 + NumMessages: 22, 1601 1668 NumExtensions: 0, 1602 1669 NumServices: 0, 1603 1670 },

spindle/mill/proto/protocol.go

Reviewed

This file has not been changed.

spindle/mill/proto/protocol_test.go

Reviewed

This file has not been changed.

+13 -6

spindle/mill/proto/spindle/mill/v1/mill.proto

Reviewed

··· 115 115 int64 exit_code = 3; 116 116 } 117 117 118 118 - // one already-encoded models.LogLine JSON line 119 119 - message LogLine { 120 120 - bytes raw_json = 1 [(buf.validate.field).bytes.min_len = 1]; 118 118 + message LogArtifact { 119 119 + string ref = 1 [(buf.validate.field).string.min_len = 1]; 120 120 + string hash = 2 [(buf.validate.field).string.min_len = 1]; 121 121 } 122 122 123 123 // terminal outcome of an attempt ··· 128 128 }]; 129 129 string error = 2 [(buf.validate.field).string.max_bytes = 65536]; 130 130 int64 exit_code = 3; 131 131 + LogArtifact log_artifact = 4; 131 132 } 132 133 134 134 + // live non-replay log frame 135 135 + message LiveLog { 136 136 + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 137 137 + bytes raw_json = 2 [(buf.validate.field).bytes.min_len = 1]; 138 138 + } 139 139 + 133 140 // one event in the executor's stream back to the mill 134 141 message Event { 135 142 uint64 seqno = 1 [(buf.validate.field).uint64.gt = 0]; ··· 138 145 oneof payload { 139 146 option (buf.validate.oneof).required = true; 140 147 StatusEvent status_event = 3; 141 141 - LogLine log_line = 4; 142 142 - AttemptResult attempt_result = 5; 148 148 + AttemptResult attempt_result = 4; 143 149 } 144 150 } 145 151 ··· 160 166 fields: [ 161 167 "hello", "resume", "node_snapshot", "reserve_seat", "reserve_result", 162 168 "commit_lease", "committed", "release_lease", "cancel_attempt", 163 163 - "cancel_ack", "event_batch", "ack" 169 169 + "cancel_ack", "event_batch", "ack", "live_log" 164 170 ], 165 171 required: true 166 172 }; ··· 177 183 CancelAck cancel_ack = 10; 178 184 EventBatch event_batch = 11; 179 185 Ack ack = 12; 186 186 + LiveLog live_log = 13; 180 187 }

spindle/mill/proto/ws.go

Reviewed

This file has not been changed.

+1 -106

spindle/mill/restore.go

Reviewed

··· 2 2 3 3 import ( 4 4 "fmt" 5 5 - "os" 6 6 - "path/filepath" 7 7 - "time" 8 8 - 9 5 "tangled.org/core/spindle/db" 10 6 millproto "tangled.org/core/spindle/mill/proto" 11 7 millv1 "tangled.org/core/spindle/mill/proto/gen" 12 8 "tangled.org/core/spindle/models" 9 9 + "time" 13 10 ) 14 11 15 12 const ( ··· 73 70 // executors get a grace window to reconnect and claim their leases 74 71 time.AfterFunc(m.cfg.ReconnectGrace, m.sweepUnclaimedOrphans) 75 72 } 76 76 - // crash between terminal and cleanup leaves rows with no owner to 77 77 - // flush them, do it now 78 78 - return m.flushOrphanedCanonicalLogs() 79 79 - } 80 80 - 81 81 - // flushes from db to a file since db is only used while wf is live 82 82 - func (m *Mill) flushCanonicalLog(workflow string) error { 83 83 - logs, err := m.db.ListCanonicalLogsFor(workflow) 84 84 - if err != nil { 85 85 - return fmt.Errorf("list canonical logs for %q: %w", workflow, err) 86 86 - } 87 87 - payloads := make([][]byte, 0, len(logs)) 88 88 - for _, log := range logs { 89 89 - payloads = append(payloads, log.Payload) 90 90 - } 91 91 - // no lines streamed means no archive file 92 92 - if len(payloads) > 0 { 93 93 - if err := writeCanonicalFile(workflow, payloads); err != nil { 94 94 - return err 95 95 - } 96 96 - } 97 97 - // delete once the file is durable, a retry just rewrites the same content 98 98 - return m.db.DeleteCanonicalLogs(workflow) 99 99 - } 100 100 - 101 101 - // flushes rows whose lease row is gone 102 102 - func (m *Mill) flushOrphanedCanonicalLogs() error { 103 103 - logs, err := m.db.ListCanonicalLogs() 104 104 - if err != nil { 105 105 - return fmt.Errorf("list canonical logs: %w", err) 106 106 - } 107 107 - leases, err := m.db.ListMillLeases() 108 108 - if err != nil { 109 109 - return err 110 110 - } 111 111 - live := make(map[string]struct{}, len(leases)) 112 112 - for _, l := range leases { 113 113 - path := models.LogFilePath(m.cfg.LogDir, models.WorkflowId{ 114 114 - PipelineId: models.PipelineId{Knot: l.Knot, Rkey: l.Rkey}, 115 115 - Name: l.Workflow, 116 116 - }) 117 117 - live[path] = struct{}{} 118 118 - } 119 119 - flushed := make(map[string]struct{}) 120 120 - for _, log := range logs { 121 121 - if _, ok := live[log.Workflow]; ok { 122 122 - continue 123 123 - } 124 124 - if _, done := flushed[log.Workflow]; done { 125 125 - continue 126 126 - } 127 127 - if err := m.flushCanonicalLog(log.Workflow); err != nil { 128 128 - return fmt.Errorf("flush orphaned canonical log %q: %w", log.Workflow, err) 129 129 - } 130 130 - flushed[log.Workflow] = struct{}{} 131 131 - } 132 73 return nil 133 74 } 134 134 - 135 135 - // atomically replaces path with the payloads to prevent half-written reads 136 136 - func writeCanonicalFile(path string, payloads [][]byte) error { 137 137 - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { 138 138 - return fmt.Errorf("create canonical log directory %q: %w", path, err) 139 139 - } 140 140 - file, err := os.CreateTemp(filepath.Dir(path), ".mill-log-*") 141 141 - if err != nil { 142 142 - return fmt.Errorf("create canonical log %q: %w", path, err) 143 143 - } 144 144 - tmpPath := file.Name() 145 145 - defer func() { 146 146 - _ = file.Close() 147 147 - _ = os.Remove(tmpPath) 148 148 - }() 149 149 - for _, p := range payloads { 150 150 - if _, err := file.Write(p); err != nil { 151 151 - return fmt.Errorf("rebuild canonical log %q: %w", path, err) 152 152 - } 153 153 - } 154 154 - if err := file.Close(); err != nil { 155 155 - return err 156 156 - } 157 157 - if err := os.Chmod(tmpPath, 0644); err != nil { 158 158 - return err 159 159 - } 160 160 - if err := os.Rename(tmpPath, path); err != nil { 161 161 - return err 162 162 - } 163 163 - return nil 164 164 - } 165 165 - 166 75 func (m *Mill) sweepUnclaimedOrphans() { 167 76 m.mu.Lock() 168 77 var unclaimed []*RemoteLease ··· 309 218 default: 310 219 return millv1.TerminalStatus_TERMINAL_STATUS_UNSPECIFIED 311 220 } 312 312 - } 313 313 - 314 314 - func derefString(p *string) string { 315 315 - if p == nil { 316 316 - return "" 317 317 - } 318 318 - return *p 319 319 - } 320 320 - 321 321 - func derefInt64(p *int64) int64 { 322 322 - if p == nil { 323 323 - return 0 324 324 - } 325 325 - return *p 326 221 }

+6 -166

spindle/mill/restore_test.go

Reviewed

··· 2 2 3 3 import ( 4 4 "context" 5 5 - "os" 6 5 "path/filepath" 7 6 "testing" 8 7 "time" ··· 47 46 }); err != nil { 48 47 t.Fatalf("SaveMillLease: %v", err) 49 48 } 50 50 - if err := bdb.SetExecutorCursor("node-1", "inc-1", 7); err != nil { 51 51 - t.Fatalf("SetExecutorCursor: %v", err) 49 49 + if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 7) }); err != nil { 50 50 + t.Fatalf("AdvanceCursor: %v", err) 52 51 } 53 52 54 53 m2 := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) ··· 76 75 } 77 76 } 78 77 79 79 - func TestRestoreStateFlushesOrphanedCanonicalLogs(t *testing.T) { 80 80 - _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 81 81 - logPath := filepath.Join(t.TempDir(), "logs", "workflow.jsonl") 82 82 - if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { 83 83 - if err := tx.AddCanonicalLog("node-1", "inc-1", 1, logPath, []byte("{\"line\":1}\n")); err != nil { 84 84 - return err 85 85 - } 86 86 - return tx.AddCanonicalLog("node-1", "inc-1", 2, logPath, []byte("{\"line\":2}\n")) 87 87 - }); err != nil { 88 88 - t.Fatalf("seed canonical logs: %v", err) 89 89 - } 90 90 - if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { 91 91 - t.Fatalf("create log directory: %v", err) 92 92 - } 93 93 - if err := os.WriteFile(logPath, []byte("stale\n"), 0644); err != nil { 94 94 - t.Fatalf("seed stale log: %v", err) 95 95 - } 96 96 - 97 97 - // rows with no lease row left (terminal committed, crash before 98 98 - // cleanup) are flushed to the cold archive at boot 99 99 - restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 100 100 - got, err := os.ReadFile(logPath) 101 101 - if err != nil { 102 102 - t.Fatalf("read flushed log: %v", err) 103 103 - } 104 104 - if want := "{\"line\":1}\n{\"line\":2}\n"; string(got) != want { 105 105 - t.Fatalf("flushed log = %q, want %q", got, want) 106 106 - } 107 107 - logs, err := bdb.ListCanonicalLogs() 108 108 - if err != nil { 109 109 - t.Fatal(err) 110 110 - } 111 111 - if len(logs) != 0 { 112 112 - t.Fatalf("canonical logs retained after orphan flush: %+v", logs) 113 113 - } 114 114 - } 115 115 - 116 116 - func TestLiveLogStaysInTableUntilCleanup(t *testing.T) { 117 117 - logDir := t.TempDir() 118 118 - m, bdb := restoreTestMill(t, Config{LogDir: logDir, ReconnectGrace: time.Minute}) 119 119 - lease := newLease("lease-1", "node-1", "inc-1", "dummy") 120 120 - lease.wid = models.WorkflowId{ 121 121 - PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, 122 122 - Name: "build", 123 123 - } 124 124 - m.mu.Lock() 125 125 - m.leases[lease.id] = lease 126 126 - m.mu.Unlock() 127 127 - 128 128 - sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 129 129 - m.attachSession(sess) 130 130 - if err := m.onEventBatch(sess, &millv1.EventBatch{ 131 131 - Epoch: "inc-1", 132 132 - Events: []*millv1.Event{{ 133 133 - Seqno: 1, 134 134 - LeaseId: lease.id, 135 135 - Payload: &millv1.Event_LogLine{LogLine: &millv1.LogLine{ 136 136 - RawJson: []byte(`{"line":1}`), 137 137 - }}, 138 138 - }}, 139 139 - }); err != nil { 140 140 - t.Fatalf("onEventBatch: %v", err) 141 141 - } 142 142 - 143 143 - // while the job runs, the table is the log: no file exists yet 144 144 - logPath := models.LogFilePath(logDir, lease.wid) 145 145 - if _, err := os.Stat(logPath); !os.IsNotExist(err) { 146 146 - t.Fatalf("log file exists mid-run, table should be the only store") 147 147 - } 148 148 - logs, err := bdb.ListCanonicalLogsFor(logPath) 149 149 - if err != nil { 150 150 - t.Fatal(err) 151 151 - } 152 152 - if len(logs) != 1 || string(logs[0].Payload) != "{\"line\":1}\n" { 153 153 - t.Fatalf("canonical rows = %+v, want the one streamed line", logs) 154 154 - } 155 155 - 156 156 - // cleanup materializes the cold archive file and drops the hot rows 157 157 - if err := m.cleanupLease(lease); err != nil { 158 158 - t.Fatalf("cleanupLease: %v", err) 159 159 - } 160 160 - got, err := os.ReadFile(logPath) 161 161 - if err != nil { 162 162 - t.Fatal(err) 163 163 - } 164 164 - if want := "{\"line\":1}\n"; string(got) != want { 165 165 - t.Fatalf("flushed log = %q, want %q", got, want) 166 166 - } 167 167 - logs, err = bdb.ListCanonicalLogs() 168 168 - if err != nil { 169 169 - t.Fatal(err) 170 170 - } 171 171 - if len(logs) != 0 { 172 172 - t.Fatalf("canonical logs retained after cleanup: %+v", logs) 173 173 - } 174 174 - } 175 175 - 176 176 - func TestMillSideLogCapDropsBeyondBudget(t *testing.T) { 177 177 - logDir := t.TempDir() 178 178 - m, bdb := restoreTestMill(t, Config{LogDir: logDir, ReconnectGrace: time.Minute, MaxLeaseLogBytes: 12}) 179 179 - lease := newLease("lease-1", "node-1", "inc-1", "dummy") 180 180 - lease.wid = models.WorkflowId{ 181 181 - PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, 182 182 - Name: "build", 183 183 - } 184 184 - m.mu.Lock() 185 185 - m.leases[lease.id] = lease 186 186 - m.mu.Unlock() 187 187 - 188 188 - sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 189 189 - m.attachSession(sess) 190 190 - logLine := func(seqno uint64, body string) *millv1.Event { 191 191 - return &millv1.Event{ 192 192 - Seqno: seqno, 193 193 - LeaseId: lease.id, 194 194 - Payload: &millv1.Event_LogLine{LogLine: &millv1.LogLine{ 195 195 - RawJson: []byte(body), 196 196 - }}, 197 197 - } 198 198 - } 199 199 - // each line is 11 bytes on disk; the cap of 12 admits exactly one 200 200 - if err := m.onEventBatch(sess, &millv1.EventBatch{ 201 201 - Epoch: "inc-1", 202 202 - Events: []*millv1.Event{logLine(1, `{"line":1}`), logLine(2, `{"line":2}`)}, 203 203 - }); err != nil { 204 204 - t.Fatalf("onEventBatch: %v", err) 205 205 - } 206 206 - 207 207 - // the dropped line never reaches the table, so it can never be flushed 208 208 - logPath := models.LogFilePath(logDir, lease.wid) 209 209 - logs, err := bdb.ListCanonicalLogsFor(logPath) 210 210 - if err != nil { 211 211 - t.Fatal(err) 212 212 - } 213 213 - if len(logs) != 1 || string(logs[0].Payload) != "{\"line\":1}\n" { 214 214 - t.Fatalf("canonical rows = %+v, want only the first line", logs) 215 215 - } 216 216 - if lease.droppedLogs != 1 { 217 217 - t.Fatalf("droppedLogs = %d, want 1", lease.droppedLogs) 218 218 - } 219 219 - if err := m.cleanupLease(lease); err != nil { 220 220 - t.Fatalf("cleanupLease: %v", err) 221 221 - } 222 222 - got, err := os.ReadFile(logPath) 223 223 - if err != nil { 224 224 - t.Fatal(err) 225 225 - } 226 226 - if want := "{\"line\":1}\n"; string(got) != want { 227 227 - t.Fatalf("flushed log = %q, want only the first line %q", got, want) 228 228 - } 229 229 - logs, err = bdb.ListCanonicalLogs() 230 230 - if err != nil { 231 231 - t.Fatal(err) 232 232 - } 233 233 - if len(logs) != 0 { 234 234 - t.Fatalf("canonical logs = %+v, want none after cleanup", logs) 235 235 - } 236 236 - } 237 237 - 238 78 func TestOrphanTerminalAuthorsStatusRow(t *testing.T) { 239 79 _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 240 80 if err := bdb.SaveMillLease(db.MillLease{ ··· 243 83 }); err != nil { 244 84 t.Fatalf("SaveMillLease: %v", err) 245 85 } 246 246 - if err := bdb.SetExecutorCursor("node-1", "inc-1", 3); err != nil { 247 247 - t.Fatalf("SetExecutorCursor: %v", err) 86 86 + if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 3) }); err != nil { 87 87 + t.Fatalf("AdvanceCursor: %v", err) 248 88 } 249 89 250 90 m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) ··· 467 307 }); err != nil { 468 308 t.Fatalf("SaveMillLease: %v", err) 469 309 } 470 470 - if err := bdb.SetExecutorCursor("node-1", "inc-1", 3); err != nil { 471 471 - t.Fatalf("SetExecutorCursor: %v", err) 310 310 + if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 3) }); err != nil { 311 311 + t.Fatalf("AdvanceCursor: %v", err) 472 312 } 473 313 if _, err := bdb.Exec(` 474 314 create trigger reject_orphan_lease_delete

+2

spindle/mill/session.go

Reviewed

··· 151 151 return m.onEventBatch(s, msg.GetEventBatch()) 152 152 case msg.GetCancelAck() != nil: 153 153 m.onCancelAck(s, msg.GetCancelAck()) 154 154 + case msg.GetLiveLog() != nil: 155 155 + return m.onLiveLog(s, msg.GetLiveLog()) 154 156 default: 155 157 s.l.Warn("session received unexpected message", "node", s.nodeID) 156 158 }

spindle/mill/token.go

Reviewed

This file has not been changed.

+64 -26

spindle/server.go

Reviewed

··· 32 32 "tangled.org/core/rbac" 33 33 "tangled.org/core/repoident" 34 34 "tangled.org/core/repoverify" 35 35 + "tangled.org/core/spindle/artifactstore" 35 36 "tangled.org/core/spindle/config" 36 37 "tangled.org/core/spindle/db" 37 38 "tangled.org/core/spindle/engine" ··· 74 75 motd []byte 75 76 motdMu sync.RWMutex 76 77 rootCtx context.Context 77 77 - 78 78 + store artifactstore.Store 79 79 + stores *artifactstore.Stores 80 80 + reader artifactstore.Reader 78 81 // set only when this spindle hosts the mill or joins one as an executor 79 82 mill *mill.Mill 80 83 exec *executor.Executor ··· 102 105 motd: defaultMotd, 103 106 rootCtx: ctx, 104 107 } 108 108 + diskFallback := "" 109 109 + if cfg.Role == config.RoleStandalone { 110 110 + diskFallback = cfg.Server.LogDir 111 111 + if cfg.ArtifactStores.Disk.Dir == "" { 112 112 + logger.Warn("using SPINDLE_SERVER_LOG_DIR as the implicit disk artifact store; configure SPINDLE_ARTIFACT_STORES_DISK_DIR explicitly") 113 113 + } 114 114 + } 115 115 + stores, err := artifactstore.NewStores(cfg.ArtifactStores, diskFallback, cfg.LegacyS3.LogBucket) 116 116 + if err != nil { 117 117 + return nil, fmt.Errorf("failed to setup artifact stores: %w", err) 118 118 + } 119 119 + spindle.stores = stores 120 120 + if cfg.LegacyS3.LogBucket != "" { 121 121 + logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") 122 122 + } 123 123 + if cfg.Role == config.RoleStandalone { 124 124 + spindle.reader = stores 125 125 + } else { 126 126 + name := cfg.Mill.ArtifactStore 127 127 + if name == "" { 128 128 + names := stores.Names() 129 129 + if len(names) != 1 { 130 130 + return nil, fmt.Errorf("%s requires SPINDLE_MILL_ARTIFACT_STORE when %d artifact stores are configured", cfg.Role, len(names)) 131 131 + } 132 132 + name = names[0] 133 133 + logger.Warn("SPINDLE_MILL_ARTIFACT_STORE is not set; inferred the only configured store", "store", name) 134 134 + } 135 135 + store, ok := stores.Store(name) 136 136 + if !ok { 137 137 + return nil, fmt.Errorf("SPINDLE_MILL_ARTIFACT_STORE=%q is not configured", name) 138 138 + } 139 139 + spindle.store = store 140 140 + spindle.reader = store 141 141 + } 105 142 if cfg.Role == config.RoleExecutor { 106 143 return spindle, nil 107 144 } ··· 137 174 return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) 138 175 } 139 176 140 140 - jq := queue.NewQueue(cfg.Server.QueueSize, cfg.Server.MaxJobCount) 141 141 - spindle.jq = jq 142 142 - logger.Info("initialized queue", "queueSize", cfg.Server.QueueSize, "numWorkers", cfg.Server.MaxJobCount) 177 177 + if cfg.Role == config.RoleStandalone { 178 178 + spindle.jq = queue.NewQueue(cfg.Server.QueueSize, cfg.Server.MaxJobCount) 179 179 + logger.Info("initialized queue", "queueSize", cfg.Server.QueueSize, "numWorkers", cfg.Server.MaxJobCount) 180 180 + } 143 181 144 182 collections := []string{ 145 183 tangled.SpindleMemberNSID, ··· 223 261 ccfg.Sources[src] = struct{}{} 224 262 } 225 263 spindle.ks = eventconsumer.NewConsumer(*ccfg) 226 226 - 227 264 if cfg.Server.Tap.Embed { 228 265 pw, err := randomAdminPassword() 229 266 if err != nil { ··· 236 273 237 274 return spindle, nil 238 275 } 239 239 - 240 240 - // DB returns the database instance. 241 276 func (s *Spindle) DB() *db.DB { 242 277 return s.db 243 278 } 244 244 - 245 245 - // Queue returns the job queue instance. 246 279 func (s *Spindle) Queue() *queue.Queue { 247 280 return s.jq 248 281 } ··· 286 319 // only standalone runs the local queue. mill hosts place directly onto 287 320 // executors, and executors only run jobs explicitly assigned by a mill 288 321 if s.cfg.Role == config.RoleStandalone { 289 289 - s.jq.Start() 290 290 - defer s.jq.Stop() 322 322 + if s.jq != nil { 323 323 + s.jq.Start() 324 324 + defer s.jq.Stop() 325 325 + } 291 326 } 292 327 293 328 // an executor dials out to its mill and takes work from it ··· 381 416 // mill host: register engines that place jobs on executors instead of 382 417 // running them. all names share one Mill 383 418 m = mill.New(log.SubLogger(logger, "mill"), mill.Config{ 419 419 + LogDir: cfg.Server.LogDir, 384 420 MaxPending: cfg.Mill.MaxPending, 385 421 ReconnectGrace: cfg.Mill.ReconnectGrace, 386 386 - LogDir: cfg.Server.LogDir, 387 422 }) 388 423 engines = map[string]models.Engine{ 389 424 "nixery": mill.NewEngine("nixery", m), ··· 422 457 } 423 458 } 424 459 if cfg.Role == config.RoleExecutor { 425 425 - s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor")) 460 460 + s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store) 426 461 if err != nil { 427 462 return err 428 463 } ··· 459 494 l := log.SubLogger(s.l, "xrpc") 460 495 461 496 x := xrpc.Xrpc{ 462 462 - Logger: l, 463 463 - Db: s.db, 464 464 - Enforcer: s.e, 465 465 - Engines: s.engs, 466 466 - Config: s.cfg, 467 467 - Resolver: s.res, 468 468 - Vault: s.vault, 469 469 - Notifier: s.Notifier(), 470 470 - ServiceAuth: serviceAuth, 471 471 - Trigger: s, 497 497 + Logger: l, 498 498 + Db: s.db, 499 499 + Enforcer: s.e, 500 500 + Engines: s.engs, 501 501 + Config: s.cfg, 502 502 + ArtifactReader: s.reader, 503 503 + Resolver: s.res, 504 504 + Vault: s.vault, 505 505 + Notifier: s.Notifier(), 506 506 + ServiceAuth: serviceAuth, 507 507 + Trigger: s, 472 508 } 473 509 474 510 return x.Router() ··· 897 933 // (AcquireWorkflowSlot) which the user sees as pending. the only bound 898 934 // is the mill's maxPending. rootCtx is the long-lived consumer context, 899 935 // so the goroutine safely outlives this call 900 900 - go engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, pipeline, pipelineId) 936 936 + go engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.rootCtx, pipeline, pipelineId) 901 937 s.l.Info("pipeline handed to mill placement", "id", pipelineId) 902 902 - } else { 938 938 + } else if s.jq != nil { 903 939 ok := s.jq.Enqueue(repoDid, queue.Job{ 904 940 Run: func() error { 905 905 - engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, pipeline, pipelineId) 941 941 + engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.rootCtx, pipeline, pipelineId) 906 942 return nil 907 943 }, 908 944 OnFail: func(jobError error) { ··· 913 949 return fmt.Errorf("failed to enqueue pipeline: queue is full") 914 950 } 915 951 s.l.Info("pipeline enqueued successfully", "id", pipelineId) 952 952 + } else { 953 953 + return fmt.Errorf("no queue or mill available to process pipeline") 916 954 } 917 955 918 956 // after successful enqueue, emit StatusPending for all workflows

+2

spindle/server_test.go

Reviewed

··· 73 73 cfg.Server.DBPath = dbPath 74 74 cfg.Server.Hostname = "executor.test" 75 75 cfg.Server.Tap.Embed = true 76 76 + cfg.ArtifactStores.Disk.Dir = t.TempDir() 77 77 + cfg.Mill.ArtifactStore = "disk" 76 78 77 79 s, err := New(ctx, cfg, d, map[string]models.Engine{}) 78 80 if err != nil {

+1 -3

spindle/stream.go

Reviewed

··· 9 9 10 10 "tangled.org/core/eventstream" 11 11 "tangled.org/core/log" 12 12 - "tangled.org/core/spindle/config" 13 12 "tangled.org/core/spindle/logview" 14 13 "tangled.org/core/spindle/models" 15 14 ··· 89 88 } 90 89 isFinished := models.StatusKind(status.Status).IsFinish() 91 90 92 92 - lines, stop, err := logview.Follow(ctx, s.db, s.cfg.Server.LogDir, wid, s.cfg.Role == config.RoleMill, isFinished) 91 91 + lines, stop, err := logview.Follow(ctx, s.db, s.reader, s.cfg.Server.LogDir, wid, isFinished) 93 92 if err != nil { 94 93 return fmt.Errorf("failed to follow workflow log: %w", err) 95 94 } 96 95 defer stop() 97 97 - 98 96 for { 99 97 select { 100 98 case <-ctx.Done():

+1 -2

spindle/xrpc/ci_pipeline_subscribe_logs.go

Reviewed

··· 12 12 "github.com/bluesky-social/indigo/atproto/syntax" 13 13 "github.com/gorilla/websocket" 14 14 "tangled.org/core/api/tangled" 15 15 - "tangled.org/core/spindle/config" 16 15 "tangled.org/core/spindle/logview" 17 16 "tangled.org/core/spindle/models" 18 17 ) ··· 166 165 isFinished = models.StatusKind(status.Status).IsFinish() 167 166 } 168 167 169 169 - lines, stop, err := logview.Follow(ctx, x.Db, x.Config.Server.LogDir, wid, x.Config.Role == config.RoleMill, isFinished) 168 168 + lines, stop, err := logview.Follow(ctx, x.Db, x.ArtifactReader, x.Config.Server.LogDir, wid, isFinished) 170 169 if err != nil { 171 170 l.Error("failed to follow workflow log", "workflow", wfName, "err", err) 172 171 return

workflow/compile.go

Reviewed

This file has not been changed.

workflow/compile_test.go

Reviewed

This file has not been changed.

workflow/def.go

Reviewed

This file has not been changed.

workflow/def_test.go

Reviewed

This file has not been changed.

+1 -1

appview/serververify/verify.go

Reviewed

··· 10 10 indigoxrpc "github.com/bluesky-social/indigo/xrpc" 11 11 "tangled.org/core/api/tangled" 12 12 "tangled.org/core/appview/db" 13 13 + "tangled.org/core/netutil" 13 14 "tangled.org/core/orm" 14 15 "tangled.org/core/rbac" 15 15 - "tangled.org/core/netutil" 16 16 "tangled.org/core/xrpc/xrpcclient" 17 17 ) 18 18

-1

docker-compose.yml

Reviewed

··· 181 181 SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images 182 182 SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays 183 183 SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11240" 184 184 - SPINDLE_S3_LOG_BUCKET: "" 185 184 SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" 186 185 # route guest nix substitution + uploads through the local ncps cache. 187 186 # ncps re-signs on serve with cache.local's key, so the guest trusts the

+22 -4

nix/modules/spindle.nix

Reviewed

··· 136 136 }; 137 137 }; 138 138 139 139 - pipelines = { 140 140 - logBucket = mkOption { 139 139 + artifactStores = { 140 140 + disk.dir = mkOption { 141 141 + type = types.path; 142 142 + default = "/var/log/spindle"; 143 143 + description = "Root directory for disk artifacts"; 144 144 + }; 145 145 + 146 146 + s3.bucket = mkOption { 141 147 type = types.str; 142 148 default = "tangled-logs"; 143 143 - description = "S3 bucket for workflow logs"; 149 149 + description = "S3 bucket for artifacts"; 150 150 + }; 151 151 + 152 152 + s3.region = mkOption { 153 153 + type = types.str; 154 154 + default = "us-east-1"; 155 155 + description = "AWS region for the artifact bucket"; 144 156 }; 157 157 + }; 158 158 + 159 159 + pipelines = { 145 160 workflowTimeout = mkOption { 146 161 type = types.str; 147 162 default = "5m"; ··· 388 403 "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.pipelines.nixCache.readUrls}" 389 404 "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.pipelines.nixCache.trustedPublicKeys}" 390 405 "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.pipelines.nixCache.uploadUrl}" 391 391 - "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}" 406 406 + "SPINDLE_ARTIFACT_STORES_DISK_DIR=${cfg.artifactStores.disk.dir}" 407 407 + "SPINDLE_ARTIFACT_STORES_S3_BUCKET=${cfg.artifactStores.s3.bucket}" 408 408 + "SPINDLE_ARTIFACT_STORES_S3_REGION=${cfg.artifactStores.s3.region}" 409 409 + "SPINDLE_MILL_ARTIFACT_STORE=s3" 392 410 ]; 393 411 ExecStart = "${cfg.package}/bin/spindle"; 394 412 Restart = "always";

+2 -1

nix/vm.nix

Reviewed

··· 162 162 }; 163 163 }; 164 164 165 165 + artifactStores.s3.bucket = envVarOr "SPINDLE_ARTIFACT_STORES_S3_BUCKET" "tangled-logs"; 166 166 + 165 167 pipelines = { 166 166 - logBucket = envVarOr "SPINDLE_S3_LOG_BUCKET" ""; 167 168 microvm.enableKVM = nestedVirt; 168 169 nixCache = { 169 170 readUrls = ["http://127.0.0.1:8501"];

+240

spindle/artifactstore/artifactstore.go

Reviewed

··· 1 1 + package artifactstore 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "errors" 6 6 + "fmt" 7 7 + "io" 8 8 + "os" 9 9 + "path/filepath" 10 10 + "strings" 11 11 + 12 12 + "github.com/aws/aws-sdk-go-v2/aws" 13 13 + "github.com/aws/aws-sdk-go-v2/config" 14 14 + "github.com/aws/aws-sdk-go-v2/service/s3" 15 15 + spindleconfig "tangled.org/core/spindle/config" 16 16 + ) 17 17 + 18 18 + type Writer interface { 19 19 + Put(ctx context.Context, ref string, r io.Reader) error 20 20 + } 21 21 + 22 22 + type Reader interface { 23 23 + Open(ctx context.Context, ref string) (io.ReadCloser, error) 24 24 + } 25 25 + 26 26 + type Store interface { 27 27 + Writer 28 28 + Reader 29 29 + } 30 30 + 31 31 + type DiskStore struct { 32 32 + root string 33 33 + } 34 34 + 35 35 + func NewDiskStore(root string) (*DiskStore, error) { 36 36 + if root == "" { 37 37 + return nil, fmt.Errorf("artifact disk directory is required") 38 38 + } 39 39 + return &DiskStore{root: filepath.Clean(root)}, nil 40 40 + } 41 41 + 42 42 + func (s *DiskStore) Put(_ context.Context, ref string, r io.Reader) error { 43 43 + path, err := s.resolve(ref) 44 44 + if err != nil { 45 45 + return err 46 46 + } 47 47 + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { 48 48 + return fmt.Errorf("mkdir for artifact %q: %w", path, err) 49 49 + } 50 50 + tmpFile, err := os.CreateTemp(filepath.Dir(path), ".tmp-artifact-*") 51 51 + if err != nil { 52 52 + return fmt.Errorf("create temp artifact: %w", err) 53 53 + } 54 54 + tmpPath := tmpFile.Name() 55 55 + defer func() { 56 56 + _ = tmpFile.Close() 57 57 + _ = os.Remove(tmpPath) 58 58 + }() 59 59 + if _, err := io.Copy(tmpFile, r); err != nil { 60 60 + return fmt.Errorf("write artifact content: %w", err) 61 61 + } 62 62 + if err := tmpFile.Sync(); err != nil { 63 63 + return fmt.Errorf("sync artifact file: %w", err) 64 64 + } 65 65 + if err := tmpFile.Close(); err != nil { 66 66 + return fmt.Errorf("close artifact file: %w", err) 67 67 + } 68 68 + if err := os.Rename(tmpPath, path); err != nil { 69 69 + return fmt.Errorf("rename artifact file to target: %w", err) 70 70 + } 71 71 + return nil 72 72 + } 73 73 + 74 74 + func (s *DiskStore) Open(_ context.Context, ref string) (io.ReadCloser, error) { 75 75 + path, err := s.resolve(ref) 76 76 + if err != nil { 77 77 + return nil, err 78 78 + } 79 79 + f, err := os.Open(path) 80 80 + if err != nil { 81 81 + return nil, fmt.Errorf("open disk artifact %q: %w", path, err) 82 82 + } 83 83 + return f, nil 84 84 + } 85 85 + 86 86 + func (s *DiskStore) resolve(ref string) (string, error) { 87 87 + if ref == "" || filepath.IsAbs(ref) { 88 88 + return "", fmt.Errorf("invalid disk artifact ref %q", ref) 89 89 + } 90 90 + path := filepath.Join(s.root, filepath.Clean(ref)) 91 91 + rel, err := filepath.Rel(s.root, path) 92 92 + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { 93 93 + return "", fmt.Errorf("artifact ref %q escapes disk root %q", ref, s.root) 94 94 + } 95 95 + return path, nil 96 96 + } 97 97 + 98 98 + type s3API interface { 99 99 + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) 100 100 + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) 101 101 + } 102 102 + 103 103 + type S3Store struct { 104 104 + client s3API 105 105 + bucket string 106 106 + } 107 107 + 108 108 + func NewS3Store(client s3API, bucket string) (*S3Store, error) { 109 109 + if client == nil { 110 110 + return nil, fmt.Errorf("s3 client is required") 111 111 + } 112 112 + if bucket == "" { 113 113 + return nil, fmt.Errorf("artifact S3 bucket is required") 114 114 + } 115 115 + return &S3Store{client: client, bucket: bucket}, nil 116 116 + } 117 117 + 118 118 + func (s *S3Store) Put(ctx context.Context, ref string, r io.Reader) error { 119 119 + if err := validateObjectRef(ref); err != nil { 120 120 + return err 121 121 + } 122 122 + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ 123 123 + Bucket: aws.String(s.bucket), 124 124 + Key: aws.String(ref), 125 125 + Body: r, 126 126 + }) 127 127 + if err != nil { 128 128 + return fmt.Errorf("s3 put object: %w", err) 129 129 + } 130 130 + return nil 131 131 + } 132 132 + 133 133 + func (s *S3Store) Open(ctx context.Context, ref string) (io.ReadCloser, error) { 134 134 + if err := validateObjectRef(ref); err != nil { 135 135 + return nil, err 136 136 + } 137 137 + res, err := s.client.GetObject(ctx, &s3.GetObjectInput{ 138 138 + Bucket: aws.String(s.bucket), 139 139 + Key: aws.String(ref), 140 140 + }) 141 141 + if err != nil { 142 142 + return nil, fmt.Errorf("s3 get object: %w", err) 143 143 + } 144 144 + return res.Body, nil 145 145 + } 146 146 + 147 147 + func validateObjectRef(ref string) error { 148 148 + if ref == "" || strings.HasPrefix(ref, "/") || strings.Contains(ref, "://") { 149 149 + return fmt.Errorf("invalid artifact ref %q", ref) 150 150 + } 151 151 + return nil 152 152 + } 153 153 + 154 154 + type Stores struct { 155 155 + order []string 156 156 + stores map[string]Store 157 157 + } 158 158 + 159 159 + func NewStores(cfg spindleconfig.ArtifactStores, diskFallback, legacyS3Bucket string) (*Stores, error) { 160 160 + stores := &Stores{stores: make(map[string]Store)} 161 161 + diskDir := cfg.Disk.Dir 162 162 + if diskDir == "" { 163 163 + diskDir = diskFallback 164 164 + } 165 165 + if diskDir != "" { 166 166 + disk, err := NewDiskStore(diskDir) 167 167 + if err != nil { 168 168 + return nil, err 169 169 + } 170 170 + stores.order = append(stores.order, "disk") 171 171 + stores.stores["disk"] = disk 172 172 + } 173 173 + 174 174 + bucket := cfg.S3.Bucket 175 175 + if bucket == "" { 176 176 + bucket = legacyS3Bucket 177 177 + } 178 178 + if bucket != "" { 179 179 + awsCfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(cfg.S3.Region)) 180 180 + if err != nil { 181 181 + return nil, fmt.Errorf("load aws config: %w", err) 182 182 + } 183 183 + s3Store, err := NewS3Store(s3.NewFromConfig(awsCfg), bucket) 184 184 + if err != nil { 185 185 + return nil, err 186 186 + } 187 187 + stores.order = append(stores.order, "s3") 188 188 + stores.stores["s3"] = s3Store 189 189 + } 190 190 + return stores, nil 191 191 + } 192 192 + 193 193 + func (s *Stores) Names() []string { 194 194 + return append([]string(nil), s.order...) 195 195 + } 196 196 + 197 197 + func (s *Stores) Store(name string) (Store, bool) { 198 198 + store, ok := s.stores[name] 199 199 + return store, ok 200 200 + } 201 201 + 202 202 + func (s *Stores) Open(ctx context.Context, ref string) (io.ReadCloser, error) { 203 203 + var errs []error 204 204 + for _, name := range s.order { 205 205 + rc, err := s.stores[name].Open(ctx, ref) 206 206 + if err == nil { 207 207 + return rc, nil 208 208 + } 209 209 + errs = append(errs, fmt.Errorf("%s: %w", name, err)) 210 210 + } 211 211 + return nil, fmt.Errorf("open artifact %q: %w", ref, errors.Join(errs...)) 212 212 + } 213 213 + 214 214 + func (s *Stores) PutFile(ctx context.Context, ref, sourcePath string) []error { 215 215 + var errs []error 216 216 + for _, name := range s.order { 217 217 + store := s.stores[name] 218 218 + if disk, ok := store.(*DiskStore); ok { 219 219 + target, err := disk.resolve(ref) 220 220 + if err == nil { 221 221 + source, sourceErr := filepath.Abs(sourcePath) 222 222 + targetAbs, targetErr := filepath.Abs(target) 223 223 + if sourceErr == nil && targetErr == nil && source == targetAbs { 224 224 + continue 225 225 + } 226 226 + } 227 227 + } 228 228 + file, err := os.Open(sourcePath) 229 229 + if err != nil { 230 230 + errs = append(errs, fmt.Errorf("%s: open source: %w", name, err)) 231 231 + continue 232 232 + } 233 233 + err = store.Put(ctx, ref, file) 234 234 + _ = file.Close() 235 235 + if err != nil { 236 236 + errs = append(errs, fmt.Errorf("%s: %w", name, err)) 237 237 + } 238 238 + } 239 239 + return errs 240 240 + }

+130

spindle/artifactstore/artifactstore_test.go

Reviewed

··· 1 1 + package artifactstore 2 2 + 3 3 + import ( 4 4 + "bytes" 5 5 + "context" 6 6 + "io" 7 7 + "os" 8 8 + "strings" 9 9 + "sync" 10 10 + "testing" 11 11 + 12 12 + "github.com/aws/aws-sdk-go-v2/service/s3" 13 13 + ) 14 14 + 15 15 + type mockS3Client struct { 16 16 + mu sync.Mutex 17 17 + store map[string][]byte 18 18 + } 19 19 + 20 20 + func newMockS3Client() *mockS3Client { 21 21 + return &mockS3Client{ 22 22 + store: make(map[string][]byte), 23 23 + } 24 24 + } 25 25 + 26 26 + func (m *mockS3Client) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { 27 27 + m.mu.Lock() 28 28 + defer m.mu.Unlock() 29 29 + b, err := io.ReadAll(params.Body) 30 30 + if err != nil { 31 31 + return nil, err 32 32 + } 33 33 + key := *params.Bucket + "/" + *params.Key 34 34 + m.store[key] = b 35 35 + return &s3.PutObjectOutput{}, nil 36 36 + } 37 37 + 38 38 + func (m *mockS3Client) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { 39 39 + m.mu.Lock() 40 40 + defer m.mu.Unlock() 41 41 + key := *params.Bucket + "/" + *params.Key 42 42 + data, ok := m.store[key] 43 43 + if !ok { 44 44 + return nil, os.ErrNotExist 45 45 + } 46 46 + return &s3.GetObjectOutput{ 47 47 + Body: io.NopCloser(bytes.NewReader(data)), 48 48 + }, nil 49 49 + } 50 50 + 51 51 + func TestDiskStore(t *testing.T) { 52 52 + tempDir := t.TempDir() 53 53 + store, err := NewDiskStore(tempDir) 54 54 + if err != nil { 55 55 + t.Fatal(err) 56 56 + } 57 57 + 58 58 + ctx := context.Background() 59 59 + ref := "logs/test.log" 60 60 + content := "hello world log content" 61 61 + 62 62 + if err := store.Put(ctx, ref, strings.NewReader(content)); err != nil { 63 63 + t.Fatalf("Put failed: %v", err) 64 64 + } 65 65 + 66 66 + rc, err := store.Open(ctx, ref) 67 67 + if err != nil { 68 68 + t.Fatalf("Open failed: %v", err) 69 69 + } 70 70 + defer rc.Close() 71 71 + 72 72 + got, err := io.ReadAll(rc) 73 73 + if err != nil { 74 74 + t.Fatalf("ReadAll failed: %v", err) 75 75 + } 76 76 + if string(got) != content { 77 77 + t.Fatalf("got content %q, want %q", string(got), content) 78 78 + } 79 79 + } 80 80 + 81 81 + func TestDiskStoreTraversalProtection(t *testing.T) { 82 82 + tempDir := t.TempDir() 83 83 + store, err := NewDiskStore(tempDir) 84 84 + if err != nil { 85 85 + t.Fatal(err) 86 86 + } 87 87 + 88 88 + ctx := context.Background() 89 89 + badRef := "../outside" 90 90 + 91 91 + err = store.Put(ctx, badRef, strings.NewReader("bad")) 92 92 + if err == nil { 93 93 + t.Fatal("expected error putting file outside diskDir, got nil") 94 94 + } 95 95 + 96 96 + _, err = store.Open(ctx, badRef) 97 97 + if err == nil { 98 98 + t.Fatal("expected error opening file outside diskDir, got nil") 99 99 + } 100 100 + } 101 101 + 102 102 + func TestS3Store(t *testing.T) { 103 103 + mock := newMockS3Client() 104 104 + store, err := NewS3Store(mock, "mybucket") 105 105 + if err != nil { 106 106 + t.Fatal(err) 107 107 + } 108 108 + 109 109 + ctx := context.Background() 110 110 + ref := "logs/run1.log" 111 111 + content := "s3 log payload" 112 112 + 113 113 + if err := store.Put(ctx, ref, strings.NewReader(content)); err != nil { 114 114 + t.Fatalf("Put to S3 failed: %v", err) 115 115 + } 116 116 + 117 117 + rc, err := store.Open(ctx, ref) 118 118 + if err != nil { 119 119 + t.Fatalf("Open from S3 failed: %v", err) 120 120 + } 121 121 + defer rc.Close() 122 122 + 123 123 + got, err := io.ReadAll(rc) 124 124 + if err != nil { 125 125 + t.Fatalf("ReadAll failed: %v", err) 126 126 + } 127 127 + if string(got) != content { 128 128 + t.Fatalf("got content %q, want %q", string(got), content) 129 129 + } 130 130 + }

-75

spindle/engine/s3.go

Reviewed

··· 1 1 - package engine 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "fmt" 6 6 - "io" 7 7 - "os" 8 8 - "path/filepath" 9 9 - 10 10 - "github.com/aws/aws-sdk-go-v2/aws" 11 11 - "github.com/aws/aws-sdk-go-v2/config" 12 12 - "github.com/aws/aws-sdk-go-v2/service/s3" 13 13 - ) 14 14 - 15 15 - type S3 struct { 16 16 - bucket string 17 17 - client *s3.Client 18 18 - } 19 19 - 20 20 - const BaseS3Path = "spindle/workflows" 21 21 - 22 22 - func NewS3(bucket string) (*S3, error) { 23 23 - if bucket == "" { 24 24 - return nil, fmt.Errorf("s3 bucket not provided") 25 25 - } 26 26 - 27 27 - ctx := context.Background() 28 28 - sdkConfig, err := config.LoadDefaultConfig(ctx) 29 29 - 30 30 - if err != nil { 31 31 - return nil, fmt.Errorf("error loading s3 config: %w", err) 32 32 - } 33 33 - s3Client := s3.NewFromConfig(sdkConfig) 34 34 - 35 35 - return &S3{ 36 36 - bucket: bucket, 37 37 - client: s3Client, 38 38 - }, nil 39 39 - } 40 40 - 41 41 - func (s *S3) WriteFile(ctx context.Context, path string) error { 42 42 - s3Key := fmt.Sprintf("%s/%s", BaseS3Path, filepath.Base(path)) 43 43 - 44 44 - file, err := os.Open(path) 45 45 - if err != nil { 46 46 - return fmt.Errorf("error opening file %s: %w", path, err) 47 47 - } 48 48 - defer file.Close() 49 49 - 50 50 - _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ 51 51 - Bucket: &s.bucket, 52 52 - Key: &s3Key, 53 53 - Body: file, 54 54 - }) 55 55 - 56 56 - if err != nil { 57 57 - return fmt.Errorf("error writing to s3: %w", err) 58 58 - } 59 59 - 60 60 - return nil 61 61 - } 62 62 - 63 63 - func (s *S3) ReadFile(ctx context.Context, name string) ([]byte, error) { 64 64 - res, err := s.client.GetObject(ctx, &s3.GetObjectInput{ 65 65 - Bucket: &s.bucket, 66 66 - Key: aws.String(name), 67 67 - }) 68 68 - 69 69 - if err != nil { 70 70 - return nil, fmt.Errorf("error reading file %s: %w", name, err) 71 71 - } 72 72 - defer res.Body.Close() 73 73 - 74 74 - return io.ReadAll(res.Body) 75 75 - }

+12 -10

spindle/xrpc/xrpc.go

Reviewed

··· 15 15 "tangled.org/core/idresolver" 16 16 "tangled.org/core/notifier" 17 17 "tangled.org/core/rbac" 18 18 + "tangled.org/core/spindle/artifactstore" 18 19 "tangled.org/core/spindle/config" 19 20 "tangled.org/core/spindle/db" 20 21 "tangled.org/core/spindle/models" ··· 41 42 } 42 43 43 44 type Xrpc struct { 44 44 - Logger *slog.Logger 45 45 - Db *db.DB 46 46 - Enforcer *rbac.Enforcer 47 47 - Engines map[string]models.Engine 48 48 - Config *config.Config 49 49 - Resolver *idresolver.Resolver 50 50 - Vault secrets.Manager 51 51 - Notifier *notifier.Notifier 52 52 - ServiceAuth *serviceauth.ServiceAuth 53 53 - Trigger PipelineTrigger 45 45 + Logger *slog.Logger 46 46 + Db *db.DB 47 47 + Enforcer *rbac.Enforcer 48 48 + Engines map[string]models.Engine 49 49 + Config *config.Config 50 50 + ArtifactReader artifactstore.Reader 51 51 + Resolver *idresolver.Resolver 52 52 + Vault secrets.Manager 53 53 + Notifier *notifier.Notifier 54 54 + ServiceAuth *serviceauth.ServiceAuth 55 55 + Trigger PipelineTrigger 54 56 } 55 57 56 58 func (x *Xrpc) Router() http.Handler {