spindle: add artifact store #3

api/tangled/cbor_gen.go

Reviewed

This file has not been changed.

api/tangled/tangledpipeline.go

Reviewed

This file has not been changed.

buf.gen.yaml

Reviewed

This file has not been changed.

buf.yaml

Reviewed

This file has not been changed.

cmd/spindle/main.go

Reviewed

This file has not been changed.

docker-compose.mill.yml

Reviewed

This file has not been changed.

docker-compose.yml

Reviewed

This file has not been changed.

eventstream/eventstream_test.go

Reviewed

This file has not been changed.

eventstream/store.go

Reviewed

This file has not been changed.

lexicons/pipeline/pipeline.json

Reviewed

This file has not been changed.

localinfra/readme.md

Reviewed

This file has not been changed.

localinfra/scripts/prepare-spindle-images.sh

Reviewed

This file has not been changed.

localinfra/spindle.Dockerfile

Reviewed

This file has not been changed.

netutil/ssrf.go

Reviewed

This file has not been changed.

nix/modules/spindle.nix

Reviewed

This file has not been changed.

nix/vm.nix

Reviewed

This file has not been changed.

spindle/artifactstore/artifactstore.go

Reviewed

This file has not been changed.

spindle/artifactstore/artifactstore_test.go

Reviewed

This file has not been changed.

spindle/config/config.go

Reviewed

This file has not been changed.

spindle/db/db.go

Reviewed

This file has not been changed.

spindle/db/events.go

Reviewed

This file has not been changed.

spindle/db/mill_state.go

Reviewed

This file has not been changed.

spindle/db/mill_state_test.go

Reviewed

This file has not been changed.

spindle/db/mill_tokens.go

Reviewed

This file has not been changed.

spindle/db/mill_tokens_test.go

Reviewed

This file has not been changed.

spindle/engine/engine.go

Reviewed

This file has not been changed.

spindle/engine/engine_test.go

Reviewed

This file has not been changed.

spindle/engine/manifest_test.go

Reviewed

This file has not been changed.

spindle/engine/placement.go

Reviewed

This file has not been changed.

spindle/engine/s3.go

Reviewed

This file has not been changed.

spindle/engine/scheduler.go

Reviewed

This file has not been changed.

spindle/engine/scheduler_test.go

Reviewed

This file has not been changed.

spindle/engine/slot.go

Reviewed

This file has not been changed.

spindle/engine/slot_test.go

Reviewed

This file has not been changed.

+10 -2

spindle/engines/dummy/engine.go

Reviewed

··· 17 17 // step output to the workflow logger. Useful for testing pipeline plumbing 18 18 // without a real execution backend. 19 19 type DummyEngine struct { 20 20 - l *slog.Logger 20 20 + l *slog.Logger 21 21 + StepDelay time.Duration 21 22 } 22 23 23 24 func New(l *slog.Logger) *DummyEngine { ··· 90 91 return nil 91 92 } 92 93 93 93 - func (e *DummyEngine) RunStep(_ context.Context, wid models.WorkflowId, w *models.Workflow, idx int, _ []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { 94 94 + func (e *DummyEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, _ []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { 94 95 step := w.Steps[idx] 95 96 e.l.Info("running step", "wid", wid, "step", step.Name(), "command", step.Command()) 96 97 fmt.Fprintf(wfLogger.DataWriter(idx, "stdout"), "$ %s", step.Command()) 98 98 + if e.StepDelay > 0 { 99 99 + select { 100 100 + case <-ctx.Done(): 101 101 + return ctx.Err() 102 102 + case <-time.After(e.StepDelay): 103 103 + } 104 104 + } 97 105 return nil 98 106 }

spindle/engines/microvm/budget.go

Reviewed

This file has not been changed.

spindle/engines/microvm/engine.go

Reviewed

This file has not been changed.

+3

spindle/engines/microvm/image.go

Reviewed

··· 209 209 210 210 // check if image name is not a path 211 211 func isPlainImageName(name string) bool { 212 212 + if name == "" || name == "." || name == ".." { 213 213 + return false 214 214 + } 212 215 if filepath.IsAbs(name) || strings.ContainsRune(name, '/') || strings.ContainsRune(name, filepath.Separator) { 213 216 return false 214 217 }

spindle/engines/microvm/image_test.go

Reviewed

This file has not been changed.

spindle/engines/microvm/placement/placement.go

Reviewed

This file has not been changed.

spindle/engines/microvm/placement/placement_test.go

Reviewed

This file has not been changed.

spindle/engines/microvm/placement_linux.go

Reviewed

This file has not been changed.

spindle/engines/microvm/qemu.go

Reviewed

This file has not been changed.

spindle/engines/microvm/vm.go

Reviewed

This file has not been changed.

spindle/engines/nixery/engine.go

Reviewed

This file has not been changed.

spindle/logview/logview.go

Reviewed

This file has not been changed.

spindle/mill/README.md

Reviewed

This file has not been changed.

spindle/mill/auth_test.go

Reviewed

This file has not been changed.

spindle/mill/engine.go

Reviewed

This file has not been changed.

spindle/mill/executor/capability_test.go

Reviewed

This file has not been changed.

spindle/mill/executor/executor.go

Reviewed

This file has not been changed.

+1 -1

spindle/mill/executor/observe.go

Reviewed

··· 218 218 var once sync.Once 219 219 res.stopTail = func() { 220 220 once.Do(func() { 221 221 - _ = t.Stop() 221 221 + _ = t.StopAtEOF() 222 222 <-done 223 223 }) 224 224 }

spindle/mill/executor/outbox.go

Reviewed

This file has not been changed.

spindle/mill/executor/reserved.go

Reviewed

This file has not been changed.

spindle/mill/executor/reserved_test.go

Reviewed

This file has not been changed.

spindle/mill/handler.go

Reviewed

This file has not been changed.

+13 -7

spindle/mill/integration_test.go

Reviewed

··· 58 58 cfg.Mill.Seats = 2 59 59 cfg.Mill.SharedSecret = "test-token" 60 60 61 61 - engines := map[string]models.Engine{"dummy": dummy.New(l)} 61 61 + dummyEng := dummy.New(l) 62 62 + dummyEng.StepDelay = 50 * time.Millisecond 63 63 + engines := map[string]models.Engine{"dummy": dummyEng} 62 64 exec, err := executor.New(cfg, engines, edb, &en, l) 63 65 if err != nil { 64 66 t.Fatalf("executor.New: %v", err) ··· 91 93 92 94 sawLogContent := make(chan bool, 1) 93 95 go func() { 96 96 + ticker := time.NewTicker(20 * time.Millisecond) 97 97 + defer ticker.Stop() 98 98 + timeout := time.After(10 * time.Second) 94 99 for { 95 100 select { 96 101 case <-ch: 97 97 - data, err := os.ReadFile(logPath) 98 98 - if err == nil && strings.Contains(string(data), "echo hi") { 99 99 - sawLogContent <- true 100 100 - return 101 101 - } 102 102 - case <-time.After(5 * time.Second): 102 102 + case <-ticker.C: 103 103 + case <-timeout: 103 104 sawLogContent <- false 105 105 + return 106 106 + } 107 107 + data, err := os.ReadFile(logPath) 108 108 + if err == nil && strings.Contains(string(data), "echo hi") { 109 109 + sawLogContent <- true 104 110 return 105 111 } 106 112 }

spindle/mill/lease.go

Reviewed

This file has not been changed.

+16 -4

spindle/mill/mill.go

Reviewed

··· 1019 1019 if lease == nil || lease.nodeID != sess.nodeID { 1020 1020 return nil 1021 1021 } 1022 1022 - if m.n != nil { 1023 1023 - m.n.NotifyAll() 1024 1024 - } 1025 1025 - 1026 1022 raw := ll.GetRawJson() 1027 1023 if m.cfg.LogDir == "" || len(raw) == 0 { 1024 1024 + if m.n != nil { 1025 1025 + m.n.NotifyAll() 1026 1026 + } 1028 1027 return nil 1029 1028 } 1030 1029 lease.mu.Lock() 1031 1030 isDone := (lease.state == leaseDone) 1032 1031 lease.mu.Unlock() 1033 1032 if isDone { 1033 1033 + if m.n != nil { 1034 1034 + m.n.NotifyAll() 1035 1035 + } 1034 1036 return nil 1035 1037 } 1036 1038 1037 1039 logPath := models.LogFilePath(m.cfg.LogDir, lease.wid) 1038 1040 if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { 1039 1041 m.l.Warn("failed to create log dir", "path", filepath.Dir(logPath), "err", err) 1042 1042 + if m.n != nil { 1043 1043 + m.n.NotifyAll() 1044 1044 + } 1040 1045 return nil 1041 1046 } 1042 1047 f, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) 1043 1048 if err != nil { 1044 1049 m.l.Warn("failed to open log file", "path", logPath, "err", err) 1050 1050 + if m.n != nil { 1051 1051 + m.n.NotifyAll() 1052 1052 + } 1045 1053 return nil 1046 1054 } 1047 1055 if _, err := f.Write(raw); err != nil { 1048 1056 m.l.Warn("failed to write log file", "path", logPath, "err", err) 1049 1057 } 1050 1058 _ = f.Close() 1059 1059 + 1060 1060 + if m.n != nil { 1061 1061 + m.n.NotifyAll() 1062 1062 + } 1051 1063 return nil 1052 1064 } 1053 1065

spindle/mill/mill_test.go

Reviewed

This file has not been changed.

spindle/mill/proto/gen/mill.pb.go

Reviewed

This file has not been changed.

spindle/mill/proto/protocol.go

Reviewed

This file has not been changed.

spindle/mill/proto/protocol_test.go

Reviewed

This file has not been changed.

spindle/mill/proto/spindle/mill/v1/mill.proto

Reviewed

This file has not been changed.

spindle/mill/proto/ws.go

Reviewed

This file has not been changed.

spindle/mill/restore.go

Reviewed

This file has not been changed.

spindle/mill/restore_test.go

Reviewed

This file has not been changed.

spindle/mill/session.go

Reviewed

This file has not been changed.

spindle/mill/token.go

Reviewed

This file has not been changed.

spindle/server.go

Reviewed

This file has not been changed.

spindle/server.go.master

Reviewed

This file has not been changed.

spindle/server.go.mill

Reviewed

This file has not been changed.

spindle/server_test.go

Reviewed

This file has not been changed.

spindle/stream.go

Reviewed

This file has not been changed.

spindle/xrpc/ci_pipeline_subscribe_logs.go

Reviewed

This file has not been changed.

spindle/xrpc/xrpc.go

Reviewed

This file has not been changed.

workflow/compile.go

Reviewed

This file has not been changed.

workflow/compile_test.go

Reviewed

This file has not been changed.

workflow/def.go

Reviewed

This file has not been changed.

workflow/def_test.go

Reviewed

This file has not been changed.