spindle: add artifact store #3

+82 -1

api/tangled/cbor_gen.go

Reviewed

··· 9749 9749 } 9750 9750 9751 9751 cw := cbg.NewCborWriter(w) 9752 9752 + fieldCount := 5 9752 9753 9753 9753 - if _, err := cw.Write([]byte{164}); err != nil { 9754 9754 + if t.RunsOn == nil { 9755 9755 + fieldCount-- 9756 9756 + } 9757 9757 + 9758 9758 + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { 9754 9759 return err 9755 9760 } 9756 9761 ··· 9838 9843 if _, err := cw.WriteString(string(t.Engine)); err != nil { 9839 9844 return err 9840 9845 } 9846 9846 + 9847 9847 + // t.RunsOn ([]string) (slice) 9848 9848 + if t.RunsOn != nil { 9849 9849 + 9850 9850 + if len("runsOn") > 1000000 { 9851 9851 + return xerrors.Errorf("Value in field \"runsOn\" was too long") 9852 9852 + } 9853 9853 + 9854 9854 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("runsOn"))); err != nil { 9855 9855 + return err 9856 9856 + } 9857 9857 + if _, err := cw.WriteString(string("runsOn")); err != nil { 9858 9858 + return err 9859 9859 + } 9860 9860 + 9861 9861 + if len(t.RunsOn) > 8192 { 9862 9862 + return xerrors.Errorf("Slice value in field t.RunsOn was too long") 9863 9863 + } 9864 9864 + 9865 9865 + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.RunsOn))); err != nil { 9866 9866 + return err 9867 9867 + } 9868 9868 + for _, v := range t.RunsOn { 9869 9869 + if len(v) > 1000000 { 9870 9870 + return xerrors.Errorf("Value in field v was too long") 9871 9871 + } 9872 9872 + 9873 9873 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { 9874 9874 + return err 9875 9875 + } 9876 9876 + if _, err := cw.WriteString(string(v)); err != nil { 9877 9877 + return err 9878 9878 + } 9879 9879 + 9880 9880 + } 9881 9881 + } 9841 9882 return nil 9842 9883 } 9843 9884 ··· 9935 9976 9936 9977 t.Engine = string(sval) 9937 9978 } 9979 9979 + // t.RunsOn ([]string) (slice) 9980 9980 + case "runsOn": 9981 9981 + 9982 9982 + maj, extra, err = cr.ReadHeader() 9983 9983 + if err != nil { 9984 9984 + return err 9985 9985 + } 9986 9986 + 9987 9987 + if extra > 8192 { 9988 9988 + return fmt.Errorf("t.RunsOn: array too large (%d)", extra) 9989 9989 + } 9990 9990 + 9991 9991 + if maj != cbg.MajArray { 9992 9992 + return fmt.Errorf("expected cbor array") 9993 9993 + } 9994 9994 + 9995 9995 + if extra > 0 { 9996 9996 + t.RunsOn = make([]string, extra) 9997 9997 + } 9998 9998 + 9999 9999 + for i := 0; i < int(extra); i++ { 10000 10000 + { 10001 10001 + var maj byte 10002 10002 + var extra uint64 10003 10003 + var err error 10004 10004 + _ = maj 10005 10005 + _ = extra 10006 10006 + _ = err 10007 10007 + 10008 10008 + { 10009 10009 + sval, err := cbg.ReadStringWithMax(cr, 1000000) 10010 10010 + if err != nil { 10011 10011 + return err 10012 10012 + } 10013 10013 + 10014 10014 + t.RunsOn[i] = string(sval) 10015 10015 + } 10016 10016 + 10017 10017 + } 10018 10018 + } 9938 10019 9939 10020 default: 9940 10021 // Field doesn't exist on this type, so ignore it

+1

api/tangled/tangledpipeline.go

Reviewed

··· 90 90 Engine string `json:"engine" cborgen:"engine"` 91 91 Name string `json:"name" cborgen:"name"` 92 92 Raw string `json:"raw" cborgen:"raw"` 93 93 + RunsOn []string `json:"runsOn,omitempty" cborgen:"runsOn,omitempty"` 93 94 }

+4

buf.gen.yaml

Reviewed

··· 9 9 out: shuttle/src/gen 10 10 opt: 11 11 - bytes=. 12 12 + # the fleet protocol is broker<->executor only (both Go); shuttle (Rust) 13 13 + # only speaks the agent protocol, so keep fleet types out of its gen tree. 14 14 + exclude_types: 15 15 + - spindle.mill.v1

+1

buf.yaml

Reviewed

··· 1 1 version: v2 2 2 modules: 3 3 - path: spindle/agentproto 4 4 + - path: spindle/mill/proto 4 5 deps: 5 6 - buf.build/bufbuild/protovalidate

+160

cmd/spindle/main.go

Reviewed

··· 2 2 3 3 import ( 4 4 "context" 5 5 + "fmt" 5 6 "log/slog" 6 7 "os" 8 8 + "strings" 9 9 + "text/tabwriter" 10 10 + "time" 7 11 8 12 "github.com/urfave/cli/v3" 9 13 tlog "tangled.org/core/log" 10 14 "tangled.org/core/spindle" 15 15 + "tangled.org/core/spindle/db" 16 16 + "tangled.org/core/spindle/mill" 11 17 ) 12 18 13 19 func main() { ··· 16 22 Usage: "spindle continuous integration runner", 17 23 Commands: []*cli.Command{ 18 24 Command(), 25 25 + millCommand(), 19 26 }, 20 27 DefaultCommand: "run", 21 28 } ··· 41 48 }, 42 49 } 43 50 } 51 51 + 52 52 + // for mill host administration and executor management 53 53 + func millCommand() *cli.Command { 54 54 + dbFlag := &cli.StringFlag{ 55 55 + Name: "db", 56 56 + Usage: "path to the spindle sqlite db", 57 57 + Value: "spindle.db", 58 58 + Sources: cli.EnvVars("SPINDLE_SERVER_DB_PATH"), 59 59 + } 60 60 + openDB := func(ctx context.Context, cmd *cli.Command) (*db.DB, error) { 61 61 + return db.Make(ctx, cmd.String("db")) 62 62 + } 63 63 + return &cli.Command{ 64 64 + Name: "mill", 65 65 + Usage: "mill host administration", 66 66 + Commands: []*cli.Command{ 67 67 + { 68 68 + Name: "executor", 69 69 + Usage: "manage executors allowed to join this mill", 70 70 + Commands: []*cli.Command{ 71 71 + { 72 72 + Name: "add", 73 73 + Usage: "register an executor and print its token", 74 74 + ArgsUsage: "<name>", 75 75 + Flags: []cli.Flag{ 76 76 + dbFlag, 77 77 + &cli.DurationFlag{ 78 78 + Name: "ttl", 79 79 + Usage: "token lifetime (e.g. 720h); omit for no expiry", 80 80 + }, 81 81 + &cli.StringSliceFlag{ 82 82 + Name: "label", 83 83 + Usage: "authorized labels for this executor", 84 84 + }, 85 85 + }, 86 86 + Action: func(ctx context.Context, cmd *cli.Command) error { 87 87 + name := cmd.Args().First() 88 88 + if name == "" { 89 89 + return fmt.Errorf("usage: spindle mill executor add <name>") 90 90 + } 91 91 + d, err := openDB(ctx, cmd) 92 92 + if err != nil { 93 93 + return err 94 94 + } 95 95 + token, err := mill.GenerateToken() 96 96 + if err != nil { 97 97 + return err 98 98 + } 99 99 + var expiresAt *time.Time 100 100 + if ttl := cmd.Duration("ttl"); ttl > 0 { 101 101 + exp := time.Now().UTC().Add(ttl) 102 102 + expiresAt = &exp 103 103 + } 104 104 + labels := cmd.StringSlice("label") 105 105 + if err := d.AddExecutorToken(name, mill.HashToken(token), expiresAt, labels); err != nil { 106 106 + return fmt.Errorf("registering executor %q: %w", name, err) 107 107 + } 108 108 + fmt.Println(token) 109 109 + return nil 110 110 + }, 111 111 + }, 112 112 + { 113 113 + Name: "list", 114 114 + Usage: "list registered executors", 115 115 + Flags: []cli.Flag{dbFlag}, 116 116 + Action: func(ctx context.Context, cmd *cli.Command) error { 117 117 + d, err := openDB(ctx, cmd) 118 118 + if err != nil { 119 119 + return err 120 120 + } 121 121 + tokens, err := d.ListExecutorTokens() 122 122 + if err != nil { 123 123 + return err 124 124 + } 125 125 + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 126 126 + fmt.Fprintln(w, "NAME\tCREATED\tEXPIRES\tLABELS\tQUARANTINE") 127 127 + for _, t := range tokens { 128 128 + expires := "never" 129 129 + if t.ExpiresAt != nil { 130 130 + expires = t.ExpiresAt.Format(time.RFC3339) 131 131 + if time.Now().After(*t.ExpiresAt) { 132 132 + expires += " (expired)" 133 133 + } 134 134 + } 135 135 + labels := strings.Join(t.Labels, ",") 136 136 + if labels == "" { 137 137 + labels = "-" 138 138 + } 139 139 + quarantine := "-" 140 140 + if t.QuarantineReason != nil { 141 141 + quarantine = *t.QuarantineReason 142 142 + if t.QuarantinedAt != nil { 143 143 + quarantine = *t.QuarantinedAt + ": " + quarantine 144 144 + } 145 145 + } 146 146 + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", t.Name, t.CreatedAt, expires, labels, quarantine) 147 147 + } 148 148 + return w.Flush() 149 149 + }, 150 150 + }, 151 151 + { 152 152 + Name: "unquarantine", 153 153 + Usage: "allow a quarantined executor to reconnect", 154 154 + ArgsUsage: "<name>", 155 155 + Flags: []cli.Flag{dbFlag}, 156 156 + Action: func(ctx context.Context, cmd *cli.Command) error { 157 157 + name := cmd.Args().First() 158 158 + if name == "" { 159 159 + return fmt.Errorf("usage: spindle mill executor unquarantine <name>") 160 160 + } 161 161 + d, err := openDB(ctx, cmd) 162 162 + if err != nil { 163 163 + return err 164 164 + } 165 165 + ok, err := d.ClearExecutorQuarantine(name) 166 166 + if err != nil { 167 167 + return err 168 168 + } 169 169 + if !ok { 170 170 + return fmt.Errorf("no such executor identity %q", name) 171 171 + } 172 172 + return nil 173 173 + }, 174 174 + }, 175 175 + { 176 176 + Name: "revoke", 177 177 + Usage: "revoke an executor's token", 178 178 + ArgsUsage: "<name>", 179 179 + Flags: []cli.Flag{dbFlag}, 180 180 + Action: func(ctx context.Context, cmd *cli.Command) error { 181 181 + name := cmd.Args().First() 182 182 + if name == "" { 183 183 + return fmt.Errorf("usage: spindle mill executor revoke <name>") 184 184 + } 185 185 + d, err := openDB(ctx, cmd) 186 186 + if err != nil { 187 187 + return err 188 188 + } 189 189 + ok, err := d.RevokeExecutorToken(name) 190 190 + if err != nil { 191 191 + return err 192 192 + } 193 193 + if !ok { 194 194 + return fmt.Errorf("no such executor identity %q", name) 195 195 + } 196 196 + return nil 197 197 + }, 198 198 + }, 199 199 + }, 200 200 + }, 201 201 + }, 202 202 + } 203 203 + }

+157

docker-compose.mill.yml

Reviewed

··· 1 1 + # turns the primary spindle into a mill host and runs a three-executor fleet 2 2 + # against it. the executors differ on two axes so placement is testable: 3 3 + # 4 4 + # executor labels images can run 5 5 + # --------- ------------- ----------- ------------------------ 6 6 + # executor-a linux, fast full image/alpine, image/nixos 7 7 + # executor-b linux, slow full image/alpine, image/nixos 8 8 + # executor-c linux, gpu alpine only image/alpine 9 9 + 10 10 + x-mill-executor: &mill-executor 11 11 + profiles: ["linux"] 12 12 + build: 13 13 + context: . 14 14 + dockerfile: localinfra/spindle.Dockerfile 15 15 + restart: unless-stopped 16 16 + environment: &mill-executor-env 17 17 + SPINDLE_ROLE: executor 18 18 + SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555 19 19 + SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db 20 20 + SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev 21 21 + SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe 22 22 + SPINDLE_SERVER_DEV: "true" 23 23 + SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev 24 24 + SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db 25 25 + SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev 26 26 + SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images 27 27 + SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays 28 28 + SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts 29 29 + SPINDLE_MILL_ARTIFACT_STORE: disk 30 30 + SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" 31 31 + SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 32 32 + SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= 33 33 + SPINDLE_NIX_CACHE_UPLOAD_URL: http://ncps:8501/upload 34 34 + # dials the mill container directly using ws 35 35 + SPINDLE_MILL_URL: ws://spindle:6555/mill 36 36 + devices: 37 37 + - /dev/vsock:/dev/vsock 38 38 + - /dev/kvm:/dev/kvm 39 39 + - /dev/vhost-vsock:/dev/vhost-vsock 40 40 + - /dev/net/tun:/dev/net/tun 41 41 + cap_add: 42 42 + - NET_ADMIN 43 43 + - SYS_ADMIN 44 44 + security_opt: 45 45 + - label=disable 46 46 + - seccomp=unconfined 47 47 + healthcheck: 48 48 + test: ["CMD", "wget", "-qO-", "http://localhost:6555/"] 49 49 + interval: 2s 50 50 + timeout: 2s 51 51 + retries: 30 52 52 + start_period: 5s 53 53 + depends_on: &mill-executor-deps 54 54 + plc: 55 55 + condition: service_started 56 56 + jetstream: 57 57 + condition: service_started 58 58 + init-accounts: 59 59 + condition: service_completed_successfully 60 60 + ncps: 61 61 + condition: service_started 62 62 + spindle: 63 63 + condition: service_healthy 64 64 + mill-tokens: 65 65 + condition: service_completed_successfully 66 66 + networks: [tngl] 67 67 + 68 68 + services: 69 69 + # configures the primary spindle container as a mill host 70 70 + spindle: 71 71 + environment: 72 72 + SPINDLE_ROLE: mill 73 73 + SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts 74 74 + SPINDLE_MILL_ARTIFACT_STORE: disk 75 75 + volumes: 76 76 + - spindle-artifacts:/var/lib/spindle/artifacts 77 77 + 78 78 + mill-tokens: 79 79 + profiles: ["linux"] 80 80 + build: 81 81 + context: . 82 82 + dockerfile: localinfra/spindle.Dockerfile 83 83 + restart: "no" 84 84 + entrypoint: ["/bin/sh", "-c"] 85 85 + environment: 86 86 + SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db 87 87 + command: 88 88 + - | 89 89 + set -eu 90 90 + seed() { 91 91 + spindle mill executor revoke "$$1" >/dev/null 2>&1 || true 92 92 + args="" 93 93 + for l in $$(echo "$$2" | tr ',' ' '); do args="$$args --label $$l"; done 94 94 + spindle mill executor add "$$1" $$args > "/shared/$$1.mill-token" 95 95 + } 96 96 + seed executor-a linux,fast 97 97 + seed executor-b linux,slow 98 98 + seed executor-c linux,gpu 99 99 + echo "seeded executor tokens" 100 100 + volumes: 101 101 + - spindle-data:/var/lib/spindle 102 102 + - init-state:/shared 103 103 + depends_on: 104 104 + spindle: 105 105 + condition: service_healthy 106 106 + networks: [tngl] 107 107 + 108 108 + spindle-executor-a: 109 109 + <<: *mill-executor 110 110 + environment: 111 111 + <<: *mill-executor-env 112 112 + SPINDLE_SERVER_HOSTNAME: executor-a.tngl.boltless.dev 113 113 + SPINDLE_MILL_LABELS: linux,fast 114 114 + SPINDLE_MILL_TOKEN_FILE: /shared/executor-a.mill-token 115 115 + SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11241" 116 116 + volumes: 117 117 + - spindle-executor-a-data:/var/lib/spindle 118 118 + - spindle-artifacts:/var/lib/spindle/artifacts 119 119 + - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro 120 120 + - init-state:/shared:ro 121 121 + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro 122 122 + 123 123 + spindle-executor-b: 124 124 + <<: *mill-executor 125 125 + environment: 126 126 + <<: *mill-executor-env 127 127 + SPINDLE_SERVER_HOSTNAME: executor-b.tngl.boltless.dev 128 128 + SPINDLE_MILL_LABELS: linux,slow 129 129 + SPINDLE_MILL_TOKEN_FILE: /shared/executor-b.mill-token 130 130 + SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11242" 131 131 + volumes: 132 132 + - spindle-executor-b-data:/var/lib/spindle 133 133 + - spindle-artifacts:/var/lib/spindle/artifacts 134 134 + - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro 135 135 + - init-state:/shared:ro 136 136 + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro 137 137 + 138 138 + spindle-executor-c: 139 139 + <<: *mill-executor 140 140 + environment: 141 141 + <<: *mill-executor-env 142 142 + SPINDLE_SERVER_HOSTNAME: executor-c.tngl.boltless.dev 143 143 + SPINDLE_MILL_LABELS: linux,gpu 144 144 + SPINDLE_MILL_TOKEN_FILE: /shared/executor-c.mill-token 145 145 + SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11243" 146 146 + volumes: 147 147 + - spindle-executor-c-data:/var/lib/spindle 148 148 + - spindle-artifacts:/var/lib/spindle/artifacts 149 149 + - ./out/localinfra-spindle-images-alpine:/var/lib/spindle/images:ro 150 150 + - init-state:/shared:ro 151 151 + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro 152 152 + 153 153 + volumes: 154 154 + spindle-executor-a-data: 155 155 + spindle-executor-b-data: 156 156 + spindle-executor-c-data: 157 157 + spindle-artifacts:

-1

docker-compose.yml

Reviewed

··· 181 181 SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images 182 182 SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays 183 183 SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11240" 184 184 - SPINDLE_S3_LOG_BUCKET: "" 185 184 SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" 186 185 # route guest nix substitution + uploads through the local ncps cache. 187 186 # ncps re-signs on serve with cache.local's key, so the guest trusts the

+49

eventstream/eventstream_test.go

Reviewed

··· 293 293 } 294 294 } 295 295 296 296 + func TestHighWaterSeedsClockFromStoredEvents(t *testing.T) { 297 297 + db, err := sql.Open("sqlite3", t.TempDir()+"/events.db") 298 298 + if err != nil { 299 299 + t.Fatalf("open: %v", err) 300 300 + } 301 301 + t.Cleanup(func() { db.Close() }) 302 302 + if _, err := db.Exec(`create table events ( 303 303 + rkey text not null, 304 304 + nsid text not null, 305 305 + event text not null, 306 306 + created integer not null, 307 307 + primary key (rkey, nsid) 308 308 + )`); err != nil { 309 309 + t.Fatalf("schema: %v", err) 310 310 + } 311 311 + 312 312 + stored := time.Now().Add(time.Hour).UnixNano() 313 313 + if _, err := db.Exec( 314 314 + `insert into events (rkey, nsid, event, created) values (?, ?, ?, ?)`, 315 315 + "stored", "sh.tangled.test", "{}", stored, 316 316 + ); err != nil { 317 317 + t.Fatalf("seed event: %v", err) 318 318 + } 319 319 + 320 320 + cut, err := HighWater(db) 321 321 + if err != nil { 322 322 + t.Fatalf("HighWater() error = %v", err) 323 323 + } 324 324 + if cut < stored { 325 325 + t.Fatalf("HighWater() = %d, want at least stored cursor %d", cut, stored) 326 326 + } 327 327 + 328 328 + n := notifier.New() 329 329 + if err := Insert(db, Event{ 330 330 + Rkey: "new", 331 331 + Nsid: "sh.tangled.test", 332 332 + EventJson: json.RawMessage("{}"), 333 333 + }, &n); err != nil { 334 334 + t.Fatalf("Insert() error = %v", err) 335 335 + } 336 336 + events, err := List(db, cut, 10) 337 337 + if err != nil { 338 338 + t.Fatalf("List() error = %v", err) 339 339 + } 340 340 + if len(events) != 1 || events[0].Rkey != "new" || events[0].Created <= cut { 341 341 + t.Fatalf("events after cut = %+v, want only new event above %d", events, cut) 342 342 + } 343 343 + } 344 344 + 296 345 func isCloseErr(err error) bool { 297 346 if err == nil { 298 347 return false

+29

eventstream/store.go

Reviewed

··· 19 19 lastNanos int64 20 20 ) 21 21 22 22 + func HighWater(s Store) (int64, error) { 23 23 + clockMu.Lock() 24 24 + defer clockMu.Unlock() 25 25 + 26 26 + rows, err := s.Query(`select coalesce(max(created), 0) from events`) 27 27 + if err != nil { 28 28 + return 0, err 29 29 + } 30 30 + defer rows.Close() 31 31 + 32 32 + var created int64 33 33 + if !rows.Next() { 34 34 + if err := rows.Err(); err != nil { 35 35 + return 0, err 36 36 + } 37 37 + return 0, sql.ErrNoRows 38 38 + } 39 39 + if err := rows.Scan(&created); err != nil { 40 40 + return 0, err 41 41 + } 42 42 + if err := rows.Err(); err != nil { 43 43 + return 0, err 44 44 + } 45 45 + if created > lastNanos { 46 46 + lastNanos = created 47 47 + } 48 48 + return lastNanos, nil 49 49 + } 50 50 + 22 51 func Insert(s Store, ev Event, n *notifier.Notifier) error { 23 52 clockMu.Lock() 24 53 defer clockMu.Unlock()

+6

lexicons/pipeline/pipeline.json

Reviewed

··· 184 184 "engine": { 185 185 "type": "string" 186 186 }, 187 187 + "runsOn": { 188 188 + "type": "array", 189 189 + "items": { 190 190 + "type": "string" 191 191 + } 192 192 + }, 187 193 "clone": { 188 194 "type": "ref", 189 195 "ref": "#cloneOpts"

+20

localinfra/readme.md

Reviewed

··· 51 51 This writes the image directory under `out/localinfra-spindle-images`. 52 52 5. `docker compose up` 53 53 6. AppView will be running on `127.0.0.1:3000` with two test users: `alice.pds.tngl.boltless.dev` and `bob.pds.tngl.boltless.dev`. Both with password `password`. 54 54 + 55 55 + ## Mill mode 56 56 + 57 57 + The default stack runs one standalone spindle. `docker-compose.mill.yml` is an overlay that splits that into the distributed arch: the primary spindle becomes the mill host (`role=mill`, it only places jobs) and a three-executor fleet (`role=executor`) runs the engines. 58 58 + 59 59 + ```bash 60 60 + docker compose -f docker-compose.yml -f docker-compose.mill.yml --profile linux up 61 61 + ``` 62 62 + 63 63 + The executors differ on the two axes placement cares about, so you can watch candidates get filtered and ranked: 64 64 + 65 65 + | executor | labels | images | runs | 66 66 + |------------|---------------|-------------|-----------------------------| 67 67 + | executor-a | `linux, fast` | full | `image/alpine, image/nixos` | 68 68 + | executor-b | `linux, slow` | full | `image/alpine, image/nixos` | 69 69 + | executor-c | `linux, gpu` | alpine only | `image/alpine` | 70 70 + 71 71 + So `image: nixos` has two candidates, `image: alpine` has three, and `runs_on: [gpu]` pins to executor-c. The alpine-only image set is staged by `prepare-spindle-images.sh` (step 4) alongside the full one, no extra step. 72 72 + 73 73 + Each executor needs its own identity (one live session per token), so the `mill-tokens` service registers a token per executor in the mill db and drops it into the shared volume for the executor to read.

+10

localinfra/scripts/prepare-spindle-images.sh

Reviewed

··· 29 29 extract_image spindle-nixos-image-tarball nixos-x86_64 nixos 30 30 extract_image spindle-alpine-image-tarball alpine-x86_64 alpine 31 31 32 32 + # a reduced image set (alpine only) for the mill overlay: an executor mounting 33 33 + # this advertises image/alpine but not image/nixos, so capability-based 34 34 + # placement is testable across a mixed fleet 35 35 + alpine_only="$repo/out/localinfra-spindle-images-alpine" 36 36 + [ -d "$alpine_only" ] && chmod -R +w "$alpine_only" || true 37 37 + rm -rf "$alpine_only" 38 38 + mkdir -p "$alpine_only" 39 39 + cp -r "$image_root/alpine-x86_64" "$alpine_only/alpine-x86_64" 40 40 + ln -s alpine-x86_64 "$alpine_only/alpine" 41 41 + 32 42 echo "prepared spindle microVM images in $image_root"

+5

localinfra/spindle.Dockerfile

Reviewed

··· 49 49 export SPINDLE_SERVER_OWNER="$(cat /shared/owner-did)" 50 50 : "${SPINDLE_SERVER_OWNER:?set via env or /shared/owner-did}" 51 51 52 52 + # executors read their mill token from a file seeded by the mill-tokens 53 53 + # service (command substitution strips the trailing newline) 54 54 + [ -z "${SPINDLE_MILL_SHARED_SECRET:-}" ] && [ -n "${SPINDLE_MILL_TOKEN_FILE:-}" ] && [ -r "${SPINDLE_MILL_TOKEN_FILE}" ] && \ 55 55 + export SPINDLE_MILL_SHARED_SECRET="$(cat "${SPINDLE_MILL_TOKEN_FILE}")" 56 56 + 52 57 mkdir -p /var/lib/spindle /var/lib/spindle/overlays /var/log/spindle 53 58 54 59 if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then

+3 -3

netutil/ssrf.go

Reviewed

··· 10 10 "github.com/gorilla/websocket" 11 11 ) 12 12 13 13 - // SSRFDialer returns a net.Dialer that refuses non-public IPs. 13 13 + // refuses non-public ips to prevent ssrf 14 14 func SSRFDialer(dev bool) *net.Dialer { 15 15 if dev { 16 16 return &net.Dialer{} ··· 18 18 return ssrf.PublicOnlyDialer() 19 19 } 20 20 21 21 - // SSRFTransport returns an http.Transport that refuses non-public IPs. 21 21 + // refuses non-public ips to prevent ssrf 22 22 func SSRFTransport(dev bool) *http.Transport { 23 23 if dev { 24 24 return &http.Transport{} ··· 26 26 return ssrf.PublicOnlyTransport() 27 27 } 28 28 29 29 - // SSRFWebsocketDialer returns a websocket.Dialer that refuses non-public IPs. 29 29 + // refuses non-public ips to prevent ssrf 30 30 func SSRFWebsocketDialer(dev bool) *websocket.Dialer { 31 31 dialer := *websocket.DefaultDialer 32 32 dialer.NetDialContext = SSRFDialer(dev).DialContext

+22 -4

nix/modules/spindle.nix

Reviewed

··· 136 136 }; 137 137 }; 138 138 139 139 - pipelines = { 140 140 - logBucket = mkOption { 139 139 + artifactStores = { 140 140 + disk.dir = mkOption { 141 141 + type = types.path; 142 142 + default = "/var/log/spindle"; 143 143 + description = "Root directory for disk artifacts"; 144 144 + }; 145 145 + 146 146 + s3.bucket = mkOption { 141 147 type = types.str; 142 148 default = "tangled-logs"; 143 143 - description = "S3 bucket for workflow logs"; 149 149 + description = "S3 bucket for artifacts"; 150 150 + }; 151 151 + 152 152 + s3.region = mkOption { 153 153 + type = types.str; 154 154 + default = "us-east-1"; 155 155 + description = "AWS region for the artifact bucket"; 144 156 }; 157 157 + }; 158 158 + 159 159 + pipelines = { 145 160 workflowTimeout = mkOption { 146 161 type = types.str; 147 162 default = "5m"; ··· 388 403 "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.pipelines.nixCache.readUrls}" 389 404 "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.pipelines.nixCache.trustedPublicKeys}" 390 405 "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.pipelines.nixCache.uploadUrl}" 391 391 - "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}" 406 406 + "SPINDLE_ARTIFACT_STORES_DISK_DIR=${cfg.artifactStores.disk.dir}" 407 407 + "SPINDLE_ARTIFACT_STORES_S3_BUCKET=${cfg.artifactStores.s3.bucket}" 408 408 + "SPINDLE_ARTIFACT_STORES_S3_REGION=${cfg.artifactStores.s3.region}" 409 409 + "SPINDLE_MILL_ARTIFACT_STORE=s3" 392 410 ]; 393 411 ExecStart = "${cfg.package}/bin/spindle"; 394 412 Restart = "always";

+2 -1

nix/vm.nix

Reviewed

··· 162 162 }; 163 163 }; 164 164 165 165 + artifactStores.s3.bucket = envVarOr "SPINDLE_ARTIFACT_STORES_S3_BUCKET" "tangled-logs"; 166 166 + 165 167 pipelines = { 166 166 - logBucket = envVarOr "SPINDLE_S3_LOG_BUCKET" ""; 167 168 microvm.enableKVM = nestedVirt; 168 169 nixCache = { 169 170 readUrls = ["http://127.0.0.1:8501"];

+240

spindle/artifactstore/artifactstore.go

Reviewed

··· 1 1 + package artifactstore 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "errors" 6 6 + "fmt" 7 7 + "io" 8 8 + "os" 9 9 + "path/filepath" 10 10 + "strings" 11 11 + 12 12 + "github.com/aws/aws-sdk-go-v2/aws" 13 13 + "github.com/aws/aws-sdk-go-v2/config" 14 14 + "github.com/aws/aws-sdk-go-v2/service/s3" 15 15 + spindleconfig "tangled.org/core/spindle/config" 16 16 + ) 17 17 + 18 18 + type Writer interface { 19 19 + Put(ctx context.Context, ref string, r io.Reader) error 20 20 + } 21 21 + 22 22 + type Reader interface { 23 23 + Open(ctx context.Context, ref string) (io.ReadCloser, error) 24 24 + } 25 25 + 26 26 + type Store interface { 27 27 + Writer 28 28 + Reader 29 29 + } 30 30 + 31 31 + type DiskStore struct { 32 32 + root string 33 33 + } 34 34 + 35 35 + func NewDiskStore(root string) (*DiskStore, error) { 36 36 + if root == "" { 37 37 + return nil, fmt.Errorf("artifact disk directory is required") 38 38 + } 39 39 + return &DiskStore{root: filepath.Clean(root)}, nil 40 40 + } 41 41 + 42 42 + func (s *DiskStore) Put(_ context.Context, ref string, r io.Reader) error { 43 43 + path, err := s.resolve(ref) 44 44 + if err != nil { 45 45 + return err 46 46 + } 47 47 + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { 48 48 + return fmt.Errorf("mkdir for artifact %q: %w", path, err) 49 49 + } 50 50 + tmpFile, err := os.CreateTemp(filepath.Dir(path), ".tmp-artifact-*") 51 51 + if err != nil { 52 52 + return fmt.Errorf("create temp artifact: %w", err) 53 53 + } 54 54 + tmpPath := tmpFile.Name() 55 55 + defer func() { 56 56 + _ = tmpFile.Close() 57 57 + _ = os.Remove(tmpPath) 58 58 + }() 59 59 + if _, err := io.Copy(tmpFile, r); err != nil { 60 60 + return fmt.Errorf("write artifact content: %w", err) 61 61 + } 62 62 + if err := tmpFile.Sync(); err != nil { 63 63 + return fmt.Errorf("sync artifact file: %w", err) 64 64 + } 65 65 + if err := tmpFile.Close(); err != nil { 66 66 + return fmt.Errorf("close artifact file: %w", err) 67 67 + } 68 68 + if err := os.Rename(tmpPath, path); err != nil { 69 69 + return fmt.Errorf("rename artifact file to target: %w", err) 70 70 + } 71 71 + return nil 72 72 + } 73 73 + 74 74 + func (s *DiskStore) Open(_ context.Context, ref string) (io.ReadCloser, error) { 75 75 + path, err := s.resolve(ref) 76 76 + if err != nil { 77 77 + return nil, err 78 78 + } 79 79 + f, err := os.Open(path) 80 80 + if err != nil { 81 81 + return nil, fmt.Errorf("open disk artifact %q: %w", path, err) 82 82 + } 83 83 + return f, nil 84 84 + } 85 85 + 86 86 + func (s *DiskStore) resolve(ref string) (string, error) { 87 87 + if ref == "" || filepath.IsAbs(ref) { 88 88 + return "", fmt.Errorf("invalid disk artifact ref %q", ref) 89 89 + } 90 90 + path := filepath.Join(s.root, filepath.Clean(ref)) 91 91 + rel, err := filepath.Rel(s.root, path) 92 92 + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { 93 93 + return "", fmt.Errorf("artifact ref %q escapes disk root %q", ref, s.root) 94 94 + } 95 95 + return path, nil 96 96 + } 97 97 + 98 98 + type s3API interface { 99 99 + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) 100 100 + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) 101 101 + } 102 102 + 103 103 + type S3Store struct { 104 104 + client s3API 105 105 + bucket string 106 106 + } 107 107 + 108 108 + func NewS3Store(client s3API, bucket string) (*S3Store, error) { 109 109 + if client == nil { 110 110 + return nil, fmt.Errorf("s3 client is required") 111 111 + } 112 112 + if bucket == "" { 113 113 + return nil, fmt.Errorf("artifact S3 bucket is required") 114 114 + } 115 115 + return &S3Store{client: client, bucket: bucket}, nil 116 116 + } 117 117 + 118 118 + func (s *S3Store) Put(ctx context.Context, ref string, r io.Reader) error { 119 119 + if err := validateObjectRef(ref); err != nil { 120 120 + return err 121 121 + } 122 122 + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ 123 123 + Bucket: aws.String(s.bucket), 124 124 + Key: aws.String(ref), 125 125 + Body: r, 126 126 + }) 127 127 + if err != nil { 128 128 + return fmt.Errorf("s3 put object: %w", err) 129 129 + } 130 130 + return nil 131 131 + } 132 132 + 133 133 + func (s *S3Store) Open(ctx context.Context, ref string) (io.ReadCloser, error) { 134 134 + if err := validateObjectRef(ref); err != nil { 135 135 + return nil, err 136 136 + } 137 137 + res, err := s.client.GetObject(ctx, &s3.GetObjectInput{ 138 138 + Bucket: aws.String(s.bucket), 139 139 + Key: aws.String(ref), 140 140 + }) 141 141 + if err != nil { 142 142 + return nil, fmt.Errorf("s3 get object: %w", err) 143 143 + } 144 144 + return res.Body, nil 145 145 + } 146 146 + 147 147 + func validateObjectRef(ref string) error { 148 148 + if ref == "" || strings.HasPrefix(ref, "/") || strings.Contains(ref, "://") { 149 149 + return fmt.Errorf("invalid artifact ref %q", ref) 150 150 + } 151 151 + return nil 152 152 + } 153 153 + 154 154 + type Stores struct { 155 155 + order []string 156 156 + stores map[string]Store 157 157 + } 158 158 + 159 159 + func NewStores(cfg spindleconfig.ArtifactStores, diskFallback, legacyS3Bucket string) (*Stores, error) { 160 160 + stores := &Stores{stores: make(map[string]Store)} 161 161 + diskDir := cfg.Disk.Dir 162 162 + if diskDir == "" { 163 163 + diskDir = diskFallback 164 164 + } 165 165 + if diskDir != "" { 166 166 + disk, err := NewDiskStore(diskDir) 167 167 + if err != nil { 168 168 + return nil, err 169 169 + } 170 170 + stores.order = append(stores.order, "disk") 171 171 + stores.stores["disk"] = disk 172 172 + } 173 173 + 174 174 + bucket := cfg.S3.Bucket 175 175 + if bucket == "" { 176 176 + bucket = legacyS3Bucket 177 177 + } 178 178 + if bucket != "" { 179 179 + awsCfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(cfg.S3.Region)) 180 180 + if err != nil { 181 181 + return nil, fmt.Errorf("load aws config: %w", err) 182 182 + } 183 183 + s3Store, err := NewS3Store(s3.NewFromConfig(awsCfg), bucket) 184 184 + if err != nil { 185 185 + return nil, err 186 186 + } 187 187 + stores.order = append(stores.order, "s3") 188 188 + stores.stores["s3"] = s3Store 189 189 + } 190 190 + return stores, nil 191 191 + } 192 192 + 193 193 + func (s *Stores) Names() []string { 194 194 + return append([]string(nil), s.order...) 195 195 + } 196 196 + 197 197 + func (s *Stores) Store(name string) (Store, bool) { 198 198 + store, ok := s.stores[name] 199 199 + return store, ok 200 200 + } 201 201 + 202 202 + func (s *Stores) Open(ctx context.Context, ref string) (io.ReadCloser, error) { 203 203 + var errs []error 204 204 + for _, name := range s.order { 205 205 + rc, err := s.stores[name].Open(ctx, ref) 206 206 + if err == nil { 207 207 + return rc, nil 208 208 + } 209 209 + errs = append(errs, fmt.Errorf("%s: %w", name, err)) 210 210 + } 211 211 + return nil, fmt.Errorf("open artifact %q: %w", ref, errors.Join(errs...)) 212 212 + } 213 213 + 214 214 + func (s *Stores) PutFile(ctx context.Context, ref, sourcePath string) []error { 215 215 + var errs []error 216 216 + for _, name := range s.order { 217 217 + store := s.stores[name] 218 218 + if disk, ok := store.(*DiskStore); ok { 219 219 + target, err := disk.resolve(ref) 220 220 + if err == nil { 221 221 + source, sourceErr := filepath.Abs(sourcePath) 222 222 + targetAbs, targetErr := filepath.Abs(target) 223 223 + if sourceErr == nil && targetErr == nil && source == targetAbs { 224 224 + continue 225 225 + } 226 226 + } 227 227 + } 228 228 + file, err := os.Open(sourcePath) 229 229 + if err != nil { 230 230 + errs = append(errs, fmt.Errorf("%s: open source: %w", name, err)) 231 231 + continue 232 232 + } 233 233 + err = store.Put(ctx, ref, file) 234 234 + _ = file.Close() 235 235 + if err != nil { 236 236 + errs = append(errs, fmt.Errorf("%s: %w", name, err)) 237 237 + } 238 238 + } 239 239 + return errs 240 240 + }

+130

spindle/artifactstore/artifactstore_test.go

Reviewed

··· 1 1 + package artifactstore 2 2 + 3 3 + import ( 4 4 + "bytes" 5 5 + "context" 6 6 + "io" 7 7 + "os" 8 8 + "strings" 9 9 + "sync" 10 10 + "testing" 11 11 + 12 12 + "github.com/aws/aws-sdk-go-v2/service/s3" 13 13 + ) 14 14 + 15 15 + type mockS3Client struct { 16 16 + mu sync.Mutex 17 17 + store map[string][]byte 18 18 + } 19 19 + 20 20 + func newMockS3Client() *mockS3Client { 21 21 + return &mockS3Client{ 22 22 + store: make(map[string][]byte), 23 23 + } 24 24 + } 25 25 + 26 26 + func (m *mockS3Client) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { 27 27 + m.mu.Lock() 28 28 + defer m.mu.Unlock() 29 29 + b, err := io.ReadAll(params.Body) 30 30 + if err != nil { 31 31 + return nil, err 32 32 + } 33 33 + key := *params.Bucket + "/" + *params.Key 34 34 + m.store[key] = b 35 35 + return &s3.PutObjectOutput{}, nil 36 36 + } 37 37 + 38 38 + func (m *mockS3Client) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { 39 39 + m.mu.Lock() 40 40 + defer m.mu.Unlock() 41 41 + key := *params.Bucket + "/" + *params.Key 42 42 + data, ok := m.store[key] 43 43 + if !ok { 44 44 + return nil, os.ErrNotExist 45 45 + } 46 46 + return &s3.GetObjectOutput{ 47 47 + Body: io.NopCloser(bytes.NewReader(data)), 48 48 + }, nil 49 49 + } 50 50 + 51 51 + func TestDiskStore(t *testing.T) { 52 52 + tempDir := t.TempDir() 53 53 + store, err := NewDiskStore(tempDir) 54 54 + if err != nil { 55 55 + t.Fatal(err) 56 56 + } 57 57 + 58 58 + ctx := context.Background() 59 59 + ref := "logs/test.log" 60 60 + content := "hello world log content" 61 61 + 62 62 + if err := store.Put(ctx, ref, strings.NewReader(content)); err != nil { 63 63 + t.Fatalf("Put failed: %v", err) 64 64 + } 65 65 + 66 66 + rc, err := store.Open(ctx, ref) 67 67 + if err != nil { 68 68 + t.Fatalf("Open failed: %v", err) 69 69 + } 70 70 + defer rc.Close() 71 71 + 72 72 + got, err := io.ReadAll(rc) 73 73 + if err != nil { 74 74 + t.Fatalf("ReadAll failed: %v", err) 75 75 + } 76 76 + if string(got) != content { 77 77 + t.Fatalf("got content %q, want %q", string(got), content) 78 78 + } 79 79 + } 80 80 + 81 81 + func TestDiskStoreTraversalProtection(t *testing.T) { 82 82 + tempDir := t.TempDir() 83 83 + store, err := NewDiskStore(tempDir) 84 84 + if err != nil { 85 85 + t.Fatal(err) 86 86 + } 87 87 + 88 88 + ctx := context.Background() 89 89 + badRef := "../outside" 90 90 + 91 91 + err = store.Put(ctx, badRef, strings.NewReader("bad")) 92 92 + if err == nil { 93 93 + t.Fatal("expected error putting file outside diskDir, got nil") 94 94 + } 95 95 + 96 96 + _, err = store.Open(ctx, badRef) 97 97 + if err == nil { 98 98 + t.Fatal("expected error opening file outside diskDir, got nil") 99 99 + } 100 100 + } 101 101 + 102 102 + func TestS3Store(t *testing.T) { 103 103 + mock := newMockS3Client() 104 104 + store, err := NewS3Store(mock, "mybucket") 105 105 + if err != nil { 106 106 + t.Fatal(err) 107 107 + } 108 108 + 109 109 + ctx := context.Background() 110 110 + ref := "logs/run1.log" 111 111 + content := "s3 log payload" 112 112 + 113 113 + if err := store.Put(ctx, ref, strings.NewReader(content)); err != nil { 114 114 + t.Fatalf("Put to S3 failed: %v", err) 115 115 + } 116 116 + 117 117 + rc, err := store.Open(ctx, ref) 118 118 + if err != nil { 119 119 + t.Fatalf("Open from S3 failed: %v", err) 120 120 + } 121 121 + defer rc.Close() 122 122 + 123 123 + got, err := io.ReadAll(rc) 124 124 + if err != nil { 125 125 + t.Fatalf("ReadAll failed: %v", err) 126 126 + } 127 127 + if string(got) != content { 128 128 + t.Fatalf("got content %q, want %q", string(got), content) 129 129 + } 130 130 + }

+63 -3

spindle/config/config.go

Reviewed

··· 2 2 3 3 import ( 4 4 "context" 5 5 + "fmt" 5 6 "time" 6 7 7 8 "github.com/bluesky-social/indigo/atproto/syntax" ··· 57 58 MaxConcurrentWorkflows int `env:"MAX_CONCURRENT_WORKFLOWS, default=8"` // max number of workflow containers running at once (memory cap) 58 59 } 59 60 60 60 - type S3 struct { 61 61 + type ArtifactStoreDisk struct { 62 62 + Dir string `env:"DIR"` 63 63 + } 64 64 + 65 65 + type ArtifactStoreS3 struct { 66 66 + Bucket string `env:"BUCKET"` 67 67 + Region string `env:"REGION, default=us-east-1"` 68 68 + } 69 69 + 70 70 + type ArtifactStores struct { 71 71 + Disk ArtifactStoreDisk `env:",prefix=DISK_"` 72 72 + S3 ArtifactStoreS3 `env:",prefix=S3_"` 73 73 + } 74 74 + 75 75 + type LegacyS3 struct { 61 76 LogBucket string `env:"LOG_BUCKET"` 62 77 } 63 78 64 79 type MicroVMPipelines struct { 65 80 ImageDir string `env:"IMAGE_DIR"` 66 66 - OverlayDir string `env:"OVERLAY_DIR, default="` // where microVM temporary disks will live 81 81 + OverlayDir string `env:"OVERLAY_DIR"` // where microVM temporary disks will live 67 82 DefaultImage string `env:"DEFAULT_IMAGE, default=nixos-x86_64"` 68 83 AgentPort uint32 `env:"AGENT_PORT, default=10240"` 69 84 EnableKVM bool `env:"ENABLE_KVM, default=true"` ··· 93 108 UploadURL string `env:"UPLOAD_URL"` 94 109 } 95 110 111 111 + // governs how spindle places and runs jobs 112 112 + type Role string 113 113 + 114 114 + const ( 115 115 + RoleStandalone Role = "standalone" 116 116 + RoleMill Role = "mill" 117 117 + RoleExecutor Role = "executor" 118 118 + ) 119 119 + 120 120 + // fields are selectively active depending on the role 121 121 + type Mill struct { 122 122 + URL string `env:"URL"` // mill websocket endpoint dialled by the executor 123 123 + SharedSecret string `env:"SHARED_SECRET"` // the executor's token for dialing the mill 124 124 + MaxPending int `env:"MAX_PENDING, default=100"` // mill pending job queue limit 125 125 + ReconnectGrace time.Duration `env:"RECONNECT_GRACE, default=45s"` // reconnect window before leases are failed 126 126 + Seats int `env:"SEATS, default=4"` // executor seats advertised to the mill 127 127 + Labels []string `env:"LABELS"` // executor capability labels 128 128 + ArtifactStore string `env:"ARTIFACT_STORE"` // store shared by mill and its executors 129 129 + } 130 130 + 96 131 type Config struct { 132 132 + Role Role `env:"SPINDLE_ROLE, default=standalone"` 97 133 Server Server `env:",prefix=SPINDLE_SERVER_"` 98 134 NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` 99 135 MicroVMPipelines MicroVMPipelines `env:",prefix=SPINDLE_MICROVM_PIPELINES_"` 100 136 NixCache NixCache `env:",prefix=SPINDLE_NIX_CACHE_"` 101 101 - S3 S3 `env:",prefix=SPINDLE_S3_"` 137 137 + ArtifactStores ArtifactStores `env:",prefix=SPINDLE_ARTIFACT_STORES_"` 138 138 + LegacyS3 LegacyS3 `env:",prefix=SPINDLE_S3_"` 139 139 + Mill Mill `env:",prefix=SPINDLE_MILL_"` 140 140 + } 141 141 + 142 142 + func (c *Config) validate() error { 143 143 + switch c.Role { 144 144 + case RoleStandalone, RoleMill: 145 145 + if c.Mill.URL != "" { 146 146 + return fmt.Errorf("SPINDLE_MILL_URL is set but SPINDLE_ROLE=%s; only an executor dials a mill", c.Role) 147 147 + } 148 148 + case RoleExecutor: 149 149 + if c.Mill.URL == "" { 150 150 + return fmt.Errorf("SPINDLE_ROLE=executor requires SPINDLE_MILL_URL (the mill to dial)") 151 151 + } 152 152 + if c.Mill.SharedSecret == "" { 153 153 + return fmt.Errorf("SPINDLE_ROLE=executor requires SPINDLE_MILL_SHARED_SECRET (its executor token)") 154 154 + } 155 155 + default: 156 156 + return fmt.Errorf("unknown SPINDLE_ROLE %q (want standalone, mill, or executor)", c.Role) 157 157 + } 158 158 + return nil 102 159 } 103 160 104 161 func Load(ctx context.Context) (*Config, error) { ··· 107 164 if err != nil { 108 165 return nil, err 109 166 } 167 167 + if err := cfg.validate(); err != nil { 168 168 + return nil, err 169 169 + } 110 170 111 171 return &cfg, nil 112 172 }

+64 -12

spindle/db/db.go

Reviewed

··· 79 79 ); 80 80 81 81 create table if not exists spindle_members ( 82 82 - -- identifiers for the record 83 82 id integer primary key autoincrement, 84 83 did text not null, 85 84 rkey text not null, 86 86 - 87 87 - -- data 88 85 instance text not null, 89 86 subject text not null, 90 87 created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), 91 91 - 92 92 - -- constraints 93 88 unique (did, rkey) 94 89 ); 95 90 96 96 - -- status event for a single workflow 97 91 create table if not exists events ( 98 92 rkey text not null, 99 93 nsid text not null, 100 100 - event text not null, -- json 101 101 - created integer not null -- unix nanos 94 94 + event text not null, 95 95 + created integer not null 102 96 ); 103 97 104 98 create table if not exists nixos_toplevel_cache ( ··· 132 126 foreign key (pipeline_id) references pipelines(id) on delete cascade 133 127 ); 134 128 129 129 + create table if not exists mill_executors ( 130 130 + name text primary key, 131 131 + token_hash text not null unique, 132 132 + created_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), 133 133 + expires_at text, 134 134 + labels text, 135 135 + quarantine_reason text, 136 136 + quarantined_at text 137 137 + ); 138 138 + 139 139 + create table if not exists mill_leases ( 140 140 + lease_id text primary key, 141 141 + node_id text not null, 142 142 + epoch text not null, 143 143 + engine text not null, 144 144 + knot text not null, 145 145 + rkey text not null, 146 146 + workflow text not null, 147 147 + state text not null 148 148 + ); 149 149 + 150 150 + create table if not exists mill_executor_cursors ( 151 151 + node_id text not null, 152 152 + epoch text not null, 153 153 + acked_seqno integer not null, 154 154 + primary key (node_id, epoch) 155 155 + ); 156 156 + 157 157 + create table if not exists mill_outbox_state ( 158 158 + epoch text not null, 159 159 + next_seqno integer not null, 160 160 + primary key (epoch) 161 161 + ); 162 162 + 163 163 + create table if not exists mill_outbox_rows ( 164 164 + epoch text not null, 165 165 + seqno integer not null, 166 166 + payload blob not null, 167 167 + byte_size integer not null, 168 168 + control integer not null, 169 169 + primary key (epoch, seqno), 170 170 + foreign key (epoch) references mill_outbox_state(epoch) on delete cascade 171 171 + ); 172 172 + 173 173 + create table if not exists mill_artifacts ( 174 174 + id integer primary key autoincrement, 175 175 + lease_id text not null, 176 176 + workflow text not null, 177 177 + ref text not null, 178 178 + hash text not null 179 179 + ); 180 180 + 181 181 + create table if not exists executor_pending_artifacts ( 182 182 + lease_id text primary key, 183 183 + workflow text not null, 184 184 + status text not null, 185 185 + error text not null default '', 186 186 + exit_code integer not null default 0, 187 187 + ref text not null, 188 188 + hash text not null 189 189 + ); 190 190 + 135 191 create table if not exists migrations ( 136 192 id integer primary key autoincrement, 137 193 name text unique 138 194 ); 139 195 `) 140 140 - if err != nil { 141 141 - return nil, err 142 142 - } 143 143 - 144 196 if err := runMigrations(ctx, conn, logger); err != nil { 145 197 return nil, err 146 198 }

+59 -15

spindle/db/events.go

Reviewed

··· 19 19 return eventstream.List(d, cursor, limit) 20 20 } 21 21 22 22 + func (d *DB) EventHighWater() (int64, error) { 23 23 + return eventstream.HighWater(d) 24 24 + } 25 25 + 22 26 func (d *DB) CreatePipelineEvent(rkey string, pipeline tangled.Pipeline, n *notifier.Notifier) error { 23 27 eventJson, err := json.Marshal(pipeline) 24 28 if err != nil { ··· 32 36 return d.insertEvent(event, n) 33 37 } 34 38 35 35 - func (d *DB) createStatusEvent( 36 36 - workflowId models.WorkflowId, 37 37 - statusKind models.StatusKind, 38 38 - workflowError *string, 39 39 - exitCode *int64, 40 40 - n *notifier.Notifier, 41 41 - ) error { 42 42 - now := time.Now() 43 43 - pipelineAtUri := workflowId.PipelineId.AtUri() 39 39 + // leaves created at zero so insertevent stamps the local clock 40 40 + func statusEvent(pipelineAtUri, workflow, status string, workflowError *string, exitCode *int64) (eventstream.Event, error) { 44 41 s := tangled.PipelineStatus{ 45 45 - CreatedAt: now.Format(time.RFC3339), 42 42 + CreatedAt: time.Now().Format(time.RFC3339), 46 43 Error: workflowError, 47 44 ExitCode: exitCode, 48 48 - Pipeline: string(pipelineAtUri), 49 49 - Workflow: workflowId.Name, 50 50 - Status: string(statusKind), 45 45 + Pipeline: pipelineAtUri, 46 46 + Workflow: workflow, 47 47 + Status: status, 51 48 } 52 49 53 50 eventJson, err := json.Marshal(s) 54 51 if err != nil { 55 55 - return err 52 52 + return eventstream.Event{}, err 56 53 } 57 54 58 58 - event := eventstream.Event{ 55 55 + return eventstream.Event{ 59 56 Rkey: tid.TID(), 60 57 Nsid: tangled.PipelineStatusNSID, 61 58 EventJson: eventJson, 59 59 + }, nil 60 60 + } 61 61 + 62 62 + func (d *DB) createStatusEvent( 63 63 + workflowId models.WorkflowId, 64 64 + statusKind models.StatusKind, 65 65 + workflowError *string, 66 66 + exitCode *int64, 67 67 + n *notifier.Notifier, 68 68 + ) error { 69 69 + event, err := statusEvent(string(workflowId.PipelineId.AtUri()), workflowId.Name, string(statusKind), workflowError, exitCode) 70 70 + if err != nil { 71 71 + return err 62 72 } 73 73 + return d.insertEvent(event, n) 74 74 + } 63 75 76 76 + // stamps the mill's own clock so it orders against the cursor like a local write 77 77 + func (d *DB) InsertEventStatus( 78 78 + pipelineAtUri string, 79 79 + workflow string, 80 80 + status string, 81 81 + workflowError *string, 82 82 + exitCode *int64, 83 83 + n *notifier.Notifier, 84 84 + ) error { 85 85 + event, err := statusEvent(pipelineAtUri, workflow, status, workflowError, exitCode) 86 86 + if err != nil { 87 87 + return err 88 88 + } 64 89 return d.insertEvent(event, n) 65 90 } 66 91 92 92 + // deleting the lease in the same transaction prevents the terminal event 93 93 + // from replaying 94 94 + func (d *DB) CompleteMillLease( 95 95 + leaseID string, 96 96 + pipelineAtUri string, 97 97 + workflow string, 98 98 + status string, 99 99 + workflowError *string, 100 100 + exitCode *int64, 101 101 + n *notifier.Notifier, 102 102 + ) error { 103 103 + return d.ApplyEventBatch(n, func(tx *EventBatchTx) error { 104 104 + if err := tx.InsertStatusEvent(pipelineAtUri, workflow, status, workflowError, exitCode); err != nil { 105 105 + return err 106 106 + } 107 107 + return tx.DeleteLease(leaseID) 108 108 + }) 109 109 + } 110 110 + 67 111 func (d *DB) GetStatus(workflowId models.WorkflowId) (*tangled.PipelineStatus, error) { 68 112 pipelineAtUri := workflowId.PipelineId.AtUri() 69 113

+392

spindle/db/mill_state.go

Reviewed

··· 1 1 + package db 2 2 + 3 3 + import ( 4 4 + "database/sql" 5 5 + "fmt" 6 6 + 7 7 + "tangled.org/core/eventstream" 8 8 + "tangled.org/core/notifier" 9 9 + ) 10 10 + 11 11 + // enough to rebuild the fencing token and workflow identity after a restart 12 12 + type MillLease struct { 13 13 + LeaseID string 14 14 + NodeID string 15 15 + Epoch string 16 16 + Engine string 17 17 + Knot string 18 18 + Rkey string 19 19 + Workflow string 20 20 + State string 21 21 + } 22 22 + 23 23 + type ExecutorCursor struct { 24 24 + NodeID string 25 25 + Epoch string 26 26 + AckedSeqno uint64 27 27 + } 28 28 + 29 29 + type OutboxRow struct { 30 30 + Epoch string 31 31 + Seqno uint64 32 32 + Payload []byte 33 33 + ByteSize int64 34 34 + Control bool 35 35 + } 36 36 + type OutboxDeletion struct { 37 37 + Rows int64 38 38 + Bytes int64 39 39 + } 40 40 + 41 41 + func (d *DB) SaveMillLease(l MillLease) error { 42 42 + _, err := d.Exec( 43 43 + `insert into mill_leases ( 44 44 + lease_id, node_id, epoch, engine, knot, rkey, workflow, state 45 45 + ) values (?, ?, ?, ?, ?, ?, ?, ?) 46 46 + on conflict(lease_id) do update set state = excluded.state`, 47 47 + l.LeaseID, l.NodeID, l.Epoch, l.Engine, l.Knot, l.Rkey, l.Workflow, l.State, 48 48 + ) 49 49 + return err 50 50 + } 51 51 + 52 52 + func (d *DB) DeleteMillLease(leaseID string) error { 53 53 + _, err := d.Exec(`delete from mill_leases where lease_id = ?`, leaseID) 54 54 + return err 55 55 + } 56 56 + 57 57 + func (d *DB) ListMillLeases() ([]MillLease, error) { 58 58 + rows, err := d.Query(` 59 59 + select lease_id, node_id, epoch, engine, knot, rkey, workflow, state 60 60 + from mill_leases 61 61 + `) 62 62 + if err != nil { 63 63 + return nil, err 64 64 + } 65 65 + defer rows.Close() 66 66 + 67 67 + var leases []MillLease 68 68 + for rows.Next() { 69 69 + var l MillLease 70 70 + if err := rows.Scan( 71 71 + &l.LeaseID, &l.NodeID, &l.Epoch, &l.Engine, &l.Knot, &l.Rkey, &l.Workflow, &l.State, 72 72 + ); err != nil { 73 73 + return nil, err 74 74 + } 75 75 + leases = append(leases, l) 76 76 + } 77 77 + return leases, rows.Err() 78 78 + } 79 79 + 80 80 + func (d *DB) ListExecutorCursors() ([]ExecutorCursor, error) { 81 81 + rows, err := d.Query(`select node_id, epoch, acked_seqno from mill_executor_cursors`) 82 82 + if err != nil { 83 83 + return nil, err 84 84 + } 85 85 + defer rows.Close() 86 86 + 87 87 + var cursors []ExecutorCursor 88 88 + for rows.Next() { 89 89 + var c ExecutorCursor 90 90 + if err := rows.Scan(&c.NodeID, &c.Epoch, &c.AckedSeqno); err != nil { 91 91 + return nil, err 92 92 + } 93 93 + cursors = append(cursors, c) 94 94 + } 95 95 + return cursors, rows.Err() 96 96 + } 97 97 + 98 98 + func (d *DB) SetOutboxEpoch(epoch string) error { 99 99 + tx, err := d.Begin() 100 100 + if err != nil { 101 101 + return err 102 102 + } 103 103 + defer tx.Rollback() 104 104 + 105 105 + if _, err := tx.Exec(`delete from mill_outbox_rows`); err != nil { 106 106 + return err 107 107 + } 108 108 + if _, err := tx.Exec(`delete from mill_outbox_state`); err != nil { 109 109 + return err 110 110 + } 111 111 + if _, err := tx.Exec(`insert into mill_outbox_state (epoch, next_seqno) values (?, 1)`, epoch); err != nil { 112 112 + return err 113 113 + } 114 114 + return tx.Commit() 115 115 + } 116 116 + 117 117 + func (d *DB) AppendOutboxRow(payload []byte, control bool) (uint64, error) { 118 118 + tx, err := d.Begin() 119 119 + if err != nil { 120 120 + return 0, err 121 121 + } 122 122 + defer tx.Rollback() 123 123 + 124 124 + var epoch string 125 125 + var nextSeqno uint64 126 126 + err = tx.QueryRow(`select epoch, next_seqno from mill_outbox_state limit 1`).Scan(&epoch, &nextSeqno) 127 127 + if err == sql.ErrNoRows { 128 128 + return 0, fmt.Errorf("no outbox epoch set") 129 129 + } else if err != nil { 130 130 + return 0, err 131 131 + } 132 132 + 133 133 + byteSize := int64(len(payload)) 134 134 + controlVal := 0 135 135 + if control { 136 136 + controlVal = 1 137 137 + } 138 138 + 139 139 + if _, err := tx.Exec( 140 140 + `insert into mill_outbox_rows (epoch, seqno, payload, byte_size, control) values (?, ?, ?, ?, ?)`, 141 141 + epoch, nextSeqno, payload, byteSize, controlVal, 142 142 + ); err != nil { 143 143 + return 0, err 144 144 + } 145 145 + 146 146 + if _, err := tx.Exec( 147 147 + `update mill_outbox_state set next_seqno = ? where epoch = ?`, 148 148 + nextSeqno+1, epoch, 149 149 + ); err != nil { 150 150 + return 0, err 151 151 + } 152 152 + 153 153 + if err := tx.Commit(); err != nil { 154 154 + return 0, err 155 155 + } 156 156 + return nextSeqno, nil 157 157 + } 158 158 + 159 159 + func (d *DB) DeleteOutboxPrefix(ackedSeqno uint64) (OutboxDeletion, error) { 160 160 + tx, err := d.Begin() 161 161 + if err != nil { 162 162 + return OutboxDeletion{}, err 163 163 + } 164 164 + defer tx.Rollback() 165 165 + 166 166 + var epoch string 167 167 + err = tx.QueryRow(`select epoch from mill_outbox_state limit 1`).Scan(&epoch) 168 168 + if err == sql.ErrNoRows { 169 169 + return OutboxDeletion{}, nil 170 170 + } 171 171 + if err != nil { 172 172 + return OutboxDeletion{}, err 173 173 + } 174 174 + 175 175 + var deleted OutboxDeletion 176 176 + if err := tx.QueryRow(` 177 177 + select count(*), 178 178 + coalesce(sum(byte_size), 0) 179 179 + from mill_outbox_rows 180 180 + where epoch = ? and seqno <= ? 181 181 + `, epoch, ackedSeqno).Scan(&deleted.Rows, &deleted.Bytes); err != nil { 182 182 + return OutboxDeletion{}, err 183 183 + } 184 184 + if _, err := tx.Exec( 185 185 + `delete from mill_outbox_rows where epoch = ? and seqno <= ?`, 186 186 + epoch, ackedSeqno, 187 187 + ); err != nil { 188 188 + return OutboxDeletion{}, err 189 189 + } 190 190 + if err := tx.Commit(); err != nil { 191 191 + return OutboxDeletion{}, err 192 192 + } 193 193 + return deleted, nil 194 194 + } 195 195 + 196 196 + func (d *DB) ListOutboxRows() ([]OutboxRow, error) { 197 197 + rows, err := d.Query(`select epoch, seqno, payload, byte_size, control from mill_outbox_rows order by seqno`) 198 198 + if err != nil { 199 199 + return nil, err 200 200 + } 201 201 + defer rows.Close() 202 202 + 203 203 + var out []OutboxRow 204 204 + for rows.Next() { 205 205 + var r OutboxRow 206 206 + var controlVal int 207 207 + if err := rows.Scan(&r.Epoch, &r.Seqno, &r.Payload, &r.ByteSize, &controlVal); err != nil { 208 208 + return nil, err 209 209 + } 210 210 + r.Control = (controlVal != 0) 211 211 + out = append(out, r) 212 212 + } 213 213 + return out, rows.Err() 214 214 + } 215 215 + func (d *DB) ListOutboxRowsAfter(seqno uint64, limit int) ([]OutboxRow, error) { 216 216 + rows, err := d.Query(` 217 217 + select epoch, seqno, payload, byte_size, control 218 218 + from mill_outbox_rows 219 219 + where epoch = (select epoch from mill_outbox_state limit 1) 220 220 + and seqno > ? 221 221 + order by seqno 222 222 + limit ? 223 223 + `, seqno, limit) 224 224 + if err != nil { 225 225 + return nil, err 226 226 + } 227 227 + defer rows.Close() 228 228 + 229 229 + var out []OutboxRow 230 230 + for rows.Next() { 231 231 + var row OutboxRow 232 232 + var control int 233 233 + if err := rows.Scan(&row.Epoch, &row.Seqno, &row.Payload, &row.ByteSize, &control); err != nil { 234 234 + return nil, err 235 235 + } 236 236 + row.Control = control != 0 237 237 + out = append(out, row) 238 238 + } 239 239 + return out, rows.Err() 240 240 + } 241 241 + 242 242 + func (d *DB) GetOutboxState() (string, uint64, error) { 243 243 + var epoch string 244 244 + var nextSeqno uint64 245 245 + err := d.QueryRow(`select epoch, next_seqno from mill_outbox_state limit 1`).Scan(&epoch, &nextSeqno) 246 246 + if err == sql.ErrNoRows { 247 247 + return "", 0, nil 248 248 + } 249 249 + return epoch, nextSeqno, err 250 250 + } 251 251 + 252 252 + type FinishedLog struct { 253 253 + LeaseID string 254 254 + Workflow string 255 255 + Ref string 256 256 + Hash string 257 257 + } 258 258 + 259 259 + func (d *DB) GetFinishedLog(workflow string) (*FinishedLog, error) { 260 260 + var fl FinishedLog 261 261 + err := d.QueryRow( 262 262 + `select lease_id, workflow, ref, hash 263 263 + from mill_artifacts 264 264 + where workflow = ? 265 265 + order by id desc limit 1`, 266 266 + workflow, 267 267 + ).Scan(&fl.LeaseID, &fl.Workflow, &fl.Ref, &fl.Hash) 268 268 + if err != nil { 269 269 + return nil, err 270 270 + } 271 271 + return &fl, nil 272 272 + } 273 273 + 274 274 + type EventBatchTx struct { 275 275 + tx *sql.Tx 276 276 + db *DB 277 277 + } 278 278 + 279 279 + func (tx *EventBatchTx) InsertStatusEvent(pipelineAtUri, workflow, status string, workflowError *string, exitCode *int64) error { 280 280 + event, err := statusEvent(pipelineAtUri, workflow, status, workflowError, exitCode) 281 281 + if err != nil { 282 282 + return err 283 283 + } 284 284 + return eventstream.Insert(tx.tx, event, nil) 285 285 + } 286 286 + 287 287 + func (tx *EventBatchTx) DeleteLease(leaseID string) error { 288 288 + _, err := tx.tx.Exec(`delete from mill_leases where lease_id = ?`, leaseID) 289 289 + return err 290 290 + } 291 291 + 292 292 + func (tx *EventBatchTx) AdvanceCursor(nodeID, epoch string, seqno uint64) error { 293 293 + _, err := tx.tx.Exec( 294 294 + `insert into mill_executor_cursors (node_id, epoch, acked_seqno) values (?, ?, ?) 295 295 + on conflict(node_id, epoch) do update set acked_seqno = excluded.acked_seqno`, 296 296 + nodeID, epoch, seqno, 297 297 + ) 298 298 + return err 299 299 + } 300 300 + 301 301 + func (tx *EventBatchTx) InsertArtifactRef(leaseID, workflow, ref, hash string) error { 302 302 + _, err := tx.tx.Exec( 303 303 + `insert into mill_artifacts (lease_id, workflow, ref, hash) 304 304 + values (?, ?, ?, ?)`, 305 305 + leaseID, workflow, ref, hash, 306 306 + ) 307 307 + return err 308 308 + } 309 309 + 310 310 + func (d *DB) SaveArtifactRef(leaseID, workflow, ref, hash string) error { 311 311 + _, err := d.Exec( 312 312 + `insert into mill_artifacts (lease_id, workflow, ref, hash) 313 313 + values (?, ?, ?, ?)`, 314 314 + leaseID, workflow, ref, hash, 315 315 + ) 316 316 + return err 317 317 + } 318 318 + 319 319 + type PendingArtifact struct { 320 320 + LeaseID string 321 321 + Workflow string 322 322 + Status string 323 323 + Error string 324 324 + ExitCode int64 325 325 + Ref string 326 326 + Hash string 327 327 + } 328 328 + 329 329 + func (d *DB) SavePendingArtifact(leaseID, workflow, status, errStr string, exitCode int64, ref, hash string) error { 330 330 + _, err := d.Exec( 331 331 + `insert into executor_pending_artifacts (lease_id, workflow, status, error, exit_code, ref, hash) 332 332 + values (?, ?, ?, ?, ?, ?, ?) 333 333 + on conflict(lease_id) do update set 334 334 + workflow = excluded.workflow, 335 335 + status = excluded.status, 336 336 + error = excluded.error, 337 337 + exit_code = excluded.exit_code, 338 338 + ref = excluded.ref, 339 339 + hash = excluded.hash`, 340 340 + leaseID, workflow, status, errStr, exitCode, ref, hash, 341 341 + ) 342 342 + return err 343 343 + } 344 344 + 345 345 + func (d *DB) RemovePendingArtifact(leaseID string) error { 346 346 + _, err := d.Exec(`delete from executor_pending_artifacts where lease_id = ?`, leaseID) 347 347 + return err 348 348 + } 349 349 + 350 350 + func (d *DB) ListPendingArtifacts() ([]PendingArtifact, error) { 351 351 + rows, err := d.Query(`select lease_id, workflow, status, error, exit_code, ref, hash from executor_pending_artifacts`) 352 352 + if err != nil { 353 353 + return nil, err 354 354 + } 355 355 + defer rows.Close() 356 356 + 357 357 + var res []PendingArtifact 358 358 + for rows.Next() { 359 359 + var p PendingArtifact 360 360 + if err := rows.Scan(&p.LeaseID, &p.Workflow, &p.Status, &p.Error, &p.ExitCode, &p.Ref, &p.Hash); err != nil { 361 361 + return nil, err 362 362 + } 363 363 + res = append(res, p) 364 364 + } 365 365 + return res, rows.Err() 366 366 + } 367 367 + 368 368 + func (d *DB) ApplyEventBatch(n *notifier.Notifier, fn func(tx *EventBatchTx) error) error { 369 369 + tx, err := d.Begin() 370 370 + if err != nil { 371 371 + return err 372 372 + } 373 373 + defer tx.Rollback() 374 374 + 375 375 + batchTx := &EventBatchTx{ 376 376 + tx: tx, 377 377 + db: d, 378 378 + } 379 379 + 380 380 + if err := fn(batchTx); err != nil { 381 381 + return err 382 382 + } 383 383 + 384 384 + if err := tx.Commit(); err != nil { 385 385 + return err 386 386 + } 387 387 + 388 388 + if n != nil { 389 389 + n.NotifyAll() 390 390 + } 391 391 + return nil 392 392 + }

+394

spindle/db/mill_state_test.go

Reviewed

··· 1 1 + package db 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "fmt" 6 6 + "path/filepath" 7 7 + "testing" 8 8 + 9 9 + "tangled.org/core/notifier" 10 10 + ) 11 11 + 12 12 + func TestMillLeaseRoundTrip(t *testing.T) { 13 13 + d := newTestDB(t) 14 14 + 15 15 + lease := MillLease{ 16 16 + LeaseID: "lease-1", 17 17 + NodeID: "node-1", 18 18 + Epoch: "inc-1", 19 19 + Engine: "dummy", 20 20 + Knot: "knot.example", 21 21 + Rkey: "rkey1", 22 22 + Workflow: "build", 23 23 + State: "reserved", 24 24 + } 25 25 + if err := d.SaveMillLease(lease); err != nil { 26 26 + t.Fatalf("SaveMillLease: %v", err) 27 27 + } 28 28 + 29 29 + lease.State = "running" 30 30 + if err := d.SaveMillLease(lease); err != nil { 31 31 + t.Fatalf("SaveMillLease(transition): %v", err) 32 32 + } 33 33 + 34 34 + leases, err := d.ListMillLeases() 35 35 + if err != nil { 36 36 + t.Fatalf("ListMillLeases: %v", err) 37 37 + } 38 38 + if len(leases) != 1 { 39 39 + t.Fatalf("ListMillLeases returned %d leases, want 1 (state transition must replace, not duplicate)", len(leases)) 40 40 + } 41 41 + if leases[0].LeaseID != lease.LeaseID || leases[0].State != "running" { 42 42 + t.Fatalf("ListMillLeases[0] = %+v, want %+v", leases[0], lease) 43 43 + } 44 44 + 45 45 + if err := d.DeleteMillLease("lease-1"); err != nil { 46 46 + t.Fatalf("DeleteMillLease: %v", err) 47 47 + } 48 48 + if leases, _ = d.ListMillLeases(); len(leases) != 0 { 49 49 + t.Fatalf("lease survived deletion: %+v", leases) 50 50 + } 51 51 + } 52 52 + 53 53 + func TestExecutorCursors(t *testing.T) { 54 54 + d := newTestDB(t) 55 55 + 56 56 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 5) }); err != nil { 57 57 + t.Fatalf("AdvanceCursor: %v", err) 58 58 + } 59 59 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 9) }); err != nil { 60 60 + t.Fatalf("AdvanceCursor(advance): %v", err) 61 61 + } 62 62 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-2", "inc-1", 1) }); err != nil { 63 63 + t.Fatalf("AdvanceCursor(node-2): %v", err) 64 64 + } 65 65 + 66 66 + cursors, err := d.ListExecutorCursors() 67 67 + if err != nil { 68 68 + t.Fatalf("ListExecutorCursors: %v", err) 69 69 + } 70 70 + if len(cursors) != 2 { 71 71 + t.Fatalf("ListExecutorCursors = %v, want 2 cursors", cursors) 72 72 + } 73 73 + 74 74 + cursorMap := make(map[string]uint64) 75 75 + for _, c := range cursors { 76 76 + cursorMap[c.NodeID+"/"+c.Epoch] = c.AckedSeqno 77 77 + } 78 78 + 79 79 + if cursorMap["node-1/inc-1"] != 9 || cursorMap["node-2/inc-1"] != 1 { 80 80 + t.Fatalf("unexpected cursors: %v", cursorMap) 81 81 + } 82 82 + 83 83 + } 84 84 + 85 85 + func TestCompleteMillLeaseIsAtomic(t *testing.T) { 86 86 + d := newTestDB(t) 87 87 + lease := MillLease{ 88 88 + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 89 89 + Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: "running", 90 90 + } 91 91 + if err := d.SaveMillLease(lease); err != nil { 92 92 + t.Fatalf("SaveMillLease: %v", err) 93 93 + } 94 94 + if _, err := d.Exec(` 95 95 + create trigger reject_mill_lease_delete 96 96 + before delete on mill_leases 97 97 + begin 98 98 + select raise(abort, 'forced delete failure'); 99 99 + end 100 100 + `); err != nil { 101 101 + t.Fatalf("create failure trigger: %v", err) 102 102 + } 103 103 + 104 104 + n := notifier.New() 105 105 + notifications := n.Subscribe() 106 106 + defer n.Unsubscribe(notifications) 107 107 + err := d.CompleteMillLease( 108 108 + "lease-1", 109 109 + "at://knot.example/sh.tangled.pipeline/rkey1", 110 110 + "build", 111 111 + "failed", 112 112 + nil, 113 113 + nil, 114 114 + &n, 115 115 + ) 116 116 + if err == nil { 117 117 + t.Fatal("CompleteMillLease succeeded despite forced lease deletion failure") 118 118 + } 119 119 + var eventCount int 120 120 + if err := d.QueryRow(`select count(*) from events`).Scan(&eventCount); err != nil { 121 121 + t.Fatalf("count events after rollback: %v", err) 122 122 + } 123 123 + if eventCount != 0 { 124 124 + t.Fatalf("terminal event count after rollback = %d, want 0", eventCount) 125 125 + } 126 126 + if leases, listErr := d.ListMillLeases(); listErr != nil || len(leases) != 1 { 127 127 + t.Fatalf("leases after rollback = %+v, err = %v; want original lease", leases, listErr) 128 128 + } 129 129 + select { 130 130 + case <-notifications: 131 131 + t.Fatal("rollback notified event subscribers") 132 132 + default: 133 133 + } 134 134 + 135 135 + if _, err := d.Exec(`drop trigger reject_mill_lease_delete`); err != nil { 136 136 + t.Fatalf("drop failure trigger: %v", err) 137 137 + } 138 138 + if err := d.CompleteMillLease( 139 139 + "lease-1", 140 140 + "at://knot.example/sh.tangled.pipeline/rkey1", 141 141 + "build", 142 142 + "failed", 143 143 + nil, 144 144 + nil, 145 145 + &n, 146 146 + ); err != nil { 147 147 + t.Fatalf("CompleteMillLease retry: %v", err) 148 148 + } 149 149 + if err := d.QueryRow(`select count(*) from events`).Scan(&eventCount); err != nil { 150 150 + t.Fatalf("count committed events: %v", err) 151 151 + } 152 152 + if eventCount != 1 { 153 153 + t.Fatalf("terminal event count after commit = %d, want 1", eventCount) 154 154 + } 155 155 + if leases, listErr := d.ListMillLeases(); listErr != nil || len(leases) != 0 { 156 156 + t.Fatalf("leases after commit = %+v, err = %v; want none", leases, listErr) 157 157 + } 158 158 + select { 159 159 + case <-notifications: 160 160 + default: 161 161 + t.Fatal("committed terminal event did not notify subscribers") 162 162 + } 163 163 + } 164 164 + 165 165 + func TestRestartPersistence(t *testing.T) { 166 166 + dbPath := filepath.Join(t.TempDir(), "persist.db") 167 167 + ctx := context.Background() 168 168 + d, err := Make(ctx, dbPath) 169 169 + if err != nil { 170 170 + t.Fatalf("Make: %v", err) 171 171 + } 172 172 + 173 173 + lease := MillLease{ 174 174 + LeaseID: "lease-p", NodeID: "node-p", Epoch: "inc-p", Engine: "dummy", 175 175 + Knot: "k", Rkey: "r", Workflow: "w", State: "running", 176 176 + } 177 177 + if err := d.SaveMillLease(lease); err != nil { 178 178 + t.Fatalf("SaveMillLease: %v", err) 179 179 + } 180 180 + 181 181 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-p", "inc-p", 42) }); err != nil { 182 182 + t.Fatalf("AdvanceCursor: %v", err) 183 183 + } 184 184 + 185 185 + if err := d.SetOutboxEpoch("inc-p"); err != nil { 186 186 + t.Fatalf("SetOutboxEpoch: %v", err) 187 187 + } 188 188 + if _, err := d.AppendOutboxRow([]byte("hello world"), true); err != nil { 189 189 + t.Fatalf("AppendOutboxRow: %v", err) 190 190 + } 191 191 + 192 192 + if err := d.Close(); err != nil { 193 193 + t.Fatalf("Close: %v", err) 194 194 + } 195 195 + 196 196 + d2, err := Make(ctx, dbPath) 197 197 + if err != nil { 198 198 + t.Fatalf("Make reopen: %v", err) 199 199 + } 200 200 + defer d2.Close() 201 201 + 202 202 + leases, err := d2.ListMillLeases() 203 203 + if err != nil { 204 204 + t.Fatalf("ListMillLeases: %v", err) 205 205 + } 206 206 + if len(leases) != 1 || leases[0].LeaseID != "lease-p" || leases[0].Epoch != "inc-p" { 207 207 + t.Fatalf("unexpected leases: %+v", leases) 208 208 + } 209 209 + 210 210 + cursors, err := d2.ListExecutorCursors() 211 211 + if err != nil { 212 212 + t.Fatalf("ListExecutorCursors: %v", err) 213 213 + } 214 214 + if len(cursors) != 1 || cursors[0].NodeID != "node-p" || cursors[0].Epoch != "inc-p" || cursors[0].AckedSeqno != 42 { 215 215 + t.Fatalf("unexpected cursors: %+v", cursors) 216 216 + } 217 217 + 218 218 + inc, nextSeqno, err := d2.GetOutboxState() 219 219 + if err != nil { 220 220 + t.Fatalf("GetOutboxState: %v", err) 221 221 + } 222 222 + if inc != "inc-p" || nextSeqno != 2 { 223 223 + t.Fatalf("unexpected outbox state: inc=%q, next=%d", inc, nextSeqno) 224 224 + } 225 225 + rows, err := d2.ListOutboxRows() 226 226 + if err != nil { 227 227 + t.Fatalf("ListOutboxRows: %v", err) 228 228 + } 229 229 + if len(rows) != 1 || string(rows[0].Payload) != "hello world" || !rows[0].Control { 230 230 + t.Fatalf("unexpected outbox rows: %+v", rows) 231 231 + } 232 232 + } 233 233 + 234 234 + func TestOutboxPrefixAck(t *testing.T) { 235 235 + d := newTestDB(t) 236 236 + 237 237 + if err := d.SetOutboxEpoch("inc-1"); err != nil { 238 238 + t.Fatalf("SetOutboxEpoch: %v", err) 239 239 + } 240 240 + 241 241 + o1, err := d.AppendOutboxRow([]byte("msg1"), false) 242 242 + if err != nil || o1 != 1 { 243 243 + t.Fatalf("AppendOutboxRow 1: %v, seqno=%d", err, o1) 244 244 + } 245 245 + o2, err := d.AppendOutboxRow([]byte("msg2"), false) 246 246 + if err != nil || o2 != 2 { 247 247 + t.Fatalf("AppendOutboxRow 2: %v, seqno=%d", err, o2) 248 248 + } 249 249 + o3, err := d.AppendOutboxRow([]byte("msg3"), true) 250 250 + if err != nil || o3 != 3 { 251 251 + t.Fatalf("AppendOutboxRow 3: %v, seqno=%d", err, o3) 252 252 + } 253 253 + 254 254 + n, err := d.DeleteOutboxPrefix(2) 255 255 + if err != nil { 256 256 + t.Fatalf("DeleteOutboxPrefix: %v", err) 257 257 + } 258 258 + if n.Rows != 2 || n.Bytes != 8 { 259 259 + t.Fatalf("deleted prefix = %+v, want 2 rows and 8 bytes", n) 260 260 + } 261 261 + 262 262 + rows, err := d.ListOutboxRows() 263 263 + if err != nil { 264 264 + t.Fatalf("ListOutboxRows: %v", err) 265 265 + } 266 266 + if len(rows) != 1 || rows[0].Seqno != 3 || string(rows[0].Payload) != "msg3" { 267 267 + t.Fatalf("expected only msg3 (seqno 3) to remain, got: %+v", rows) 268 268 + } 269 269 + } 270 270 + 271 271 + func TestCompositeCursors(t *testing.T) { 272 272 + d := newTestDB(t) 273 273 + 274 274 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 10) }); err != nil { 275 275 + t.Fatalf("AdvanceCursor: %v", err) 276 276 + } 277 277 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-2", 20) }); err != nil { 278 278 + t.Fatalf("AdvanceCursor: %v", err) 279 279 + } 280 280 + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-2", "inc-1", 5) }); err != nil { 281 281 + t.Fatalf("AdvanceCursor: %v", err) 282 282 + } 283 283 + 284 284 + cursors, err := d.ListExecutorCursors() 285 285 + if err != nil { 286 286 + t.Fatalf("ListExecutorCursors: %v", err) 287 287 + } 288 288 + if len(cursors) != 3 { 289 289 + t.Fatalf("expected 3 cursors, got %d", len(cursors)) 290 290 + } 291 291 + 292 292 + cursorMap := make(map[string]uint64) 293 293 + for _, c := range cursors { 294 294 + key := c.NodeID + "/" + c.Epoch 295 295 + cursorMap[key] = c.AckedSeqno 296 296 + } 297 297 + 298 298 + if cursorMap["node-1/inc-1"] != 10 || cursorMap["node-1/inc-2"] != 20 || cursorMap["node-2/inc-1"] != 5 { 299 299 + t.Fatalf("unexpected cursor values: %v", cursorMap) 300 300 + } 301 301 + 302 302 + } 303 303 + 304 304 + func TestBatchRollback(t *testing.T) { 305 305 + d := newTestDB(t) 306 306 + 307 307 + lease := MillLease{ 308 308 + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 309 309 + Knot: "k", Rkey: "r", Workflow: "w", State: "running", 310 310 + } 311 311 + if err := d.SaveMillLease(lease); err != nil { 312 312 + t.Fatalf("SaveMillLease: %v", err) 313 313 + } 314 314 + 315 315 + n := notifier.New() 316 316 + err := d.ApplyEventBatch(&n, func(tx *EventBatchTx) error { 317 317 + if err := tx.DeleteLease("lease-1"); err != nil { 318 318 + return err 319 319 + } 320 320 + if err := tx.AdvanceCursor("node-1", "inc-1", 100); err != nil { 321 321 + return err 322 322 + } 323 323 + return fmt.Errorf("forced batch failure") 324 324 + }) 325 325 + 326 326 + if err == nil { 327 327 + t.Fatal("expected ApplyEventBatch to return error") 328 328 + } 329 329 + 330 330 + leases, err := d.ListMillLeases() 331 331 + if err != nil { 332 332 + t.Fatalf("ListMillLeases: %v", err) 333 333 + } 334 334 + if len(leases) != 1 { 335 335 + t.Fatalf("lease was deleted despite rollback: %+v", leases) 336 336 + } 337 337 + 338 338 + cursors, err := d.ListExecutorCursors() 339 339 + if err != nil { 340 340 + t.Fatalf("ListExecutorCursors: %v", err) 341 341 + } 342 342 + if len(cursors) != 0 { 343 343 + t.Fatalf("cursor was advanced despite rollback: %+v", cursors) 344 344 + } 345 345 + } 346 346 + 347 347 + func TestTerminalCursorAtomicity(t *testing.T) { 348 348 + d := newTestDB(t) 349 349 + 350 350 + lease := MillLease{ 351 351 + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 352 352 + Knot: "k", Rkey: "r", Workflow: "w", State: "running", 353 353 + } 354 354 + if err := d.SaveMillLease(lease); err != nil { 355 355 + t.Fatalf("SaveMillLease: %v", err) 356 356 + } 357 357 + 358 358 + n := notifier.New() 359 359 + notifications := n.Subscribe() 360 360 + defer n.Unsubscribe(notifications) 361 361 + 362 362 + err := d.ApplyEventBatch(&n, func(tx *EventBatchTx) error { 363 363 + if err := tx.DeleteLease("lease-1"); err != nil { 364 364 + return err 365 365 + } 366 366 + return tx.AdvanceCursor("node-1", "inc-1", 100) 367 367 + }) 368 368 + 369 369 + if err != nil { 370 370 + t.Fatalf("ApplyEventBatch: %v", err) 371 371 + } 372 372 + 373 373 + leases, err := d.ListMillLeases() 374 374 + if err != nil { 375 375 + t.Fatalf("ListMillLeases: %v", err) 376 376 + } 377 377 + if len(leases) != 0 { 378 378 + t.Fatalf("lease not deleted: %+v", leases) 379 379 + } 380 380 + 381 381 + cursors, err := d.ListExecutorCursors() 382 382 + if err != nil { 383 383 + t.Fatalf("ListExecutorCursors: %v", err) 384 384 + } 385 385 + if len(cursors) != 1 || cursors[0].AckedSeqno != 100 { 386 386 + t.Fatalf("cursor not advanced correctly: %+v", cursors) 387 387 + } 388 388 + 389 389 + select { 390 390 + case <-notifications: 391 391 + default: 392 392 + t.Fatal("notifier was not fired after batch commit") 393 393 + } 394 394 + }

+167

spindle/db/mill_tokens.go

Reviewed

··· 1 1 + package db 2 2 + 3 3 + import ( 4 4 + "database/sql" 5 5 + "encoding/json" 6 6 + "slices" 7 7 + "strings" 8 8 + "time" 9 9 + ) 10 10 + 11 11 + // the raw token is never stored, only its hash 12 12 + type ExecutorToken struct { 13 13 + Name string 14 14 + CreatedAt string 15 15 + ExpiresAt *time.Time 16 16 + Labels []string 17 17 + QuarantineReason *string 18 18 + QuarantinedAt *string 19 19 + } 20 20 + 21 21 + func (d *DB) AddExecutorToken(name, tokenHash string, expiresAt *time.Time, labels []string) error { 22 22 + normalized := normalizeLabels(labels) 23 23 + labelsBytes, err := json.Marshal(normalized) 24 24 + if err != nil { 25 25 + return err 26 26 + } 27 27 + _, err = d.Exec( 28 28 + `insert into mill_executors (name, token_hash, expires_at, labels) values (?, ?, ?, ?)`, 29 29 + name, tokenHash, expiryArg(expiresAt), string(labelsBytes), 30 30 + ) 31 31 + return err 32 32 + } 33 33 + 34 34 + func (d *DB) ResolveExecutorToken(tokenHash string) (string, []string, bool, error) { 35 35 + var name string 36 36 + var expires sql.NullString 37 37 + var labelsRaw, quarantineReason sql.NullString 38 38 + err := d.QueryRow( 39 39 + `select name, expires_at, labels, quarantine_reason from mill_executors where token_hash = ?`, tokenHash, 40 40 + ).Scan(&name, &expires, &labelsRaw, &quarantineReason) 41 41 + if err == sql.ErrNoRows { 42 42 + return "", nil, false, nil 43 43 + } 44 44 + if err != nil { 45 45 + return "", nil, false, err 46 46 + } 47 47 + if quarantineReason.Valid { 48 48 + return "", nil, false, nil 49 49 + } 50 50 + exp, hasExpiry, err := parseExpiry(expires) 51 51 + if err != nil { 52 52 + return "", nil, false, err 53 53 + } 54 54 + if hasExpiry && time.Now().After(exp) { 55 55 + return "", nil, false, nil 56 56 + } 57 57 + var labels []string 58 58 + if labelsRaw.Valid && labelsRaw.String != "" { 59 59 + if err := json.Unmarshal([]byte(labelsRaw.String), &labels); err != nil { 60 60 + return "", nil, false, err 61 61 + } 62 62 + } 63 63 + return name, labels, true, nil 64 64 + } 65 65 + 66 66 + func (d *DB) QuarantineExecutor(name, reason string) error { 67 67 + _, err := d.Exec( 68 68 + `update mill_executors 69 69 + set quarantine_reason = ?, quarantined_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') 70 70 + where name = ?`, 71 71 + reason, name, 72 72 + ) 73 73 + return err 74 74 + } 75 75 + 76 76 + func (d *DB) ClearExecutorQuarantine(name string) (bool, error) { 77 77 + res, err := d.Exec( 78 78 + `update mill_executors set quarantine_reason = null, quarantined_at = null where name = ?`, 79 79 + name, 80 80 + ) 81 81 + if err != nil { 82 82 + return false, err 83 83 + } 84 84 + n, err := res.RowsAffected() 85 85 + return n > 0, err 86 86 + } 87 87 + 88 88 + func (d *DB) RevokeExecutorToken(name string) (bool, error) { 89 89 + res, err := d.Exec(`delete from mill_executors where name = ?`, name) 90 90 + if err != nil { 91 91 + return false, err 92 92 + } 93 93 + n, err := res.RowsAffected() 94 94 + return n > 0, err 95 95 + } 96 96 + 97 97 + func (d *DB) ListExecutorTokens() ([]ExecutorToken, error) { 98 98 + rows, err := d.Query(`select name, created_at, expires_at, labels, quarantine_reason, quarantined_at from mill_executors order by name`) 99 99 + if err != nil { 100 100 + return nil, err 101 101 + } 102 102 + defer rows.Close() 103 103 + 104 104 + var out []ExecutorToken 105 105 + for rows.Next() { 106 106 + var t ExecutorToken 107 107 + var expires, labelsRaw, quarantineReason, quarantinedAt sql.NullString 108 108 + if err := rows.Scan(&t.Name, &t.CreatedAt, &expires, &labelsRaw, &quarantineReason, &quarantinedAt); err != nil { 109 109 + return nil, err 110 110 + } 111 111 + exp, hasExpiry, err := parseExpiry(expires) 112 112 + if err != nil { 113 113 + return nil, err 114 114 + } 115 115 + if hasExpiry { 116 116 + t.ExpiresAt = &exp 117 117 + } 118 118 + if labelsRaw.Valid && labelsRaw.String != "" { 119 119 + if err := json.Unmarshal([]byte(labelsRaw.String), &t.Labels); err != nil { 120 120 + return nil, err 121 121 + } 122 122 + } 123 123 + if quarantineReason.Valid { 124 124 + t.QuarantineReason = &quarantineReason.String 125 125 + } 126 126 + if quarantinedAt.Valid { 127 127 + t.QuarantinedAt = &quarantinedAt.String 128 128 + } 129 129 + out = append(out, t) 130 130 + } 131 131 + return out, rows.Err() 132 132 + } 133 133 + 134 134 + func normalizeLabels(labels []string) []string { 135 135 + var out []string 136 136 + seen := make(map[string]bool) 137 137 + for _, l := range labels { 138 138 + trimmed := strings.TrimSpace(l) 139 139 + if trimmed == "" { 140 140 + continue 141 141 + } 142 142 + if !seen[trimmed] { 143 143 + seen[trimmed] = true 144 144 + out = append(out, trimmed) 145 145 + } 146 146 + } 147 147 + slices.Sort(out) 148 148 + return out 149 149 + } 150 150 + 151 151 + func expiryArg(t *time.Time) any { 152 152 + if t == nil { 153 153 + return nil 154 154 + } 155 155 + return t.UTC().Format(time.RFC3339) 156 156 + } 157 157 + 158 158 + func parseExpiry(s sql.NullString) (time.Time, bool, error) { 159 159 + if !s.Valid { 160 160 + return time.Time{}, false, nil 161 161 + } 162 162 + t, err := time.Parse(time.RFC3339, s.String) 163 163 + if err != nil { 164 164 + return time.Time{}, false, err 165 165 + } 166 166 + return t, true, nil 167 167 + }

+272

spindle/db/mill_tokens_test.go

Reviewed

··· 1 1 + package db 2 2 + 3 3 + import ( 4 4 + "slices" 5 5 + "testing" 6 6 + "time" 7 7 + ) 8 8 + 9 9 + func TestAddExecutorTokenRejectsDuplicateName(t *testing.T) { 10 10 + d := newTestDB(t) 11 11 + 12 12 + if err := d.AddExecutorToken("exec-1", "hash-a", nil, nil); err != nil { 13 13 + t.Fatalf("AddExecutorToken: %v", err) 14 14 + } 15 15 + if err := d.AddExecutorToken("exec-1", "hash-b", nil, nil); err == nil { 16 16 + t.Fatal("AddExecutorToken re-registered an existing name; a duplicate must be rejected") 17 17 + } 18 18 + 19 19 + name, _, ok, err := d.ResolveExecutorToken("hash-a") 20 20 + if err != nil { 21 21 + t.Fatalf("ResolveExecutorToken(hash-a): %v", err) 22 22 + } 23 23 + if !ok || name != "exec-1" { 24 24 + t.Fatalf("ResolveExecutorToken(hash-a) = (%q, %v), want (exec-1, true)", name, ok) 25 25 + } 26 26 + if _, _, ok, _ := d.ResolveExecutorToken("hash-b"); ok { 27 27 + t.Fatal("rejected duplicate's token resolved; the failed insert leaked a credential") 28 28 + } 29 29 + } 30 30 + 31 31 + func TestResolveExecutorTokenMissAndHit(t *testing.T) { 32 32 + d := newTestDB(t) 33 33 + 34 34 + name, _, ok, err := d.ResolveExecutorToken("no-such-hash") 35 35 + if err != nil { 36 36 + t.Fatalf("ResolveExecutorToken(miss): %v", err) 37 37 + } 38 38 + if ok || name != "" { 39 39 + t.Fatalf("ResolveExecutorToken(miss) = (%q, %v), want (\"\", false)", name, ok) 40 40 + } 41 41 + 42 42 + if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil { 43 43 + t.Fatalf("AddExecutorToken: %v", err) 44 44 + } 45 45 + 46 46 + name, _, ok, err = d.ResolveExecutorToken("hash-1") 47 47 + if err != nil { 48 48 + t.Fatalf("ResolveExecutorToken(hit): %v", err) 49 49 + } 50 50 + if !ok || name != "exec-1" { 51 51 + t.Fatalf("ResolveExecutorToken(hash-1) = (%q, %v), want (exec-1, true)", name, ok) 52 52 + } 53 53 + 54 54 + if _, _, ok, _ := d.ResolveExecutorToken("hash-unregistered"); ok { 55 55 + t.Fatal("ResolveExecutorToken matched an unregistered hash") 56 56 + } 57 57 + } 58 58 + 59 59 + func TestRevokeExecutorToken(t *testing.T) { 60 60 + d := newTestDB(t) 61 61 + 62 62 + if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil { 63 63 + t.Fatalf("AddExecutorToken: %v", err) 64 64 + } 65 65 + 66 66 + deleted, err := d.RevokeExecutorToken("exec-1") 67 67 + if err != nil { 68 68 + t.Fatalf("RevokeExecutorToken: %v", err) 69 69 + } 70 70 + if !deleted { 71 71 + t.Fatal("RevokeExecutorToken reported no deletion for an existing identity") 72 72 + } 73 73 + 74 74 + if _, _, ok, _ := d.ResolveExecutorToken("hash-1"); ok { 75 75 + t.Fatal("revoked token still resolves; revocation is not enforced") 76 76 + } 77 77 + 78 78 + if deleted, err := d.RevokeExecutorToken("exec-1"); err != nil || deleted { 79 79 + t.Fatalf("RevokeExecutorToken(already-gone) = (%v, %v), want (false, nil)", deleted, err) 80 80 + } 81 81 + 82 82 + if deleted, err := d.RevokeExecutorToken("ghost"); err != nil || deleted { 83 83 + t.Fatalf("RevokeExecutorToken(unknown) = (%v, %v), want (false, nil)", deleted, err) 84 84 + } 85 85 + } 86 86 + 87 87 + func TestExecutorQuarantineIsVisibleAndReversible(t *testing.T) { 88 88 + d := newTestDB(t) 89 89 + if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil { 90 90 + t.Fatalf("AddExecutorToken: %v", err) 91 91 + } 92 92 + if err := d.QuarantineExecutor("exec-1", "missed cancel deadline"); err != nil { 93 93 + t.Fatalf("QuarantineExecutor: %v", err) 94 94 + } 95 95 + if _, _, ok, err := d.ResolveExecutorToken("hash-1"); err != nil || ok { 96 96 + t.Fatalf("ResolveExecutorToken(quarantined) = (ok=%v, err=%v), want (false, nil)", ok, err) 97 97 + } 98 98 + tokens, err := d.ListExecutorTokens() 99 99 + if err != nil { 100 100 + t.Fatalf("ListExecutorTokens: %v", err) 101 101 + } 102 102 + if len(tokens) != 1 || tokens[0].QuarantineReason == nil || *tokens[0].QuarantineReason != "missed cancel deadline" || tokens[0].QuarantinedAt == nil { 103 103 + t.Fatalf("quarantined token not surfaced: %+v", tokens) 104 104 + } 105 105 + if cleared, err := d.ClearExecutorQuarantine("exec-1"); err != nil || !cleared { 106 106 + t.Fatalf("ClearExecutorQuarantine = (%v, %v), want (true, nil)", cleared, err) 107 107 + } 108 108 + if _, _, ok, err := d.ResolveExecutorToken("hash-1"); err != nil || !ok { 109 109 + t.Fatalf("ResolveExecutorToken(cleared) = (ok=%v, err=%v), want (true, nil)", ok, err) 110 110 + } 111 111 + if cleared, err := d.ClearExecutorQuarantine("missing"); err != nil || cleared { 112 112 + t.Fatalf("ClearExecutorQuarantine(missing) = (%v, %v), want (false, nil)", cleared, err) 113 113 + } 114 114 + } 115 115 + 116 116 + func TestListExecutorTokens(t *testing.T) { 117 117 + d := newTestDB(t) 118 118 + 119 119 + tokens, err := d.ListExecutorTokens() 120 120 + if err != nil { 121 121 + t.Fatalf("ListExecutorTokens(empty): %v", err) 122 122 + } 123 123 + if len(tokens) != 0 { 124 124 + t.Fatalf("ListExecutorTokens on empty table = %d rows, want 0", len(tokens)) 125 125 + } 126 126 + 127 127 + // insert out of alphabetical order to test query sorting 128 128 + for _, name := range []string{"charlie", "alice", "bob"} { 129 129 + if err := d.AddExecutorToken(name, "hash-"+name, nil, nil); err != nil { 130 130 + t.Fatalf("AddExecutorToken(%s): %v", name, err) 131 131 + } 132 132 + } 133 133 + 134 134 + tokens, err = d.ListExecutorTokens() 135 135 + if err != nil { 136 136 + t.Fatalf("ListExecutorTokens: %v", err) 137 137 + } 138 138 + want := []string{"alice", "bob", "charlie"} 139 139 + if len(tokens) != len(want) { 140 140 + t.Fatalf("ListExecutorTokens = %d rows, want %d", len(tokens), len(want)) 141 141 + } 142 142 + for i := range want { 143 143 + if tokens[i].Name != want[i] { 144 144 + t.Fatalf("ListExecutorTokens[%d].Name = %q, want %q (ordered by name)", i, tokens[i].Name, want[i]) 145 145 + } 146 146 + } 147 147 + } 148 148 + 149 149 + // expired tokens must fail closed 150 150 + func TestResolveExecutorTokenExpiry(t *testing.T) { 151 151 + cases := []struct { 152 152 + name string 153 153 + seqno time.Duration 154 154 + noExpiry bool 155 155 + wantOK bool 156 156 + }{ 157 157 + {"future expiry resolves", time.Hour, false, true}, 158 158 + {"past expiry fails closed", -time.Hour, false, false}, 159 159 + {"nil expiry never expires", 0, true, true}, 160 160 + } 161 161 + for _, tc := range cases { 162 162 + t.Run(tc.name, func(t *testing.T) { 163 163 + d := newTestDB(t) 164 164 + 165 165 + var expires *time.Time 166 166 + if !tc.noExpiry { 167 167 + exp := time.Now().Add(tc.seqno) 168 168 + expires = &exp 169 169 + } 170 170 + if err := d.AddExecutorToken("exec-1", "hash-1", expires, nil); err != nil { 171 171 + t.Fatalf("AddExecutorToken: %v", err) 172 172 + } 173 173 + 174 174 + name, _, ok, err := d.ResolveExecutorToken("hash-1") 175 175 + if err != nil { 176 176 + t.Fatalf("ResolveExecutorToken: %v", err) 177 177 + } 178 178 + if ok != tc.wantOK { 179 179 + t.Fatalf("ResolveExecutorToken ok = %v, want %v", ok, tc.wantOK) 180 180 + } 181 181 + if tc.wantOK && name != "exec-1" { 182 182 + t.Fatalf("ResolveExecutorToken name = %q, want exec-1", name) 183 183 + } 184 184 + if !tc.wantOK && name != "" { 185 185 + t.Fatalf("ResolveExecutorToken name = %q, want \"\" when failing closed", name) 186 186 + } 187 187 + }) 188 188 + } 189 189 + } 190 190 + 191 191 + func TestResolveExecutorTokenRejectsMalformedExpiry(t *testing.T) { 192 192 + d := newTestDB(t) 193 193 + if _, err := d.Exec( 194 194 + `insert into mill_executors (name, token_hash, expires_at) values (?, ?, ?)`, 195 195 + "exec-1", "hash-1", "not-a-timestamp", 196 196 + ); err != nil { 197 197 + t.Fatalf("insert malformed token: %v", err) 198 198 + } 199 199 + 200 200 + name, _, ok, err := d.ResolveExecutorToken("hash-1") 201 201 + if err == nil { 202 202 + t.Fatal("ResolveExecutorToken accepted a malformed non-NULL expiry") 203 203 + } 204 204 + if ok || name != "" { 205 205 + t.Fatalf("ResolveExecutorToken = (%q, %v, %v), want (\"\", false, error)", name, ok, err) 206 206 + } 207 207 + } 208 208 + 209 209 + // expiry storage is RFC3339 second precision UTC so 210 210 + // round trips compare to the second 211 211 + func TestListExecutorTokensSurfacesExpiry(t *testing.T) { 212 212 + d := newTestDB(t) 213 213 + 214 214 + exp := time.Now().Add(24 * time.Hour) 215 215 + if err := d.AddExecutorToken("expiring", "hash-exp", &exp, nil); err != nil { 216 216 + t.Fatalf("AddExecutorToken(expiring): %v", err) 217 217 + } 218 218 + if err := d.AddExecutorToken("forever", "hash-forever", nil, nil); err != nil { 219 219 + t.Fatalf("AddExecutorToken(forever): %v", err) 220 220 + } 221 221 + 222 222 + tokens, err := d.ListExecutorTokens() 223 223 + if err != nil { 224 224 + t.Fatalf("ListExecutorTokens: %v", err) 225 225 + } 226 226 + 227 227 + got := make(map[string]*time.Time, len(tokens)) 228 228 + for _, tok := range tokens { 229 229 + got[tok.Name] = tok.ExpiresAt 230 230 + } 231 231 + 232 232 + e, present := got["forever"] 233 233 + if !present { 234 234 + t.Fatal("ListExecutorTokens omitted the non-expiring identity") 235 235 + } 236 236 + if e != nil { 237 237 + t.Fatalf("forever.ExpiresAt = %v, want nil (never expires)", e) 238 238 + } 239 239 + 240 240 + e, present = got["expiring"] 241 241 + if !present { 242 242 + t.Fatal("ListExecutorTokens omitted the expiring identity") 243 243 + } 244 244 + if e == nil { 245 245 + t.Fatal("expiring.ExpiresAt = nil, want the stored expiry") 246 246 + } 247 247 + if e.Unix() != exp.Unix() { 248 248 + t.Fatalf("expiring.ExpiresAt = %d (unix), want %d", e.Unix(), exp.Unix()) 249 249 + } 250 250 + } 251 251 + 252 252 + func TestExecutorTokenLabels(t *testing.T) { 253 253 + d := newTestDB(t) 254 254 + 255 255 + labels := []string{" foo ", "bar", " foo", ""} 256 256 + if err := d.AddExecutorToken("exec-1", "hash-1", nil, labels); err != nil { 257 257 + t.Fatalf("AddExecutorToken: %v", err) 258 258 + } 259 259 + 260 260 + name, resolvedLabels, ok, err := d.ResolveExecutorToken("hash-1") 261 261 + if err != nil { 262 262 + t.Fatalf("ResolveExecutorToken: %v", err) 263 263 + } 264 264 + if !ok || name != "exec-1" { 265 265 + t.Fatalf("ResolveExecutorToken: ok=%v, name=%q, want true, exec-1", ok, name) 266 266 + } 267 267 + 268 268 + wantLabels := []string{"bar", "foo"} 269 269 + if !slices.Equal(resolvedLabels, wantLabels) { 270 270 + t.Fatalf("resolved labels = %v, want %v", resolvedLabels, wantLabels) 271 271 + } 272 272 + }

+100 -40

spindle/engine/engine.go

Reviewed

··· 2 2 3 3 import ( 4 4 "context" 5 5 + "crypto/sha256" 6 6 + "encoding/hex" 5 7 "errors" 6 8 "fmt" 9 9 + "io" 7 10 "log/slog" 8 8 - "path/filepath" 11 11 + "os" 9 12 "sync" 13 13 + "time" 10 14 11 15 "tangled.org/core/notifier" 16 16 + "tangled.org/core/spindle/artifactstore" 12 17 "tangled.org/core/spindle/config" 13 18 "tangled.org/core/spindle/db" 14 19 "tangled.org/core/spindle/models" ··· 62 67 } 63 68 } 64 69 65 65 - type workflowFinalizer interface { 66 66 - FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error 70 70 + // the mill streams the executor's real log file into place itself 71 71 + // a local logger here would only write competing lines 72 72 + type workflowLoggerProvider interface { 73 73 + WorkflowLogger(wid models.WorkflowId) models.WorkflowLogger 67 74 } 68 75 69 69 - func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { 76 76 + // for engines that manage status updates outside StartWorkflows 77 77 + type RemoteStatusEngine interface { 78 78 + AuthorsRemoteStatus() 79 79 + } 80 80 + 81 81 + func reportWorkflowStatusError(l *slog.Logger, database *db.DB, n *notifier.Notifier, wid models.WorkflowId, err error) { 82 82 + if errors.Is(err, ErrTimedOut) { 83 83 + dbErr := database.StatusTimeout(wid, n) 84 84 + if dbErr != nil { 85 85 + l.Error("failed to set workflow status to timeout", "wid", wid, "err", dbErr) 86 86 + } 87 87 + } else if errors.Is(err, ErrWorkflowCanceled) { 88 88 + dbErr := database.StatusCancelled(wid, err.Error(), -1, n) 89 89 + if dbErr != nil { 90 90 + l.Error("failed to set workflow status to cancelled", "wid", wid, "err", dbErr) 91 91 + } 92 92 + } else { 93 93 + dbErr := database.StatusFailed(wid, err.Error(), -1, n) 94 94 + if dbErr != nil { 95 95 + l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr) 96 96 + } 97 97 + } 98 98 + } 99 99 + 100 100 + func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, stores *artifactstore.Stores, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { 70 101 l.Info("starting all workflows in parallel", "pipeline", pipelineId) 71 102 72 103 var allSecrets []secrets.UnlockedSecret ··· 84 115 secretValues[i] = s.Value 85 116 } 86 117 87 87 - s3, err := NewS3(cfg.S3.LogBucket) 88 88 - if err != nil { 89 89 - l.Error("error creating s3 client", "err", err) 90 90 - } 91 91 - 92 118 // wid.String() is lossy so two different names can map to the same key 93 119 // eg. "foo bar" and "foo-bar"... 94 120 wfCounts := make(map[string]int) ··· 101 127 wfCounts[wid.String()]++ 102 128 } 103 129 } 104 104 - 105 130 var wg sync.WaitGroup 106 131 for eng, wfs := range pipeline.Workflows { 107 132 workflowTimeout := eng.WorkflowTimeout() ··· 128 153 l.Info("skipping finished workflow", "wid", wid, "status", st.Status) 129 154 return 130 155 } 131 131 - defer func() { 132 132 - if s3 != nil { 133 133 - logFile := filepath.Join(cfg.Server.LogDir, fmt.Sprintf("%s.log", wid.String())) 134 134 - if err := s3.WriteFile(ctx, logFile); err != nil { 135 135 - l.Error("error uploading logs", "err", err) 136 136 - } 137 137 - } 138 138 - }() 139 139 - 140 140 - wfLogger, err := models.NewFileWorkflowLogger(cfg.Server.LogDir, wid, secretValues) 141 141 - if err != nil { 156 156 + var err error 157 157 + var wfLogger models.WorkflowLogger 158 158 + if p, ok := eng.(workflowLoggerProvider); ok { 159 159 + wfLogger = p.WorkflowLogger(wid) 160 160 + } else if fileLogger, err := models.NewFileWorkflowLogger(cfg.Server.LogDir, wid, secretValues); err != nil { 142 161 l.Warn("failed to setup step logger; logs will not be persisted", "error", err) 143 162 wfLogger = models.NullLogger{} 144 163 } else { 145 164 l.Info("setup step logger; logs will be persisted", "logDir", cfg.Server.LogDir, "wid", wid) 146 146 - defer wfLogger.Close() 165 165 + wfLogger = fileLogger 166 166 + defer archiveWorkflowLog(l, stores, db, cfg.Server.LogDir, wid) 167 167 + defer fileLogger.Close() 147 168 } 148 169 149 170 timeoutCtx, timeoutCancel := context.WithTimeout(ctx, workflowTimeout) ··· 164 185 165 186 l.Info("waiting for slot", "wid", wid) 166 187 slot := WorkflowSlot(NoopSlot{}) 188 188 + _, remoteStatus := eng.(RemoteStatusEngine) 189 189 + 167 190 if s, ok := eng.(WorkflowSlotter); ok { 168 168 - slot, err = s.AcquireWorkflowSlot(wfCtx, wid, &w) 191 191 + slot, err = s.AcquireWorkflowSlot(wfCtx, wid, &w, Wait) 169 192 if err != nil { 170 193 writeWfError(db, n, l, wfCtx, wid, "waiting for slot", err) 171 194 return ··· 173 196 } 174 197 defer slot.Release() 175 198 176 176 - err = db.StatusRunning(wid, n) 177 177 - if err != nil { 178 178 - l.Error("failed to set workflow status to running", "wid", wid, "err", err) 179 179 - return 199 199 + if !remoteStatus { 200 200 + err := db.StatusRunning(wid, n) 201 201 + if err != nil { 202 202 + l.Error("failed to set workflow status to running", "wid", wid, "err", err) 203 203 + return 204 204 + } 180 205 } 181 206 182 207 err = eng.SetupWorkflow(wfCtx, wid, &w, wfLogger) ··· 186 211 l.Error("failed to destroy workflow after setup failure", "error", destroyErr) 187 212 } 188 213 } 189 189 - writeWfError(db, n, l, wfCtx, wid, "setting up workflow", err) 214 214 + if !remoteStatus { 215 215 + writeWfError(db, n, l, wfCtx, wid, "setting up workflow", err) 216 216 + } 190 217 return 191 218 } 192 219 defer eng.DestroyWorkflow(ctx, wid) ··· 207 234 } 208 235 209 236 if err != nil { 210 210 - writeWfError(db, n, l, wfCtx, wid, "running step", err) 211 211 - return 212 212 - } 213 213 - } 214 214 - 215 215 - if finalizer, ok := eng.(workflowFinalizer); ok { 216 216 - if err := finalizer.FinalizeWorkflow(wfCtx, wid, &w, wfLogger); err != nil { 217 217 - writeWfError(db, n, l, wfCtx, wid, "finalizing", err) 237 237 + if !remoteStatus { 238 238 + writeWfError(db, n, l, wfCtx, wid, "running step", err) 239 239 + } 218 240 return 219 241 } 220 242 } 221 243 222 244 if isCanceled(wfCtx) { 223 223 - writeWfError(db, n, l, wfCtx, wid, "before success", nil) 245 245 + if !remoteStatus { 246 246 + writeWfError(db, n, l, wfCtx, wid, "before success", nil) 247 247 + } 224 248 return 225 249 } 226 250 227 227 - err = db.StatusSuccess(wid, n) 228 228 - if err != nil { 229 229 - l.Error("failed to set workflow status to success", "wid", wid, "err", err) 251 251 + if !remoteStatus { 252 252 + err = db.StatusSuccess(wid, n) 253 253 + if err != nil { 254 254 + l.Error("failed to set workflow status to success", "wid", wid, "err", err) 255 255 + } 230 256 } 231 257 }) 232 258 } ··· 235 261 wg.Wait() 236 262 l.Info("all workflows completed") 237 263 } 264 264 + 265 265 + func archiveWorkflowLog(l *slog.Logger, stores *artifactstore.Stores, database *db.DB, logDir string, wid models.WorkflowId) { 266 266 + if stores == nil { 267 267 + return 268 268 + } 269 269 + logPath := models.LogFilePath(logDir, wid) 270 270 + file, err := os.Open(logPath) 271 271 + if err != nil { 272 272 + l.Error("open workflow log for archival", "wid", wid, "err", err) 273 273 + return 274 274 + } 275 275 + hash := sha256.New() 276 276 + if _, err := io.Copy(hash, file); err != nil { 277 277 + _ = file.Close() 278 278 + l.Error("hash workflow log", "wid", wid, "err", err) 279 279 + return 280 280 + } 281 281 + _ = file.Close() 282 282 + 283 283 + ref := wid.String() + ".log" 284 284 + uploadCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) 285 285 + defer cancel() 286 286 + errs := stores.PutFile(uploadCtx, ref, logPath) 287 287 + for _, err := range errs { 288 288 + l.Error("archive workflow log", "wid", wid, "err", err) 289 289 + } 290 290 + if len(errs) == len(stores.Names()) { 291 291 + return 292 292 + } 293 293 + digest := "sha256:" + hex.EncodeToString(hash.Sum(nil)) 294 294 + if err := database.SaveArtifactRef(wid.String(), wid.Name, ref, digest); err != nil { 295 295 + l.Error("save workflow log artifact", "wid", wid, "err", err) 296 296 + } 297 297 + }

+3 -3

spindle/engine/engine_test.go

Reviewed

··· 114 114 } 115 115 116 116 cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} 117 117 - StartWorkflows(logger, nil, cfg, testDB, nil, context.Background(), pipeline, pipelineId) 117 117 + StartWorkflows(logger, nil, cfg, nil, testDB, nil, context.Background(), pipeline, pipelineId) 118 118 119 119 eng.mu.Lock() 120 120 setupCalls := append([]models.WorkflowId(nil), eng.setupCalls...) ··· 195 195 cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} 196 196 doneChan := make(chan struct{}) 197 197 go func() { 198 198 - StartWorkflows(logger, nil, cfg, testDB, nil, context.Background(), pipeline, pipelineId) 198 198 + StartWorkflows(logger, nil, cfg, nil, testDB, nil, context.Background(), pipeline, pipelineId) 199 199 close(doneChan) 200 200 }() 201 201 ··· 250 250 } 251 251 252 252 cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} 253 253 - StartWorkflows(logger, nil, cfg, testDB, nil, context.Background(), pipeline, pipelineId) 253 253 + StartWorkflows(logger, nil, cfg, nil, testDB, nil, context.Background(), pipeline, pipelineId) 254 254 255 255 st, err := testDB.GetStatus(wid) 256 256 if err != nil {

+8

spindle/engine/manifest_test.go

Reviewed

··· 79 79 } 80 80 } 81 81 82 82 + func TestDescribeManifestErrorAcceptsRunsOnGenericWorkflowKey(t *testing.T) { 83 83 + raw := "engine: microvm\nruns_on: [linux/arm64, kvm]\nimage: nixos\n" 84 84 + 85 85 + if err := DescribeManifestError(raw, testManifest{}); err != nil { 86 86 + t.Fatalf("DescribeManifestError(%q) = %v, want nil", raw, err) 87 87 + } 88 88 + } 89 89 + 82 90 func TestDescribeManifestErrorNoFalsePositives(t *testing.T) { 83 91 cases := []string{ 84 92 // well-formed manifest

+10

spindle/engine/placement.go

Reviewed

··· 1 1 + package engine 2 2 + 3 3 + import ( 4 4 + "tangled.org/core/spindle/models" 5 5 + ) 6 6 + 7 7 + // checked before an executor accepts and holds a remote lease 8 8 + type WorkflowPlacementValidator interface { 9 9 + ValidateWorkflowPlacement(wf *models.Workflow) error 10 10 + }

-75

spindle/engine/s3.go

Reviewed

··· 1 1 - package engine 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "fmt" 6 6 - "io" 7 7 - "os" 8 8 - "path/filepath" 9 9 - 10 10 - "github.com/aws/aws-sdk-go-v2/aws" 11 11 - "github.com/aws/aws-sdk-go-v2/config" 12 12 - "github.com/aws/aws-sdk-go-v2/service/s3" 13 13 - ) 14 14 - 15 15 - type S3 struct { 16 16 - bucket string 17 17 - client *s3.Client 18 18 - } 19 19 - 20 20 - const BaseS3Path = "spindle/workflows" 21 21 - 22 22 - func NewS3(bucket string) (*S3, error) { 23 23 - if bucket == "" { 24 24 - return nil, fmt.Errorf("s3 bucket not provided") 25 25 - } 26 26 - 27 27 - ctx := context.Background() 28 28 - sdkConfig, err := config.LoadDefaultConfig(ctx) 29 29 - 30 30 - if err != nil { 31 31 - return nil, fmt.Errorf("error loading s3 config: %w", err) 32 32 - } 33 33 - s3Client := s3.NewFromConfig(sdkConfig) 34 34 - 35 35 - return &S3{ 36 36 - bucket: bucket, 37 37 - client: s3Client, 38 38 - }, nil 39 39 - } 40 40 - 41 41 - func (s *S3) WriteFile(ctx context.Context, path string) error { 42 42 - s3Key := fmt.Sprintf("%s/%s", BaseS3Path, filepath.Base(path)) 43 43 - 44 44 - file, err := os.Open(path) 45 45 - if err != nil { 46 46 - return fmt.Errorf("error opening file %s: %w", path, err) 47 47 - } 48 48 - defer file.Close() 49 49 - 50 50 - _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ 51 51 - Bucket: &s.bucket, 52 52 - Key: &s3Key, 53 53 - Body: file, 54 54 - }) 55 55 - 56 56 - if err != nil { 57 57 - return fmt.Errorf("error writing to s3: %w", err) 58 58 - } 59 59 - 60 60 - return nil 61 61 - } 62 62 - 63 63 - func (s *S3) ReadFile(ctx context.Context, name string) ([]byte, error) { 64 64 - res, err := s.client.GetObject(ctx, &s3.GetObjectInput{ 65 65 - Bucket: &s.bucket, 66 66 - Key: aws.String(name), 67 67 - }) 68 68 - 69 69 - if err != nil { 70 70 - return nil, fmt.Errorf("error reading file %s: %w", name, err) 71 71 - } 72 72 - defer res.Body.Close() 73 73 - 74 74 - return io.ReadAll(res.Body) 75 75 - }

+12 -8

spindle/engine/scheduler.go

Reviewed

··· 50 50 } 51 51 } 52 52 53 53 - func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R) (WorkflowSlot, error) { 53 53 + // the mill owns the backlog so a NoWait caller must have room immediately or fail 54 54 + func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R, mode AcquireMode) (WorkflowSlot, error) { 54 55 if s == nil { 55 56 return NoopSlot{}, nil 56 57 } ··· 60 61 s.mu.Unlock() 61 62 return nil, fmt.Errorf("%w: request=%v budget=%v max=%v", ErrNoWorkflowSlots, req, s.budget, s.max) 62 63 } 63 63 - if len(s.queue) == 0 && s.used.Add(req).Fits(s.budget) { 64 64 + // NoWait ignores the queue because it never blocks 65 65 + // Wait only bypasses empty queues to prevent starvation 66 66 + if s.used.Add(req).Fits(s.budget) && (mode == NoWait || len(s.queue) == 0) { 64 67 s.used = s.used.Add(req) 65 68 s.mu.Unlock() 66 69 return &resourceLease[R]{scheduler: s, req: req}, nil 67 70 } 71 71 + if mode == NoWait { 72 72 + used := s.used 73 73 + s.mu.Unlock() 74 74 + return nil, fmt.Errorf("%w: request=%v used=%v budget=%v", ErrNoWorkflowSlots, req, used, s.budget) 75 75 + } 68 76 69 77 waiter := &resourceWaiter[R]{req: req, ready: make(chan struct{}), enqueuedAt: s.now()} 70 78 s.queue = append(s.queue, waiter) ··· 108 116 109 117 // start every waiter whose request fits. once a waiter is older than 110 118 // agingThreshold, count its request as already used so younger waiters 111 111 - // stop being scheduled ahead of it. 119 119 + // stop being scheduled ahead of it 112 120 func (s *ResourceScheduler[R]) schedule() { 113 121 var reserved R 114 122 now := s.now() ··· 129 137 } 130 138 131 139 func (s *ResourceScheduler[R]) remove(waiter *resourceWaiter[R]) { 132 132 - for i, candidate := range s.queue { 133 133 - if candidate != waiter { 134 134 - continue 135 135 - } 140 140 + if i := slices.Index(s.queue, waiter); i >= 0 { 136 141 s.queue = slices.Delete(s.queue, i, i+1) 137 137 - return 138 142 } 139 143 }

+75 -9

spindle/engine/scheduler_test.go

Reviewed

··· 36 36 37 37 scheduler := NewResourceScheduler(ru{}, ru{}, 0) 38 38 39 39 - slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}) 39 39 + slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}, Wait) 40 40 if err != nil { 41 41 t.Fatalf("Acquire() error = %v", err) 42 42 } ··· 48 48 49 49 scheduler := NewResourceScheduler(ru{a: 1024, b: 10_000}, ru{a: 512, b: 5_000}, 0) 50 50 51 51 - _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}) 51 51 + _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}, Wait) 52 52 if !errors.Is(err, ErrNoWorkflowSlots) { 53 53 t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err) 54 54 } 55 55 56 56 - _, err = scheduler.Acquire(context.Background(), ru{a: 128, b: 12_000}) 56 56 + _, err = scheduler.Acquire(context.Background(), ru{a: 128, b: 12_000}, Wait) 57 57 if !errors.Is(err, ErrNoWorkflowSlots) { 58 58 t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err) 59 59 } ··· 64 64 65 65 scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 0) 66 66 67 67 - first, err := scheduler.Acquire(context.Background(), ru{a: 1024}) 67 67 + first, err := scheduler.Acquire(context.Background(), ru{a: 1024}, Wait) 68 68 if err != nil { 69 69 t.Fatalf("first Acquire() error = %v", err) 70 70 } ··· 85 85 86 86 scheduler := NewResourceScheduler(ru{a: 1}, ru{}, 0) 87 87 88 88 - slot, err := scheduler.Acquire(context.Background(), ru{a: 1}) 88 88 + slot, err := scheduler.Acquire(context.Background(), ru{a: 1}, Wait) 89 89 if err != nil { 90 90 t.Fatalf("Acquire() error = %v", err) 91 91 } ··· 93 93 slot.Release() 94 94 slot.Release() 95 95 96 96 - second, err := scheduler.Acquire(context.Background(), ru{a: 1}) 96 96 + second, err := scheduler.Acquire(context.Background(), ru{a: 1}, Wait) 97 97 if err != nil { 98 98 t.Fatalf("Acquire() after double release error = %v", err) 99 99 } ··· 105 105 106 106 scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, time.Hour) // disable aging so we test pure backfill 107 107 108 108 - hold, err := scheduler.Acquire(context.Background(), ru{a: 512}) 108 108 + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait) 109 109 if err != nil { 110 110 t.Fatalf("hold Acquire() error = %v", err) 111 111 } ··· 128 128 fakeNow := time.Now() 129 129 scheduler.now = func() time.Time { return fakeNow } 130 130 131 131 - hold, err := scheduler.Acquire(context.Background(), ru{a: 512}) 131 131 + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait) 132 132 if err != nil { 133 133 t.Fatalf("hold Acquire() error = %v", err) 134 134 } ··· 151 151 big.Release() 152 152 } 153 153 154 154 + func TestResourceSchedulerTryRejectsWhenNoRoomNow(t *testing.T) { 155 155 + t.Parallel() 156 156 + 157 157 + scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 0) 158 158 + 159 159 + first, err := scheduler.Acquire(context.Background(), ru{a: 1024}, NoWait) 160 160 + if err != nil { 161 161 + t.Fatalf("first Acquire(NoWait) error = %v", err) 162 162 + } 163 163 + 164 164 + if _, err := scheduler.Acquire(context.Background(), ru{a: 1}, NoWait); !errors.Is(err, ErrNoWorkflowSlots) { 165 165 + t.Fatalf("Acquire(NoWait) error = %v, want ErrNoWorkflowSlots", err) 166 166 + } 167 167 + 168 168 + first.Release() 169 169 + 170 170 + second, err := scheduler.Acquire(context.Background(), ru{a: 1}, NoWait) 171 171 + if err != nil { 172 172 + t.Fatalf("Acquire(NoWait) after release error = %v", err) 173 173 + } 174 174 + second.Release() 175 175 + } 176 176 + 177 177 + func TestResourceSchedulerTryRejectsRequestsThatCanNeverFit(t *testing.T) { 178 178 + t.Parallel() 179 179 + 180 180 + scheduler := NewResourceScheduler(ru{a: 1024, b: 10_000}, ru{a: 512, b: 5_000}, 0) 181 181 + 182 182 + if _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}, NoWait); !errors.Is(err, ErrNoWorkflowSlots) { 183 183 + t.Fatalf("Acquire(NoWait) error = %v, want ErrNoWorkflowSlots", err) 184 184 + } 185 185 + } 186 186 + 187 187 + func TestResourceSchedulerTryIgnoresQueuedWaiters(t *testing.T) { 188 188 + t.Parallel() 189 189 + 190 190 + scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, time.Hour) 191 191 + 192 192 + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait) 193 193 + if err != nil { 194 194 + t.Fatalf("hold Acquire() error = %v", err) 195 195 + } 196 196 + defer hold.Release() 197 197 + 198 198 + bigCh := acquireAsync(context.Background(), scheduler, ru{a: 768}) 199 199 + assertAcquireBlocked(t, bigCh) 200 200 + 201 201 + slot, err := scheduler.Acquire(context.Background(), ru{a: 256}, NoWait) 202 202 + if err != nil { 203 203 + t.Fatalf("Acquire(NoWait) error = %v, want success past queued waiter", err) 204 204 + } 205 205 + slot.Release() 206 206 + } 207 207 + 208 208 + func TestResourceSchedulerZeroLimitsTryDoesNotReject(t *testing.T) { 209 209 + t.Parallel() 210 210 + 211 211 + scheduler := NewResourceScheduler(ru{}, ru{}, 0) 212 212 + 213 213 + slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}, NoWait) 214 214 + if err != nil { 215 215 + t.Fatalf("Acquire(NoWait) error = %v", err) 216 216 + } 217 217 + slot.Release() 218 218 + } 219 219 + 154 220 func acquireAsync(ctx context.Context, scheduler *ResourceScheduler[ru], req ru) <-chan acquireResult { 155 221 ch := make(chan acquireResult, 1) 156 222 go func() { 157 157 - slot, err := scheduler.Acquire(ctx, req) 223 223 + slot, err := scheduler.Acquire(ctx, req, Wait) 158 224 ch <- acquireResult{slot: slot, err: err} 159 225 }() 160 226 return ch

+21 -2

spindle/engine/slot.go

Reviewed

··· 13 13 Release() 14 14 } 15 15 16 16 + // governs blocking behaviour when acquiring a slot 17 17 + type AcquireMode int 18 18 + 19 19 + const ( 20 20 + // blocks until a slot is free 21 21 + Wait AcquireMode = iota 22 22 + // fails immediately if full 23 23 + // executors use this because the mill owns the backlog 24 24 + NoWait 25 25 + ) 26 26 + 16 27 type WorkflowSlotter interface { 17 17 - AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) 28 28 + AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode AcquireMode) (WorkflowSlot, error) 18 29 } 19 30 20 31 type releaseFunc func() ··· 41 52 return &SemaphoreSlotter{slots: make(chan struct{}, maxConcurrent)} 42 53 } 43 54 44 44 - func (a *SemaphoreSlotter) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) { 55 55 + func (a *SemaphoreSlotter) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode AcquireMode) (WorkflowSlot, error) { 45 56 if a == nil || a.slots == nil { 46 57 return NoopSlot{}, nil 47 58 } 59 59 + if mode == NoWait { 60 60 + select { 61 61 + case a.slots <- struct{}{}: 62 62 + return releaseFunc(func() { <-a.slots }), nil 63 63 + default: 64 64 + return nil, ErrNoWorkflowSlots 65 65 + } 66 66 + } 48 67 select { 49 68 case a.slots <- struct{}{}: 50 69 return releaseFunc(func() { <-a.slots }), nil

+42 -5

spindle/engine/slot_test.go

Reviewed

··· 15 15 slotter := NewSemaphoreSlotter(0) 16 16 17 17 for range 10 { 18 18 - slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) 18 18 + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) 19 19 if err != nil { 20 20 t.Fatalf("AcquireWorkflowSlot() error = %v", err) 21 21 } ··· 28 28 29 29 slotter := NewSemaphoreSlotter(1) 30 30 31 31 - first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) 31 31 + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) 32 32 if err != nil { 33 33 t.Fatalf("first AcquireWorkflowSlot() error = %v", err) 34 34 } ··· 42 42 acquired := make(chan WorkflowSlot, 1) 43 43 errs := make(chan error, 1) 44 44 go func() { 45 45 - slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) 45 45 + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) 46 46 if err != nil { 47 47 errs <- err 48 48 return ··· 64 64 65 65 slotter := NewSemaphoreSlotter(1) 66 66 67 67 - first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) 67 67 + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) 68 68 if err != nil { 69 69 t.Fatalf("first AcquireWorkflowSlot() error = %v", err) 70 70 } ··· 73 73 ctx, cancel := context.WithCancel(context.Background()) 74 74 cancel() 75 75 76 76 - _, err = slotter.AcquireWorkflowSlot(ctx, zeroWorkflowID(), nil) 76 76 + _, err = slotter.AcquireWorkflowSlot(ctx, zeroWorkflowID(), nil, Wait) 77 77 if !errors.Is(err, context.Canceled) { 78 78 t.Fatalf("AcquireWorkflowSlot() error = %v, want context.Canceled", err) 79 79 } 80 80 } 81 81 82 82 + func TestSemaphoreSlotterTryDisabledDoesNotReject(t *testing.T) { 83 83 + t.Parallel() 84 84 + 85 85 + slotter := NewSemaphoreSlotter(0) 86 86 + 87 87 + for range 10 { 88 88 + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait) 89 89 + if err != nil { 90 90 + t.Fatalf("AcquireWorkflowSlot(NoWait) error = %v", err) 91 91 + } 92 92 + slot.Release() 93 93 + } 94 94 + } 95 95 + 96 96 + func TestSemaphoreSlotterTryRejectsWhenFull(t *testing.T) { 97 97 + t.Parallel() 98 98 + 99 99 + slotter := NewSemaphoreSlotter(1) 100 100 + 101 101 + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait) 102 102 + if err != nil { 103 103 + t.Fatalf("first AcquireWorkflowSlot(NoWait) error = %v", err) 104 104 + } 105 105 + 106 106 + if _, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait); !errors.Is(err, ErrNoWorkflowSlots) { 107 107 + t.Fatalf("AcquireWorkflowSlot(NoWait) error = %v, want ErrNoWorkflowSlots", err) 108 108 + } 109 109 + 110 110 + first.Release() 111 111 + 112 112 + second, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait) 113 113 + if err != nil { 114 114 + t.Fatalf("AcquireWorkflowSlot(NoWait) after release error = %v", err) 115 115 + } 116 116 + second.Release() 117 117 + } 118 118 + 82 119 func assertNotAcquired(t *testing.T, acquired <-chan WorkflowSlot, errs <-chan error) { 83 120 t.Helper() 84 121

+16 -2

spindle/engines/dummy/engine.go

Reviewed

··· 8 8 9 9 "gopkg.in/yaml.v3" 10 10 "tangled.org/core/api/tangled" 11 11 + "tangled.org/core/spindle/engine" 11 12 "tangled.org/core/spindle/models" 12 13 "tangled.org/core/spindle/secrets" 13 14 ) ··· 16 17 // step output to the workflow logger. Useful for testing pipeline plumbing 17 18 // without a real execution backend. 18 19 type DummyEngine struct { 19 19 - l *slog.Logger 20 20 + l *slog.Logger 21 21 + StepDelay time.Duration 20 22 } 21 23 22 24 func New(l *slog.Logger) *DummyEngine { ··· 79 81 return 5 * time.Minute 80 82 } 81 83 84 84 + // no capacity limit, so always a no-op slot regardless of mode 85 85 + func (e *DummyEngine) AcquireWorkflowSlot(_ context.Context, _ models.WorkflowId, _ *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) { 86 86 + return engine.NoopSlot{}, nil 87 87 + } 88 88 + 82 89 func (e *DummyEngine) DestroyWorkflow(_ context.Context, wid models.WorkflowId) error { 83 90 e.l.Info("destroying workflow", "wid", wid) 84 91 return nil 85 92 } 86 93 87 87 - func (e *DummyEngine) RunStep(_ context.Context, wid models.WorkflowId, w *models.Workflow, idx int, _ []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { 94 94 + func (e *DummyEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, _ []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { 88 95 step := w.Steps[idx] 89 96 e.l.Info("running step", "wid", wid, "step", step.Name(), "command", step.Command()) 90 97 fmt.Fprintf(wfLogger.DataWriter(idx, "stdout"), "$ %s", step.Command()) 98 98 + if e.StepDelay > 0 { 99 99 + select { 100 100 + case <-ctx.Done(): 101 101 + return ctx.Err() 102 102 + case <-time.After(e.StepDelay): 103 103 + } 104 104 + } 91 105 return nil 92 106 }

+2 -5

spindle/engines/microvm/budget.go

Reviewed

··· 68 68 return budget, maxReq, cfg.AgingThreshold 69 69 } 70 70 71 71 - func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (engine.WorkflowSlot, error) { 71 71 + func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode engine.AcquireMode) (engine.WorkflowSlot, error) { 72 72 state, ok := wf.Data.(*workflowState) 73 73 if !ok || state == nil { 74 74 return nil, fmt.Errorf("microVM workflow state is not initialized") ··· 77 77 return engine.NoopSlot{}, nil 78 78 } 79 79 req := resourcesForImage(state.ImageSpec) 80 80 - if req.MemoryMiB < 0 || req.VCPUs < 0 || req.DiskMiB < 0 { 81 81 - return nil, fmt.Errorf("microVM resource request must not be negative: %s", req) 82 82 - } 83 83 - return e.scheduler.Acquire(ctx, req) 80 80 + return e.scheduler.Acquire(ctx, req, mode) 84 81 } 85 82 86 83 func resourcesForImage(spec ImageSpec) Resources {

+4 -4

spindle/engines/microvm/engine.go

Reviewed

··· 52 52 agent *agentHub 53 53 scheduler *engine.ResourceScheduler[Resources] 54 54 cgroupParent *CgroupParent 55 55 + budget Resources 56 56 + maxWorkflow Resources 55 57 56 58 cleanupMu sync.Mutex 57 59 cleanup map[string][]cleanupFunc ··· 91 93 db: d, 92 94 scheduler: engine.NewResourceScheduler(budget, max, agingThreshold), 93 95 cgroupParent: cgroupParent, 96 96 + budget: budget, 97 97 + maxWorkflow: max, 94 98 cleanup: make(map[string][]cleanupFunc), 95 99 }, nil 96 100 } ··· 561 565 return cleanupErr 562 566 } 563 567 564 564 - func (e *Engine) FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, w *models.Workflow, wfLogger models.WorkflowLogger) error { 565 565 - return nil 566 566 - } 567 567 - 568 568 func (e *Engine) WorkflowTimeout() time.Duration { 569 569 d, err := time.ParseDuration(e.cfg.MicroVMPipelines.WorkflowTimeout) 570 570 if err != nil {

+14 -13

spindle/engines/microvm/image.go

Reviewed

··· 240 240 } 241 241 return "", false, err 242 242 } 243 243 - if !info.IsDir() { 244 244 - return candidate, true, nil 243 243 + if info.IsDir() { 244 244 + candidate = filepath.Join(candidate, imageSpecFileName) 245 245 + info, err = os.Stat(candidate) 246 246 + if err != nil { 247 247 + if errors.Is(err, os.ErrNotExist) { 248 248 + return "", false, fmt.Errorf("microVM image directory %q does not contain %s", filepath.Dir(candidate), imageSpecFileName) 249 249 + } 250 250 + return "", false, err 251 251 + } 252 252 + if info.IsDir() { 253 253 + return "", false, fmt.Errorf("microVM image spec %q is a directory", candidate) 254 254 + } 245 255 } 246 256 247 247 - spec := filepath.Join(candidate, imageSpecFileName) 248 248 - info, err = os.Stat(spec) 257 257 + path, err := filepath.EvalSymlinks(candidate) 249 258 if err != nil { 250 250 - if errors.Is(err, os.ErrNotExist) { 251 251 - return "", false, fmt.Errorf("microVM image directory %q does not contain %s", candidate, imageSpecFileName) 252 252 - } 253 259 return "", false, err 254 260 } 255 255 - // this only happens if there is a directory named `spec.json` which would be very silly. 256 256 - // but better output an error for it anyway :p 257 257 - if info.IsDir() { 258 258 - return "", false, fmt.Errorf("microVM image spec %q is a directory", spec) 259 259 - } 260 260 - return spec, true, nil 261 261 + return path, true, nil 261 262 }

+58

spindle/engines/microvm/image_test.go

Reviewed

··· 65 65 }) 66 66 } 67 67 } 68 68 + func TestImageSpecPathPinsSymlinkTarget(t *testing.T) { 69 69 + dir := t.TempDir() 70 70 + imageA := filepath.Join(dir, "image-a") 71 71 + imageB := filepath.Join(dir, "image-b") 72 72 + writeSpecFile(t, filepath.Join(imageA, imageSpecFileName)) 73 73 + writeSpecFile(t, filepath.Join(imageB, imageSpecFileName)) 74 74 + 75 75 + alias := filepath.Join(dir, "default") 76 76 + if err := os.Symlink(imageA, alias); err != nil { 77 77 + t.Fatal(err) 78 78 + } 79 79 + got, ok, err := imageSpecPath(alias) 80 80 + if err != nil { 81 81 + t.Fatal(err) 82 82 + } 83 83 + if !ok { 84 84 + t.Fatal("imageSpecPath() did not resolve symlinked image") 85 85 + } 86 86 + 87 87 + if err := os.Remove(alias); err != nil { 88 88 + t.Fatal(err) 89 89 + } 90 90 + if err := os.Symlink(imageB, alias); err != nil { 91 91 + t.Fatal(err) 92 92 + } 93 93 + want := filepath.Join(imageA, imageSpecFileName) 94 94 + if got != want { 95 95 + t.Fatalf("imageSpecPath() = %q after alias resolution, want pinned target %q", got, want) 96 96 + } 97 97 + } 68 98 69 99 func TestResolveImageDirectoryMissingSpec(t *testing.T) { 70 100 dir := t.TempDir() ··· 135 165 t.Fatalf("spec without shell should fail validation, got: %v", err) 136 166 } 137 167 } 168 168 + func TestMkfsExt4ForVolumesSkipsLookupWithoutVolumes(t *testing.T) { 169 169 + t.Setenv("PATH", "") 170 170 + path, err := mkfsExt4ForVolumes(nil, "") 171 171 + if err != nil { 172 172 + t.Fatalf("mkfsExt4ForVolumes() with no volumes returned error: %v", err) 173 173 + } 174 174 + if path != "" { 175 175 + t.Fatalf("mkfsExt4ForVolumes() = %q with no volumes, want empty path", path) 176 176 + } 177 177 + 178 178 + _, err = mkfsExt4ForVolumes([]Volume{{Image: "workspace"}}, "") 179 179 + if err == nil || !strings.Contains(err.Error(), "mkfs.ext4") { 180 180 + t.Fatalf("mkfsExt4ForVolumes() with a volume and no formatter returned %v", err) 181 181 + } 182 182 + } 183 183 + 184 184 + func TestQEMURunnerValidateRejectsUnsupportedNetworkTypeBeforeHostChecks(t *testing.T) { 185 185 + spec := validImageSpec() 186 186 + spec.NetworkInterfaces = []NetworkInterface{{ 187 187 + Type: "tap", 188 188 + ID: "net0", 189 189 + MAC: "02:00:00:00:00:01", 190 190 + }} 191 191 + err := (qemuRunner{}).Validate(spec, false) 192 192 + if err == nil || !strings.Contains(err.Error(), `unsupported microvm network interface type "tap"`) { 193 193 + t.Fatalf("Validate() error = %v, want unsupported network type", err) 194 194 + } 195 195 + }

+15

spindle/engines/microvm/placement/placement.go

Reviewed

··· 1 1 + package placement 2 2 + 3 3 + func IsNativeArchitecture(imageArch, goArch string) bool { 4 4 + normalize := func(arch string) string { 5 5 + switch arch { 6 6 + case "x86_64", "amd64": 7 7 + return "amd64" 8 8 + case "aarch64", "arm64": 9 9 + return "arm64" 10 10 + default: 11 11 + return arch 12 12 + } 13 13 + } 14 14 + return imageArch != "" && normalize(imageArch) == normalize(goArch) 15 15 + }

+26

spindle/engines/microvm/placement/placement_test.go

Reviewed

··· 1 1 + package placement 2 2 + 3 3 + import ( 4 4 + "testing" 5 5 + ) 6 6 + 7 7 + func TestIsNativeArchitecture(t *testing.T) { 8 8 + tests := []struct { 9 9 + image string 10 10 + host string 11 11 + want bool 12 12 + }{ 13 13 + {image: "x86_64", host: "amd64", want: true}, 14 14 + {image: "amd64", host: "amd64", want: true}, 15 15 + {image: "aarch64", host: "arm64", want: true}, 16 16 + {image: "arm64", host: "arm64", want: true}, 17 17 + {image: "x86_64", host: "arm64", want: false}, 18 18 + {image: "aarch64", host: "amd64", want: false}, 19 19 + {image: "", host: "amd64", want: false}, 20 20 + } 21 21 + for _, tt := range tests { 22 22 + if got := IsNativeArchitecture(tt.image, tt.host); got != tt.want { 23 23 + t.Errorf("IsNativeArchitecture(%q, %q) = %v, want %v", tt.image, tt.host, got, tt.want) 24 24 + } 25 25 + } 26 26 + }

+61

spindle/engines/microvm/placement_linux.go

Reviewed

··· 1 1 + //go:build linux 2 2 + 3 3 + package microvm 4 4 + 5 5 + import ( 6 6 + "fmt" 7 7 + "os/exec" 8 8 + "runtime" 9 9 + 10 10 + "tangled.org/core/spindle/engines/microvm/placement" 11 11 + "tangled.org/core/spindle/models" 12 12 + ) 13 13 + 14 14 + func (e *Engine) ValidateWorkflowPlacement(wf *models.Workflow) error { 15 15 + state, ok := wf.Data.(*workflowState) 16 16 + if !ok || state == nil { 17 17 + return fmt.Errorf("microVM workflow state is not initialized") 18 18 + } 19 19 + return e.validateImagePlacement(state.ImageSpec) 20 20 + } 21 21 + 22 22 + func (e *Engine) validateImagePlacement(spec ImageSpec) error { 23 23 + if err := spec.Validate(); err != nil { 24 24 + return err 25 25 + } 26 26 + if !placement.IsNativeArchitecture(spec.Arch, runtime.GOARCH) { 27 27 + return fmt.Errorf("microVM image architecture %q is not native to executor architecture %q", spec.Arch, runtime.GOARCH) 28 28 + } 29 29 + if err := spec.validateImageFiles(); err != nil { 30 30 + return err 31 31 + } 32 32 + runner, err := runnerFor(spec.RunnerType) 33 33 + if err != nil { 34 34 + return err 35 35 + } 36 36 + if err := runner.Validate(spec, e.cfg.MicroVMPipelines.EnableKVM); err != nil { 37 37 + return err 38 38 + } 39 39 + if len(spec.Volumes) > 0 { 40 40 + if _, err := exec.LookPath("mkfs.ext4"); err != nil { 41 41 + return fmt.Errorf("required host command %q not found in PATH: %w", "mkfs.ext4", err) 42 42 + } 43 43 + for _, volume := range spec.Volumes { 44 44 + if volume.ReadOnly { 45 45 + return fmt.Errorf("read-only microvm volume %q is not supported yet", volume.Image) 46 46 + } 47 47 + if volume.FSType != "ext4" { 48 48 + return fmt.Errorf("microvm volume %q uses unsupported fsType %q", volume.Image, volume.FSType) 49 49 + } 50 50 + if volume.ImageType != "" && volume.ImageType != "raw" { 51 51 + return fmt.Errorf("microvm volume %q uses unsupported imageType %q", volume.Image, volume.ImageType) 52 52 + } 53 53 + } 54 54 + } 55 55 + 56 56 + request := resourcesForImage(spec) 57 57 + if !request.Fits(e.budget) || !request.Fits(e.maxWorkflow) { 58 58 + return fmt.Errorf("microVM image resources exceed executor limits: request=%v budget=%v max=%v", request, e.budget, e.maxWorkflow) 59 59 + } 60 60 + return nil 61 61 + }

+4

spindle/engines/microvm/qemu.go

Reviewed

··· 73 73 type qemuRunner struct{} 74 74 75 75 func (qemuRunner) Validate(spec ImageSpec, enableKVM bool) error { 76 76 + b := newArgBuilder(len(spec.NetworkInterfaces) * 4) 77 77 + if err := addQEMUNetworkArgs(&b, spec.NetworkInterfaces); err != nil { 78 78 + return err 79 79 + } 76 80 if _, err := exec.LookPath(spec.RunnerCmd()); err != nil { 77 81 return fmt.Errorf("required host command %q not found in PATH: %w", spec.RunnerCmd(), err) 78 82 }

+14 -6

spindle/engines/microvm/vm.go

Reviewed

··· 47 47 return nil 48 48 } 49 49 50 50 + func mkfsExt4ForVolumes(volumes []Volume, configured string) (string, error) { 51 51 + if len(volumes) == 0 || configured != "" { 52 52 + return configured, nil 53 53 + } 54 54 + path, err := exec.LookPath("mkfs.ext4") 55 55 + if err != nil { 56 56 + return "", fmt.Errorf("mkfs.ext4 command not found in PATH: %w", err) 57 57 + } 58 58 + return path, nil 59 59 + } 60 60 + 50 61 func prepareVolumes(ctx context.Context, workDir string, volumes []Volume, mkfsExt4 string) (map[string]string, error) { 51 62 paths := make(map[string]string, len(volumes)) 52 63 for _, volume := range volumes { ··· 361 372 return nil, err 362 373 } 363 374 364 364 - mkfsExt4 := cfg.MkfsExt4 365 365 - if mkfsExt4 == "" { 366 366 - mkfsExt4, err = exec.LookPath("mkfs.ext4") 367 367 - if err != nil { 368 368 - return nil, fmt.Errorf("mkfs.ext4 command not found in PATH: %w", err) 369 369 - } 375 375 + mkfsExt4, err := mkfsExt4ForVolumes(cfg.Image.Volumes, cfg.MkfsExt4) 376 376 + if err != nil { 377 377 + return nil, err 370 378 } 371 379 volumePaths, err := prepareVolumes(ctx, cfg.WorkDir, cfg.Image.Volumes, mkfsExt4) 372 380 if err != nil {

+2 -1

spindle/engines/nixery/engine.go

Reviewed

··· 198 198 ctx context.Context, 199 199 wid models.WorkflowId, 200 200 wf *models.Workflow, 201 201 + mode engine.AcquireMode, 201 202 ) (engine.WorkflowSlot, error) { 202 203 if e.slotter == nil { 203 204 return engine.NoopSlot{}, nil 204 205 } 205 206 206 206 - return e.slotter.AcquireWorkflowSlot(ctx, wid, wf) 207 207 + return e.slotter.AcquireWorkflowSlot(ctx, wid, wf, mode) 207 208 } 208 209 209 210 func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) (err error) {

+52

spindle/logview/logview.go

Reviewed

··· 1 1 + package logview 2 2 + 3 3 + import ( 4 4 + "bufio" 5 5 + "context" 6 6 + "io" 7 7 + "strings" 8 8 + 9 9 + "github.com/hpcloud/tail" 10 10 + "tangled.org/core/spindle/artifactstore" 11 11 + "tangled.org/core/spindle/db" 12 12 + "tangled.org/core/spindle/models" 13 13 + ) 14 14 + 15 15 + // streams a workflow's log lines: tails the log file while running, reads 16 16 + // the uploaded artifact once finished, same for every role. stop ends a 17 17 + // live follow early, the channel closes when the source drains or ctx ends 18 18 + func Follow(ctx context.Context, d *db.DB, reader artifactstore.Reader, logDir string, wid models.WorkflowId, finished bool) (<-chan *tail.Line, func(), error) { 19 19 + if finished && reader != nil && d != nil { 20 20 + if fl, err := d.GetFinishedLog(wid.Name); err == nil && fl.Ref != "" { 21 21 + rc, err := reader.Open(ctx, fl.Ref) 22 22 + if err == nil { 23 23 + ch := make(chan *tail.Line, 64) 24 24 + followCtx, cancel := context.WithCancel(ctx) 25 25 + go func() { 26 26 + defer close(ch) 27 27 + defer rc.Close() 28 28 + scanner := bufio.NewScanner(rc) 29 29 + for scanner.Scan() { 30 30 + select { 31 31 + case <-followCtx.Done(): 32 32 + return 33 33 + case ch <- &tail.Line{Text: strings.TrimSuffix(scanner.Text(), "\r")}: 34 34 + } 35 35 + } 36 36 + }() 37 37 + return ch, cancel, nil 38 38 + } 39 39 + } 40 40 + } 41 41 + 42 42 + t, err := tail.TailFile(models.LogFilePath(logDir, wid), tail.Config{ 43 43 + Follow: !finished, 44 44 + ReOpen: !finished, 45 45 + MustExist: false, 46 46 + Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}, 47 47 + }) 48 48 + if err != nil { 49 49 + return nil, nil, err 50 50 + } 51 51 + return t.Lines, func() { _ = t.Stop() }, nil 52 52 + }

+159

spindle/mill/README.md

Reviewed

··· 1 1 + # spindle mill 2 2 + 3 3 + This document describes the architecture of the mill, spindle's distributed 4 4 + job placement layer. A mill is a spindle that doesn't run jobs itself: it 5 5 + places them on remote executors, which run the real engines (microvm, nixery, 6 6 + dummy) exactly as they would standalone. 7 7 + 8 8 + ## The engine mirroring model 9 9 + 10 10 + Neither side's execution path changes between local and remote runs. 11 11 + 12 12 + On the mill host, engines are registered under the real engine names 13 13 + ("microvm", "nixery", "dummy"), but each is a stand-in that places jobs 14 14 + remotely instead of running them. `InitWorkflow` on the stand-in parses 15 15 + nothing, it stashes the raw pipeline and workflow and builds a synthetic 16 16 + one-step workflow, so everything upstream (trigger handling, pending status, 17 17 + `TANGLED_*` env) behaves exactly like a local run. The real `InitWorkflow` 18 18 + runs exactly once, on the executor. 19 19 + 20 20 + On the executor, the reserve handler wraps the real engine in a 21 21 + `reservedEngine`: the slot acquired when the reservation is accepted is the 22 22 + same slot `StartWorkflows` later runs on. Nothing acquires twice, and from the 23 23 + engine's point of view the job is indistinguishable from a local one. 24 24 + 25 25 + The same mirroring applies to output: the executor's jobs write status rows 26 26 + and log lines exactly like a standalone spindle, and the mill re-authors them 27 27 + into its own event stream and log store, so appview and the log endpoints see 28 28 + no difference between a local and a remote job. 29 29 + 30 30 + The only real divergence: secrets are withheld from untrusted pipelines at the 31 31 + mill, and they cross the wire exactly once, inside `CommitLease`, to the one 32 32 + node that won the bid. Losing bidders never see them. 33 33 + 34 34 + ## Placement: labels, seats, and rejection 35 35 + 36 36 + Placement decides which executors can run a workflow right now, using one 37 37 + kind of fact: 38 38 + 39 39 + - **labels** are operator-defined strings on the executor's token, checked 40 40 + against the workflow's `runs_on`. they're for coarse fleet partitioning 41 41 + ("this pool is for trusted jobs", "this box has a gpu"), nothing more 42 42 + 43 43 + Matching is exact intersection: candidate sessions whose snapshot has the 44 44 + engine available, a free seat, and labels covering `runs_on`. Among the 45 45 + eligible candidates the mill ranks least-loaded first and bids the top-K 46 46 + concurrently with `ReserveSeat`. The best-ranked accept wins, losers get 47 47 + `ReleaseLease` so they free their held seats immediately. If nobody 48 48 + accepts, the job waits for a change: a new executor connecting, a snapshot 49 49 + flipping availability, or a lease finishing somewhere. 50 50 + 51 51 + Image and arch compatibility is deliberately *not* a mill concern. The mill 52 52 + never parses an image name or an arch string; the executor re-validates 53 53 + everything at reserve time against its own disk (engine exists, workflow 54 54 + parses, the image spec validates and is natively runnable, the runner is 55 55 + usable) and rejects what it can't run with `incompatible`. A reject rotates 56 56 + the bid to the next candidate, and if every candidate rejects, the user 57 57 + gets the collected reasons as the placement error. Multi-arch falls out of 58 58 + this: an arm64 box simply cannot accept an x86_64 image, so it never holds 59 59 + one. If you *want* to pin an arch or an image explicitly, label the nodes 60 60 + and use `runs_on`. 61 61 + 62 62 + ### Example: an alpine microvm job 63 63 + 64 64 + Say a workflow asks for the microvm engine with `image: alpine`, no 65 65 + `runs_on`, and the fleet looks like this: 66 66 + 67 67 + | node | arch | labels | engines | load | 68 68 + |---------|---------|---------|-------------------|--------| 69 69 + | ci-1 | x86_64 | [linux] | microvm | busy | 70 70 + | ci-2 | aarch64 | [linux] | microvm | idle | 71 71 + | ci-3 | x86_64 | [linux] | microvm, nixery | idle | 72 72 + | ci-4 | x86_64 | [gpu] | microvm | idle | 73 73 + 74 74 + The walkthrough: 75 75 + 76 76 + ```mermaid 77 77 + flowchart TD 78 78 + W["workflow<br/><small>engine: microvm, image: alpine</small>"] --> F{"candidate filter"} 79 79 + F -->|"all pass runs_on (empty)"| L{"has microvm<br/>available?"} 80 80 + L -->|all four| R{"rank by load"} 81 81 + R -->|"ci-1 busy"| X2["ci-1 ranked last"] 82 82 + R --> B["bid top-K: ci-2, ci-4, ci-1"] 83 83 + ``` 84 84 + 85 85 + Say ci-2's alpine image is actually x86_64-only: the mill offers the bid 86 86 + anyway, ci-2's reserve-time validation rejects it as incompatible, and the 87 87 + bid rotates to ci-4. The mill learns "ci-2 can't run alpine" from the 88 88 + rejection, not from any advertisement, and the reason reaches the user if 89 89 + every candidate fails the same way. 90 90 + 91 91 + The bid then runs concurrently: 92 92 + 93 93 + ```mermaid 94 94 + sequenceDiagram 95 95 + participant M as mill 96 96 + participant C2 as ci-2 97 97 + participant C4 as ci-4 98 98 + participant C1 as ci-1 99 99 + 100 100 + par bids 101 101 + M->>C2: ReserveSeat (raw pipeline+workflow) 102 102 + M->>C4: ReserveSeat 103 103 + M->>C1: ReserveSeat 104 104 + end 105 105 + C2-->>M: accept (idle) 106 106 + C4-->>M: accept 107 107 + C1-->>M: reject (transient, seats full) 108 108 + Note over M: ci-2 ranked above ci-4,<br/>ci-4 gets ReleaseLease 109 109 + M->>C2: CommitLease (secrets) 110 110 + C2-->>M: Committed 111 111 + M->>C4: ReleaseLease 112 112 + ``` 113 113 + 114 114 + Both idle nodes accepted, so rank breaks the tie: ci-2 keeps the seat, ci-4 115 115 + frees its immediately, and only ci-2 ever sees the secrets. From here ci-2 116 116 + runs the job exactly like a standalone spindle would, booting the alpine 117 117 + image under QEMU, while the mill blocks on the terminal event. 118 118 + 119 119 + ## The protocol's durability model 120 120 + 121 121 + Executor to mill is a single websocket per node, and everything the executor 122 122 + reports (status, logs, terminal results) travels as sequenced `Event`s. Two 123 123 + identifiers keep it all consistent: 124 124 + 125 125 + - the **epoch** names one lifetime of the executor process. a restarted 126 126 + executor connects with a fresh epoch, and anything arriving for an old one 127 127 + is invalid. leases are bound to node+epoch, so a zombie from a previous 128 128 + process can't act on them 129 129 + - the **seqno** is a dense per-epoch counter on events. the executor persists 130 130 + events in a local outbox before sending and trims it only when the mill 131 131 + acks. on reconnect it resumes from the mill's acked position and replays. 132 132 + the mill applies idempotently: replays drop, gaps kill the session 133 133 + 134 134 + The mill is equally restartable: leases, acked positions and the canonical 135 135 + log all live in its db. Restored leases start as orphans and must be 136 136 + reclaimed by the executor's first snapshot, or a sweep fails them after one 137 137 + grace window. The invariant throughout: at any moment, for any lease, exactly 138 138 + one of {executor outbox, mill db} holds the newest state, and the seqno/epoch 139 139 + pair says who. 140 140 + 141 141 + ## Leases 142 142 + 143 143 + A lease is the mill-side handle for one placed job: 144 144 + 145 145 + ```mermaid 146 146 + stateDiagram-v2 147 147 + [*] --> reserved: bid won 148 148 + reserved --> committing: CommitLease sent 149 149 + committing --> running: Committed 150 150 + running --> done: terminal arrived 151 151 + reserved --> done: released / expired 152 152 + committing --> done: released / expired 153 153 + ``` 154 154 + 155 155 + Commit retries ride reconnects: a reservation outlives one disconnect, so a 156 156 + lost session means wait and retry, not a failed job. What ends a job is the 157 157 + job timeout, a terminal event, or the executor being declared dead after 158 158 + reconnect grace expires, at which point every lease the node held fails 159 159 + (preserving a pending cancellation as the reason).

+482

spindle/mill/auth_test.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "io" 6 6 + "log/slog" 7 7 + "net/http" 8 8 + "net/http/httptest" 9 9 + "path/filepath" 10 10 + "strings" 11 11 + "testing" 12 12 + "time" 13 13 + 14 14 + "github.com/gorilla/websocket" 15 15 + "tangled.org/core/notifier" 16 16 + "tangled.org/core/spindle/db" 17 17 + "tangled.org/core/spindle/models" 18 18 + 19 19 + millproto "tangled.org/core/spindle/mill/proto" 20 20 + millv1 "tangled.org/core/spindle/mill/proto/gen" 21 21 + ) 22 22 + 23 23 + func discardLogger() *slog.Logger { 24 24 + return slog.New(slog.NewTextHandler(io.Discard, nil)) 25 25 + } 26 26 + 27 27 + func nopEncoder() scriptedEncoder { 28 28 + return scriptedEncoder(func(*millproto.Message) error { return nil }) 29 29 + } 30 30 + 31 31 + func TestHashToken(t *testing.T) { 32 32 + const raw = "super-secret-executor-token" 33 33 + 34 34 + if HashToken(raw) != HashToken(raw) { 35 35 + t.Fatal("HashToken is not deterministic; the same token would stop authenticating") 36 36 + } 37 37 + if HashToken("token-a") == HashToken("token-b") { 38 38 + t.Fatal("HashToken collided two distinct tokens") 39 39 + } 40 40 + if HashToken(raw) == raw { 41 41 + t.Fatal("HashToken returned the raw token; a hash leak would expose a usable credential") 42 42 + } 43 43 + } 44 44 + 45 45 + func TestGenerateTokenDistinct(t *testing.T) { 46 46 + const n = 100 47 47 + seen := make(map[string]struct{}, n) 48 48 + for i := range n { 49 49 + tok, err := GenerateToken() 50 50 + if err != nil { 51 51 + t.Fatalf("GenerateToken: %v", err) 52 52 + } 53 53 + if tok == "" { 54 54 + t.Fatalf("GenerateToken returned an empty token on call %d", i) 55 55 + } 56 56 + if _, dup := seen[tok]; dup { 57 57 + t.Fatalf("GenerateToken repeated a token after %d calls: %q", i, tok) 58 58 + } 59 59 + seen[tok] = struct{}{} 60 60 + } 61 61 + } 62 62 + 63 63 + func TestAttachSessionRejectsSecondLiveSession(t *testing.T) { 64 64 + l := discardLogger() 65 65 + m := New(l, Config{ReconnectGrace: time.Minute}) 66 66 + 67 67 + sessionOf := func(node string) *millSession { 68 68 + m.mu.Lock() 69 69 + defer m.mu.Unlock() 70 70 + return m.sessions[node] 71 71 + } 72 72 + 73 73 + sess1 := newSession("node-1", "inc-1", nil, nopEncoder(), l) 74 74 + if _, ok := m.attachSession(sess1); !ok { 75 75 + t.Fatal("first attach of a node was rejected; want accept") 76 76 + } 77 77 + if sessionOf("node-1") != sess1 { 78 78 + t.Fatal("first session was not registered as the live session") 79 79 + } 80 80 + 81 81 + sess2 := newSession("node-1", "inc-2", nil, nopEncoder(), l) 82 82 + if _, ok := m.attachSession(sess2); ok { 83 83 + t.Fatal("second live attach for an already-live node was accepted; a valid token hijacked the executor") 84 84 + } 85 85 + if sessionOf("node-1") != sess1 { 86 86 + t.Fatal("rejected newcomer evicted the incumbent session") 87 87 + } 88 88 + 89 89 + m.detachSession(sess1) 90 90 + sess3 := newSession("node-1", "inc-3", nil, nopEncoder(), l) 91 91 + if _, ok := m.attachSession(sess3); !ok { 92 92 + t.Fatal("attach during the incumbent's reconnect grace was rejected; want adopt") 93 93 + } 94 94 + if sessionOf("node-1") != sess3 { 95 95 + t.Fatal("adopted session was not installed as the live session") 96 96 + } 97 97 + } 98 98 + 99 99 + func TestOnAttemptResultIgnoresForeignLease(t *testing.T) { 100 100 + ctx := context.Background() 101 101 + l := discardLogger() 102 102 + bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 103 103 + if err != nil { 104 104 + t.Fatalf("db.Make: %v", err) 105 105 + } 106 106 + t.Cleanup(func() { bdb.Close() }) 107 107 + n := notifier.New() 108 108 + 109 109 + m := New(l, Config{ReconnectGrace: time.Minute}) 110 110 + m.Attach(bdb, &n) 111 111 + 112 112 + foreign := newLease("lease-foreign", "node-a", "inc-a", "dummy") 113 113 + m.mu.Lock() 114 114 + m.leases[foreign.id] = foreign 115 115 + m.mu.Unlock() 116 116 + 117 117 + sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l) 118 118 + m.attachSession(sessB) 119 119 + _ = m.onEventBatch(sessB, &millv1.EventBatch{ 120 120 + Epoch: sessB.epoch, 121 121 + Events: []*millv1.Event{ 122 122 + { 123 123 + Seqno: 1, 124 124 + LeaseId: foreign.id, 125 125 + Payload: &millv1.Event_AttemptResult{ 126 126 + AttemptResult: &millv1.AttemptResult{ 127 127 + Status: millv1.TerminalStatus_SUCCESS, 128 128 + }, 129 129 + }, 130 130 + }, 131 131 + }, 132 132 + }) 133 133 + 134 134 + if _, ok := pollTerminal(foreign); ok { 135 135 + t.Fatal("attempt-result on a foreign lease delivered a terminal; an executor forged another node's job result") 136 136 + } 137 137 + if foreign.getState() == leaseDone { 138 138 + t.Fatal("attempt-result on a foreign lease sealed the lease") 139 139 + } 140 140 + } 141 141 + 142 142 + func TestOnAttemptResultIgnoresAbsentLease(t *testing.T) { 143 143 + ctx := context.Background() 144 144 + l := discardLogger() 145 145 + bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 146 146 + if err != nil { 147 147 + t.Fatalf("db.Make: %v", err) 148 148 + } 149 149 + t.Cleanup(func() { bdb.Close() }) 150 150 + n := notifier.New() 151 151 + 152 152 + m := New(l, Config{ReconnectGrace: time.Minute}) 153 153 + m.Attach(bdb, &n) 154 154 + 155 155 + // bystander lease the reporting node owns, proving an absent-lease stream 156 156 + // does not spill onto another lease 157 157 + bystander := newLease("lease-bystander", "node-b", "inc-b", "dummy") 158 158 + m.mu.Lock() 159 159 + m.leases[bystander.id] = bystander 160 160 + m.mu.Unlock() 161 161 + 162 162 + sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l) 163 163 + m.attachSession(sessB) 164 164 + 165 165 + _ = m.onEventBatch(sessB, &millv1.EventBatch{ 166 166 + Epoch: sessB.epoch, 167 167 + Events: []*millv1.Event{ 168 168 + { 169 169 + Seqno: 1, 170 170 + LeaseId: "lease-nonexistent", 171 171 + Payload: &millv1.Event_AttemptResult{ 172 172 + AttemptResult: &millv1.AttemptResult{ 173 173 + Status: millv1.TerminalStatus_SUCCESS, 174 174 + }, 175 175 + }, 176 176 + }, 177 177 + }, 178 178 + }) 179 179 + 180 180 + if _, ok := pollTerminal(bystander); ok { 181 181 + t.Fatal("attempt-result for an absent lease delivered a terminal to a bystander lease") 182 182 + } 183 183 + if bystander.getState() == leaseDone { 184 184 + t.Fatal("attempt-result for an absent lease sealed a bystander lease") 185 185 + } 186 186 + } 187 187 + 188 188 + // owned-lease path proves ignore tests above are not passing merely because 189 189 + // delivery is broken. correctly owned terminal is delivered 190 190 + func TestOnAttemptResultDeliversOwnedLease(t *testing.T) { 191 191 + ctx := context.Background() 192 192 + l := discardLogger() 193 193 + bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 194 194 + if err != nil { 195 195 + t.Fatalf("db.Make: %v", err) 196 196 + } 197 197 + t.Cleanup(func() { bdb.Close() }) 198 198 + n := notifier.New() 199 199 + 200 200 + m := New(l, Config{ReconnectGrace: time.Minute}) 201 201 + m.Attach(bdb, &n) 202 202 + 203 203 + owned := newLease("lease-owned", "node-b", "inc-b", "dummy") 204 204 + m.mu.Lock() 205 205 + m.leases[owned.id] = owned 206 206 + m.mu.Unlock() 207 207 + 208 208 + sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l) 209 209 + m.attachSession(sessB) 210 210 + _ = m.onEventBatch(sessB, &millv1.EventBatch{ 211 211 + Epoch: sessB.epoch, 212 212 + Events: []*millv1.Event{ 213 213 + { 214 214 + Seqno: 1, 215 215 + LeaseId: owned.id, 216 216 + Payload: &millv1.Event_AttemptResult{ 217 217 + AttemptResult: &millv1.AttemptResult{ 218 218 + Status: millv1.TerminalStatus_SUCCESS, 219 219 + }, 220 220 + }, 221 221 + }, 222 222 + }, 223 223 + }) 224 224 + 225 225 + res, ok := pollTerminal(owned) 226 226 + if !ok { 227 227 + t.Fatal("attempt-result on an owned lease was not delivered") 228 228 + } 229 229 + if got := res.GetStatus(); got != millv1.TerminalStatus_SUCCESS { 230 230 + t.Fatalf("delivered terminal status = %v, want %v", got, millv1.TerminalStatus_SUCCESS) 231 231 + } 232 232 + if owned.getState() != leaseDone { 233 233 + t.Fatal("owned lease was not sealed after its terminal was delivered") 234 234 + } 235 235 + } 236 236 + 237 237 + func TestOnStatusEventOwnership(t *testing.T) { 238 238 + ctx := context.Background() 239 239 + l := discardLogger() 240 240 + 241 241 + bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 242 242 + if err != nil { 243 243 + t.Fatalf("db.Make: %v", err) 244 244 + } 245 245 + t.Cleanup(func() { bdb.Close() }) 246 246 + n := notifier.New() 247 247 + 248 248 + m := New(l, Config{ReconnectGrace: time.Minute}) 249 249 + m.Attach(bdb, &n) 250 250 + 251 251 + foreign := newLease("lease-foreign", "node-x", "inc-x", "dummy") 252 252 + foreign.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "foreign"}, Name: "build"} 253 253 + owned := newLease("lease-owned", "node-z", "inc-z", "dummy") 254 254 + owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "owned"}, Name: "build"} 255 255 + m.mu.Lock() 256 256 + m.leases[foreign.id] = foreign 257 257 + m.leases[owned.id] = owned 258 258 + m.mu.Unlock() 259 259 + 260 260 + sessY := newSession("node-y", "inc-y", nil, nopEncoder(), l) 261 261 + m.attachSession(sessY) 262 262 + sessZ := newSession("node-z", "inc-z", nil, nopEncoder(), l) 263 263 + m.attachSession(sessZ) 264 264 + 265 265 + _ = m.onEventBatch(sessY, &millv1.EventBatch{ 266 266 + Epoch: sessY.epoch, 267 267 + Events: []*millv1.Event{ 268 268 + { 269 269 + Seqno: 1, 270 270 + LeaseId: foreign.id, 271 271 + Payload: &millv1.Event_StatusEvent{ 272 272 + StatusEvent: &millv1.StatusEvent{ 273 273 + Status: millv1.NonterminalStatus_RUNNING, 274 274 + }, 275 275 + }, 276 276 + }, 277 277 + }, 278 278 + }) 279 279 + if _, err := bdb.GetStatus(foreign.wid); err == nil { 280 280 + t.Fatal("status stream for a foreign lease authored a status row; an executor forged another pipeline's status") 281 281 + } 282 282 + 283 283 + _ = m.onEventBatch(sessZ, &millv1.EventBatch{ 284 284 + Epoch: sessZ.epoch, 285 285 + Events: []*millv1.Event{ 286 286 + { 287 287 + Seqno: 1, 288 288 + LeaseId: owned.id, 289 289 + Payload: &millv1.Event_StatusEvent{ 290 290 + StatusEvent: &millv1.StatusEvent{ 291 291 + Status: millv1.NonterminalStatus_RUNNING, 292 292 + }, 293 293 + }, 294 294 + }, 295 295 + }, 296 296 + }) 297 297 + st, err := bdb.GetStatus(owned.wid) 298 298 + if err != nil { 299 299 + t.Fatalf("owned status stream did not author a status row: %v", err) 300 300 + } 301 301 + if st.Status != "running" { 302 302 + t.Fatalf("owned status = %q, want %q", st.Status, "running") 303 303 + } 304 304 + } 305 305 + 306 306 + func setupTestServer(t *testing.T, authorizedLabels []string) (*Mill, *db.DB, *httptest.Server, string) { 307 307 + ctx := context.Background() 308 308 + bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 309 309 + if err != nil { 310 310 + t.Fatalf("db.Make: %v", err) 311 311 + } 312 312 + t.Cleanup(func() { bdb.Close() }) 313 313 + n := notifier.New() 314 314 + 315 315 + m := New(discardLogger(), Config{ 316 316 + ReconnectGrace: time.Minute, 317 317 + }) 318 318 + m.Attach(bdb, &n) 319 319 + 320 320 + const secret = "test-secret" 321 321 + if err := bdb.AddExecutorToken("dev-node", HashToken(secret), nil, authorizedLabels); err != nil { 322 322 + t.Fatalf("AddExecutorToken: %v", err) 323 323 + } 324 324 + 325 325 + server := httptest.NewServer(http.HandlerFunc(m.HandleExecutorConn)) 326 326 + t.Cleanup(server.Close) 327 327 + 328 328 + return m, bdb, server, secret 329 329 + } 330 330 + 331 331 + func TestAuthLabelEscalation(t *testing.T) { 332 332 + _, _, server, secret := setupTestServer(t, []string{"linux", "amd64"}) 333 333 + 334 334 + wsUrl := "ws" + strings.TrimPrefix(server.URL, "http") 335 335 + 336 336 + { 337 337 + header := http.Header{} 338 338 + header.Set("Authorization", "Bearer bad-token") 339 339 + _, resp, err := websocket.DefaultDialer.Dial(wsUrl, header) 340 340 + if err == nil { 341 341 + t.Fatal("expected connection with invalid token to fail") 342 342 + } 343 343 + if resp != nil && resp.StatusCode != http.StatusUnauthorized { 344 344 + t.Fatalf("expected 401 Unauthorized, got %d", resp.StatusCode) 345 345 + } 346 346 + } 347 347 + 348 348 + { 349 349 + header := http.Header{} 350 350 + header.Set("Authorization", "Bearer "+secret) 351 351 + conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header) 352 352 + if err != nil { 353 353 + t.Fatalf("dial failed: %v", err) 354 354 + } 355 355 + defer conn.Close() 356 356 + 357 357 + stream := millproto.NewWSStream(conn) 358 358 + enc := millproto.NewEncoder(stream) 359 359 + dec := millproto.NewDecoder(stream) 360 360 + 361 361 + hello := &millproto.Message{Hello: &millv1.Hello{ 362 362 + ProtocolVersion: millproto.ProtocolVersion, 363 363 + Arch: "amd64", 364 364 + Labels: []string{"linux", "gpu"}, 365 365 + Epoch: "inc-1", 366 366 + }} 367 367 + if err := enc.Encode(hello); err != nil { 368 368 + t.Fatalf("encode hello: %v", err) 369 369 + } 370 370 + 371 371 + _, err = dec.Decode() 372 372 + if err == nil { 373 373 + t.Fatal("expected server to close connection for unauthorized label, but got a message") 374 374 + } 375 375 + } 376 376 + 377 377 + { 378 378 + header := http.Header{} 379 379 + header.Set("Authorization", "Bearer "+secret) 380 380 + conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header) 381 381 + if err != nil { 382 382 + t.Fatalf("dial failed: %v", err) 383 383 + } 384 384 + defer conn.Close() 385 385 + 386 386 + stream := millproto.NewWSStream(conn) 387 387 + enc := millproto.NewEncoder(stream) 388 388 + dec := millproto.NewDecoder(stream) 389 389 + 390 390 + hello := &millproto.Message{Hello: &millv1.Hello{ 391 391 + ProtocolVersion: millproto.ProtocolVersion, 392 392 + Arch: "amd64", 393 393 + Labels: []string{"linux"}, 394 394 + Epoch: "inc-1", 395 395 + }} 396 396 + if err := enc.Encode(hello); err != nil { 397 397 + t.Fatalf("encode hello: %v", err) 398 398 + } 399 399 + 400 400 + msg, err := dec.Decode() 401 401 + if err != nil { 402 402 + t.Fatalf("expected resume message, got error: %v", err) 403 403 + } 404 404 + res := msg.GetResume() 405 405 + if res == nil { 406 406 + t.Fatal("expected Resume message, got nil") 407 407 + } 408 408 + if res.GetEpoch() != "inc-1" { 409 409 + t.Fatalf("expected epoch inc-1, got %q", res.GetEpoch()) 410 410 + } 411 411 + } 412 412 + } 413 413 + 414 414 + func TestHandshakeTimeoutAndConcurrency(t *testing.T) { 415 415 + _, _, server, secret := setupTestServer(t, []string{"linux"}) 416 416 + wsUrl := "ws" + strings.TrimPrefix(server.URL, "http") 417 417 + 418 418 + // executor that never sends Hello is dropped after the 5s pre-hello deadline 419 419 + { 420 420 + header := http.Header{} 421 421 + header.Set("Authorization", "Bearer "+secret) 422 422 + conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header) 423 423 + if err != nil { 424 424 + t.Fatalf("dial failed: %v", err) 425 425 + } 426 426 + defer conn.Close() 427 427 + 428 428 + time.Sleep(6 * time.Second) 429 429 + 430 430 + stream := millproto.NewWSStream(conn) 431 431 + enc := millproto.NewEncoder(stream) 432 432 + hello := &millproto.Message{Hello: &millv1.Hello{ 433 433 + ProtocolVersion: millproto.ProtocolVersion, 434 434 + Arch: "amd64", 435 435 + Labels: []string{"linux"}, 436 436 + Epoch: "inc-1", 437 437 + }} 438 438 + err = enc.Encode(hello) 439 439 + dec := millproto.NewDecoder(stream) 440 440 + _, readErr := dec.Decode() 441 441 + if readErr == nil { 442 442 + t.Fatal("expected server to have closed connection due to handshake timeout") 443 443 + } 444 444 + } 445 445 + 446 446 + // second live session for one identity is rejected with 409 before the ws upgrade 447 447 + { 448 448 + header := http.Header{} 449 449 + header.Set("Authorization", "Bearer "+secret) 450 450 + 451 451 + conn1, _, err := websocket.DefaultDialer.Dial(wsUrl, header) 452 452 + if err != nil { 453 453 + t.Fatalf("dial 1 failed: %v", err) 454 454 + } 455 455 + defer conn1.Close() 456 456 + 457 457 + stream1 := millproto.NewWSStream(conn1) 458 458 + enc1 := millproto.NewEncoder(stream1) 459 459 + dec1 := millproto.NewDecoder(stream1) 460 460 + hello1 := &millproto.Message{Hello: &millv1.Hello{ 461 461 + ProtocolVersion: millproto.ProtocolVersion, 462 462 + Arch: "amd64", 463 463 + Labels: []string{"linux"}, 464 464 + Epoch: "inc-1", 465 465 + }} 466 466 + if err := enc1.Encode(hello1); err != nil { 467 467 + t.Fatalf("encode hello 1: %v", err) 468 468 + } 469 469 + _, err = dec1.Decode() 470 470 + if err != nil { 471 471 + t.Fatalf("first connection handshake failed: %v", err) 472 472 + } 473 473 + 474 474 + _, resp, err := websocket.DefaultDialer.Dial(wsUrl, header) 475 475 + if err == nil { 476 476 + t.Fatal("expected second connection for same live identity to be rejected") 477 477 + } 478 478 + if resp != nil && resp.StatusCode != http.StatusConflict { 479 479 + t.Fatalf("expected 409 Conflict for duplicate session, got %d", resp.StatusCode) 480 480 + } 481 481 + } 482 482 + }

+85

spindle/mill/engine.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "log/slog" 6 6 + "time" 7 7 + 8 8 + "tangled.org/core/api/tangled" 9 9 + "tangled.org/core/spindle/engine" 10 10 + "tangled.org/core/spindle/models" 11 11 + "tangled.org/core/spindle/secrets" 12 12 + ) 13 13 + 14 14 + // raw pipeline/workflow carried forward, executor runs the real InitWorkflow 15 15 + type millWorkflowState struct { 16 16 + RawWorkflow tangled.Pipeline_Workflow 17 17 + RawPipeline tangled.Pipeline 18 18 + Lease *RemoteLease 19 19 + } 20 20 + 21 21 + // stand-in for a real engine, registered under the real names 22 22 + // ("microvm", "nixery"), all sharing one Mill 23 23 + type Engine struct { 24 24 + name string 25 25 + mill *Mill 26 26 + l *slog.Logger 27 27 + } 28 28 + 29 29 + func (e *Engine) AuthorsRemoteStatus() {} 30 30 + 31 31 + func NewEngine(name string, mill *Mill) *Engine { 32 32 + return &Engine{name: name, mill: mill, l: mill.l.With("engine", "mill:"+name)} 33 33 + } 34 34 + 35 35 + // synthetic one-step workflow so processPipeline injects TANGLED_* env 36 36 + // and marks pending normally. the real InitWorkflow runs exactly once, on 37 37 + // the executor inside ReserveSeat, and commit reuses that workflow 38 38 + func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { 39 39 + return &models.Workflow{ 40 40 + Name: twf.Name, 41 41 + Environment: map[string]string{}, 42 42 + Steps: []models.Step{remoteStep{}}, 43 43 + Data: &millWorkflowState{ 44 44 + RawWorkflow: twf, 45 45 + RawPipeline: tpl, 46 46 + }, 47 47 + }, nil 48 48 + } 49 49 + 50 50 + // no-op logger for the synthetic workflow. the executor's lines stream 51 51 + // into this wid's log directly, a local logger would just write competing 52 52 + // lines 53 53 + func (e *Engine) WorkflowLogger(wid models.WorkflowId) models.WorkflowLogger { 54 54 + return models.NullLogger{} 55 55 + } 56 56 + 57 57 + // the placement seam, blocks on remote placement which the user sees as 58 58 + // "pending". only StartWorkflows calls this, always Wait 59 59 + func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) { 60 60 + return e.mill.place(ctx, e.name, wid, wf) 61 61 + } 62 62 + 63 63 + // no-op. real setup happens on the executor 64 64 + func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error { 65 65 + e.l.Info("remote job placed, awaiting commit", "wid", wid) 66 66 + return nil 67 67 + } 68 68 + 69 69 + // hands over the secrets and blocks on the terminal result streamed over the 70 70 + // session 71 71 + func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, unlocked []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { 72 72 + return e.mill.commitAndWait(ctx, w, unlocked) 73 73 + } 74 74 + 75 75 + // deliberately generous, the executor enforces the real timeout. the mill 76 76 + // only caps a hung or silent executor, true death is caught by reconnect grace 77 77 + func (e *Engine) WorkflowTimeout() time.Duration { 78 78 + return e.mill.cfg.JobTimeout 79 79 + } 80 80 + 81 81 + // cancels a still-running attempt. no-op if already terminal 82 82 + func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { 83 83 + e.mill.destroy(wid) 84 84 + return nil 85 85 + }

+119

spindle/mill/executor/capability_test.go

Reviewed

··· 1 1 + package executor 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "encoding/json" 6 6 + "errors" 7 7 + "io" 8 8 + "log/slog" 9 9 + "testing" 10 10 + 11 11 + "tangled.org/core/api/tangled" 12 12 + millv1 "tangled.org/core/spindle/mill/proto/gen" 13 13 + "tangled.org/core/spindle/models" 14 14 + ) 15 15 + 16 16 + type placementEngine struct { 17 17 + *fakeEngine 18 18 + validationErr error 19 19 + validated bool 20 20 + } 21 21 + 22 22 + func (e *placementEngine) ValidateWorkflowPlacement(*models.Workflow) error { 23 23 + e.validated = true 24 24 + return e.validationErr 25 25 + } 26 26 + 27 27 + func TestHandleReserveRejectsMissingTriggerMetadata(t *testing.T) { 28 28 + enc := newCaptureEncoder() 29 29 + e := &Executor{ 30 30 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 31 31 + enc: enc, 32 32 + seats: 1, 33 33 + engines: map[string]models.Engine{"microvm": &fakeEngine{}}, 34 34 + active: make(map[string]*reservation), 35 35 + } 36 36 + twf, err := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) 37 37 + if err != nil { 38 38 + t.Fatal(err) 39 39 + } 40 40 + tpl, err := json.Marshal(tangled.Pipeline{}) 41 41 + if err != nil { 42 42 + t.Fatal(err) 43 43 + } 44 44 + 45 45 + // engines deref TriggerMetadata unconditionally. this must be a reject, 46 46 + // not a panic that takes the whole executor down 47 47 + e.handleReserve(context.Background(), &millv1.ReserveSeat{ 48 48 + LeaseId: "lease-1", 49 49 + TargetEngine: "microvm", 50 50 + RawWorkflowJson: string(twf), 51 51 + RawPipelineJson: string(tpl), 52 52 + Knot: "k", 53 53 + Rkey: "r", 54 54 + }) 55 55 + 56 56 + result := (<-enc.messages).GetReserveResult() 57 57 + if result == nil { 58 58 + t.Fatal("handleReserve() did not send ReserveResult") 59 59 + } 60 60 + if result.GetAccepted() { 61 61 + t.Fatal("handleReserve() accepted a pipeline without trigger metadata") 62 62 + } 63 63 + if result.GetRejectClass() != millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE { 64 64 + t.Fatalf("reject class = %v, want incompatible", result.GetRejectClass()) 65 65 + } 66 66 + if len(e.active) != 0 { 67 67 + t.Fatalf("active reservations = %d, want 0", len(e.active)) 68 68 + } 69 69 + } 70 70 + 71 71 + func TestHandleReserveValidatesPlacementBeforeAcquiringSlot(t *testing.T) { 72 72 + enc := newCaptureEncoder() 73 73 + validationErr := errors.New("image architecture is not native") 74 74 + eng := &placementEngine{fakeEngine: &fakeEngine{}, validationErr: validationErr} 75 75 + e := &Executor{ 76 76 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 77 77 + enc: enc, 78 78 + seats: 1, 79 79 + engines: map[string]models.Engine{"microvm": eng}, 80 80 + active: make(map[string]*reservation), 81 81 + } 82 82 + twf, err := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) 83 83 + if err != nil { 84 84 + t.Fatal(err) 85 85 + } 86 86 + tpl, err := json.Marshal(tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) 87 87 + if err != nil { 88 88 + t.Fatal(err) 89 89 + } 90 90 + 91 91 + e.handleReserve(context.Background(), &millv1.ReserveSeat{ 92 92 + LeaseId: "lease-1", 93 93 + TargetEngine: "microvm", 94 94 + RawWorkflowJson: string(twf), 95 95 + RawPipelineJson: string(tpl), 96 96 + Knot: "k", 97 97 + Rkey: "r", 98 98 + }) 99 99 + 100 100 + result := (<-enc.messages).GetReserveResult() 101 101 + if result == nil { 102 102 + t.Fatal("handleReserve() did not send ReserveResult") 103 103 + } 104 104 + if result.GetAccepted() { 105 105 + t.Fatal("handleReserve() accepted placement validation failure") 106 106 + } 107 107 + if result.GetRejectClass() != millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE { 108 108 + t.Fatalf("reject class = %v, want incompatible", result.GetRejectClass()) 109 109 + } 110 110 + if !eng.validated { 111 111 + t.Fatal("placement validator was not called") 112 112 + } 113 113 + if eng.acquireCalled { 114 114 + t.Fatal("slot acquisition ran after placement validation failed") 115 115 + } 116 116 + if len(e.active) != 0 { 117 117 + t.Fatalf("active reservations = %d, want 0", len(e.active)) 118 118 + } 119 119 + }

+613

spindle/mill/executor/executor.go

Reviewed

··· 1 1 + package executor 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "encoding/json" 6 6 + "errors" 7 7 + "fmt" 8 8 + "maps" 9 9 + "net/http" 10 10 + "runtime" 11 11 + "sync" 12 12 + "time" 13 13 + 14 14 + "github.com/bluesky-social/indigo/atproto/syntax" 15 15 + "log/slog" 16 16 + "strings" 17 17 + 18 18 + "tangled.org/core/api/tangled" 19 19 + "tangled.org/core/netutil" 20 20 + "tangled.org/core/notifier" 21 21 + "tangled.org/core/spindle/artifactstore" 22 22 + "tangled.org/core/spindle/config" 23 23 + "tangled.org/core/spindle/db" 24 24 + "tangled.org/core/spindle/engine" 25 25 + millproto "tangled.org/core/spindle/mill/proto" 26 26 + millv1 "tangled.org/core/spindle/mill/proto/gen" 27 27 + "tangled.org/core/spindle/models" 28 28 + ) 29 29 + 30 30 + const ( 31 31 + dialBackoffMin = 1 * time.Second 32 32 + dialBackoffMax = 30 * time.Second 33 33 + snapshotEvery = 15 * time.Second 34 34 + defaultSeats = 4 35 35 + ) 36 36 + 37 37 + type Executor struct { 38 38 + millURL string 39 39 + token string 40 40 + nodeID string 41 41 + seats int 42 42 + labels []string 43 43 + 44 44 + engines map[string]models.Engine 45 45 + db *db.DB 46 46 + n *notifier.Notifier 47 47 + cfg *config.Config 48 48 + l *slog.Logger 49 49 + writer artifactstore.Writer 50 50 + 51 51 + epoch string 52 52 + outboxBytes int64 53 53 + maxOutboxBytes int64 // 10 MiB default outbox cap 54 54 + 55 55 + eventMu sync.Mutex 56 56 + sendMu sync.Mutex 57 57 + flushMu sync.Mutex 58 58 + 59 59 + sentSeqno uint64 60 60 + 61 61 + connMu sync.Mutex 62 62 + enc messageEncoder 63 63 + sessionCancel context.CancelFunc 64 64 + 65 65 + mu sync.Mutex 66 66 + active map[string]*reservation 67 67 + draining bool 68 68 + 69 69 + snapshotMu sync.Mutex 70 70 + nextSeqno uint64 71 71 + 72 72 + lifecycleCtx context.Context 73 73 + jobsWG sync.WaitGroup 74 74 + } 75 75 + 76 76 + type reservation struct { 77 77 + leaseID string 78 78 + wid models.WorkflowId 79 79 + realEngine models.Engine 80 80 + slot engine.WorkflowSlot 81 81 + wf *models.Workflow 82 82 + repoDid syntax.DID 83 83 + vault *memVault 84 84 + 85 85 + committed bool 86 86 + cancelled bool 87 87 + cancel context.CancelFunc 88 88 + ttlTimer *time.Timer 89 89 + stopTail func() 90 90 + } 91 91 + 92 92 + type messageEncoder interface { 93 93 + Encode(*millproto.Message) error 94 94 + } 95 95 + 96 96 + func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *notifier.Notifier, l *slog.Logger, writers ...artifactstore.Writer) (*Executor, error) { 97 97 + seats := defaultSeats 98 98 + millURL := "" 99 99 + token := "" 100 100 + nodeID := "" 101 101 + var labels []string 102 102 + if cfg != nil { 103 103 + if cfg.Mill.Seats > 0 { 104 104 + seats = cfg.Mill.Seats 105 105 + } 106 106 + labels = normalizeLabels(cfg.Mill.Labels) 107 107 + millURL = cfg.Mill.URL 108 108 + token = cfg.Mill.SharedSecret 109 109 + nodeID = cfg.Server.Hostname 110 110 + } 111 111 + if d == nil || n == nil { 112 112 + return nil, fmt.Errorf("executor requires a database and notifier") 113 113 + } 114 114 + var writer artifactstore.Writer 115 115 + if len(writers) > 0 { 116 116 + writer = writers[0] 117 117 + } 118 118 + e := &Executor{ 119 119 + millURL: millURL, 120 120 + token: token, 121 121 + nodeID: nodeID, 122 122 + seats: seats, 123 123 + labels: labels, 124 124 + engines: engines, 125 125 + db: d, 126 126 + n: n, 127 127 + cfg: cfg, 128 128 + l: l.With("component", "mill.executor"), 129 129 + writer: writer, 130 130 + active: make(map[string]*reservation), 131 131 + maxOutboxBytes: 10 * 1024 * 1024, 132 132 + } 133 133 + if err := e.initOutbox(); err != nil { 134 134 + return nil, fmt.Errorf("initialize executor outbox: %w", err) 135 135 + } 136 136 + return e, nil 137 137 + } 138 138 + 139 139 + func (e *Executor) Connect(ctx context.Context) { 140 140 + e.lifecycleCtx = ctx 141 141 + sub := e.n.Subscribe() 142 142 + cursor, err := e.db.EventHighWater() 143 143 + if err != nil { 144 144 + e.n.Unsubscribe(sub) 145 145 + e.l.Error("establish event cursor failed", "err", err) 146 146 + return 147 147 + } 148 148 + e.drainEvents(&cursor) 149 149 + observerCtx, stopObserver := context.WithCancel(ctx) 150 150 + observerDone := make(chan struct{}) 151 151 + go func() { 152 152 + defer close(observerDone) 153 153 + e.observeLoop(observerCtx, sub, cursor) 154 154 + }() 155 155 + defer e.n.Unsubscribe(sub) 156 156 + 157 157 + backoff := dialBackoffMin 158 158 + for { 159 159 + if ctx.Err() != nil { 160 160 + break 161 161 + } 162 162 + err := e.runSession(ctx) 163 163 + if ctx.Err() != nil { 164 164 + break 165 165 + } 166 166 + e.l.Warn("mill session ended; reconnecting", "err", err, "backoff", backoff) 167 167 + select { 168 168 + case <-ctx.Done(): 169 169 + break 170 170 + case <-time.After(backoff): 171 171 + } 172 172 + backoff = min(backoff*2, dialBackoffMax) 173 173 + } 174 174 + 175 175 + e.jobsWG.Wait() 176 176 + stopObserver() 177 177 + <-observerDone 178 178 + e.drainEvents(&cursor) 179 179 + } 180 180 + 181 181 + func (e *Executor) runSession(ctx context.Context) error { 182 182 + dev := e.cfg == nil || e.cfg.Server.Dev 183 183 + if _, err := netutil.EnforceWSSURL(e.millURL, dev); err != nil { 184 184 + return fmt.Errorf("mill url: %w", err) 185 185 + } 186 186 + header := http.Header{} 187 187 + if e.token != "" { 188 188 + header.Set("Authorization", "Bearer "+e.token) 189 189 + } 190 190 + conn, _, err := netutil.SSRFWebsocketDialer(dev).DialContext(ctx, e.millURL, header) 191 191 + if err != nil { 192 192 + return fmt.Errorf("dial mill: %w", err) 193 193 + } 194 194 + defer conn.Close() 195 195 + 196 196 + sessionCtx, cancelSession := context.WithCancel(ctx) 197 197 + defer cancelSession() 198 198 + stopClose := context.AfterFunc(sessionCtx, func() { _ = conn.Close() }) 199 199 + defer stopClose() 200 200 + 201 201 + stream := millproto.NewWSStream(conn) 202 202 + enc := millproto.NewEncoder(stream) 203 203 + dec := millproto.NewDecoder(stream) 204 204 + 205 205 + hello := &millproto.Message{Hello: &millv1.Hello{ 206 206 + ProtocolVersion: millproto.ProtocolVersion, 207 207 + Arch: runtime.GOARCH, 208 208 + Labels: e.labels, 209 209 + Epoch: e.epoch, 210 210 + }} 211 211 + if err := enc.Encode(hello); err != nil { 212 212 + return fmt.Errorf("send hello: %w", err) 213 213 + } 214 214 + 215 215 + resumeMsg, err := dec.Decode() 216 216 + if err != nil { 217 217 + return fmt.Errorf("read resume: %w", err) 218 218 + } 219 219 + resume := resumeMsg.GetResume() 220 220 + if resume == nil { 221 221 + return fmt.Errorf("expected resume, got something else") 222 222 + } 223 223 + if resume.GetEpoch() != e.epoch { 224 224 + return fmt.Errorf("resume epoch mismatch: got %q, want %q", resume.GetEpoch(), e.epoch) 225 225 + } 226 226 + 227 227 + e.connMu.Lock() 228 228 + e.sessionCancel = cancelSession 229 229 + e.enc = enc 230 230 + e.connMu.Unlock() 231 231 + defer func() { 232 232 + e.connMu.Lock() 233 233 + e.enc = nil 234 234 + e.sessionCancel = nil 235 235 + e.connMu.Unlock() 236 236 + }() 237 237 + 238 238 + readErr := make(chan error, 1) 239 239 + go func() { 240 240 + for { 241 241 + msg, err := dec.Decode() 242 242 + if err != nil { 243 243 + readErr <- fmt.Errorf("read: %w", err) 244 244 + return 245 245 + } 246 246 + e.dispatch(sessionCtx, msg) 247 247 + } 248 248 + }() 249 249 + 250 250 + if err := e.replay(resume.GetAckSeqno()); err != nil { 251 251 + cancelSession() 252 252 + <-readErr 253 253 + return fmt.Errorf("replay failed: %w", err) 254 254 + } 255 255 + e.pushSnapshot() 256 256 + e.l.Info("connected to mill", "node", e.nodeID, "resumeFrom", resume.GetAckSeqno()) 257 257 + 258 258 + go e.snapshotLoop(sessionCtx, enc) 259 259 + return <-readErr 260 260 + } 261 261 + func (e *Executor) send(msg *millproto.Message) { 262 262 + e.connMu.Lock() 263 263 + enc := e.enc 264 264 + cancel := e.sessionCancel 265 265 + e.connMu.Unlock() 266 266 + if enc != nil { 267 267 + e.sendMu.Lock() 268 268 + err := enc.Encode(msg) 269 269 + e.sendMu.Unlock() 270 270 + if err != nil { 271 271 + e.l.Error("send failed, ending session", "err", err) 272 272 + if cancel != nil { 273 273 + cancel() 274 274 + } 275 275 + } 276 276 + } 277 277 + } 278 278 + 279 279 + func (e *Executor) dispatch(ctx context.Context, msg *millproto.Message) { 280 280 + switch { 281 281 + case msg.GetReserveSeat() != nil: 282 282 + e.handleReserve(ctx, msg.GetReserveSeat()) 283 283 + case msg.GetCommitLease() != nil: 284 284 + e.handleCommit(ctx, msg.GetCommitLease()) 285 285 + case msg.GetReleaseLease() != nil: 286 286 + e.handleRelease(msg.GetReleaseLease().GetLeaseId()) 287 287 + case msg.GetCancelAttempt() != nil: 288 288 + e.handleCancel(msg.GetCancelAttempt().GetLeaseId()) 289 289 + case msg.GetAck() != nil: 290 290 + e.handleAck(msg.GetAck()) 291 291 + default: 292 292 + e.l.Warn("unhandled incoming message", "type", fmt.Sprintf("%T", msg)) 293 293 + } 294 294 + } 295 295 + 296 296 + func (e *Executor) sendReject(leaseID string, reason string, class millv1.RejectClass) { 297 297 + e.send(&millproto.Message{ReserveResult: &millv1.ReserveResult{ 298 298 + LeaseId: leaseID, 299 299 + Accepted: false, 300 300 + RejectReason: reason, 301 301 + RejectClass: class, 302 302 + }}) 303 303 + } 304 304 + 305 305 + func (e *Executor) sendCommitted(leaseID string) { 306 306 + e.send(&millproto.Message{Committed: &millv1.Committed{LeaseId: leaseID}}) 307 307 + } 308 308 + 309 309 + func (e *Executor) sendCancelAck(leaseID string) { 310 310 + e.send(&millproto.Message{CancelAck: &millv1.CancelAck{LeaseId: leaseID}}) 311 311 + } 312 312 + 313 313 + func (e *Executor) releaseReservation(cleanup func()) { 314 314 + if cleanup != nil { 315 315 + cleanup() 316 316 + } 317 317 + e.pushSnapshot() 318 318 + } 319 319 + 320 320 + func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { 321 321 + reject := func(reason string, class millv1.RejectClass) { 322 322 + e.sendReject(rs.GetLeaseId(), reason, class) 323 323 + } 324 324 + 325 325 + e.mu.Lock() 326 326 + draining := e.draining 327 327 + e.mu.Unlock() 328 328 + if draining { 329 329 + reject("draining", millv1.RejectClass_REJECT_CLASS_TRANSIENT) 330 330 + return 331 331 + } 332 332 + 333 333 + realEngine, ok := e.engines[rs.GetTargetEngine()] 334 334 + if !ok { 335 335 + reject("unknown engine "+rs.GetTargetEngine(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 336 336 + return 337 337 + } 338 338 + slotter, ok := realEngine.(engine.WorkflowSlotter) 339 339 + if !ok { 340 340 + reject("engine does not support workflow slots", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 341 341 + return 342 342 + } 343 343 + 344 344 + var twf tangled.Pipeline_Workflow 345 345 + if err := json.Unmarshal([]byte(rs.GetRawWorkflowJson()), &twf); err != nil { 346 346 + reject("bad workflow json", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 347 347 + return 348 348 + } 349 349 + var tpl tangled.Pipeline 350 350 + if err := json.Unmarshal([]byte(rs.GetRawPipelineJson()), &tpl); err != nil { 351 351 + reject("bad pipeline json", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 352 352 + return 353 353 + } 354 354 + if tpl.TriggerMetadata == nil { 355 355 + reject("pipeline missing trigger metadata", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 356 356 + return 357 357 + } 358 358 + 359 359 + pipelineId := models.PipelineId{Knot: rs.GetKnot(), Rkey: rs.GetRkey()} 360 360 + wid := models.WorkflowId{PipelineId: pipelineId, Name: twf.Name} 361 361 + 362 362 + wf, err := realEngine.InitWorkflow(twf, tpl) 363 363 + if err != nil { 364 364 + reject("init workflow: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 365 365 + return 366 366 + } 367 367 + if validator, ok := realEngine.(engine.WorkflowPlacementValidator); ok { 368 368 + if err := validator.ValidateWorkflowPlacement(wf); err != nil { 369 369 + reject("validate workflow placement: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 370 370 + return 371 371 + } 372 372 + } 373 373 + if wf.Environment == nil { 374 374 + wf.Environment = make(map[string]string) 375 375 + } 376 376 + maps.Copy(wf.Environment, models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId)) 377 377 + 378 378 + slot, err := slotter.AcquireWorkflowSlot(ctx, wid, wf, engine.NoWait) 379 379 + if err != nil { 380 380 + class := millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE 381 381 + if errors.Is(err, engine.ErrNoWorkflowSlots) { 382 382 + class = millv1.RejectClass_REJECT_CLASS_TRANSIENT 383 383 + } 384 384 + reject(err.Error(), class) 385 385 + return 386 386 + } 387 387 + 388 388 + var repoDid syntax.DID 389 389 + if tpl.TriggerMetadata != nil && tpl.TriggerMetadata.Repo != nil && tpl.TriggerMetadata.Repo.RepoDid != nil { 390 390 + repoDid, _ = syntax.ParseDID(*tpl.TriggerMetadata.Repo.RepoDid) 391 391 + } 392 392 + 393 393 + res := &reservation{ 394 394 + leaseID: rs.GetLeaseId(), 395 395 + wid: wid, 396 396 + realEngine: realEngine, 397 397 + slot: slot, 398 398 + wf: wf, 399 399 + repoDid: repoDid, 400 400 + } 401 401 + 402 402 + e.snapshotMu.Lock() 403 403 + e.mu.Lock() 404 404 + e.active[res.leaseID] = res 405 405 + res.ttlTimer = time.AfterFunc(ttlDuration(rs.GetTtlSeconds()), func() { e.expireReservation(res.leaseID) }) 406 406 + e.mu.Unlock() 407 407 + 408 408 + e.send(&millproto.Message{ReserveResult: &millv1.ReserveResult{ 409 409 + LeaseId: rs.GetLeaseId(), 410 410 + Accepted: true, 411 411 + }}) 412 412 + e.pushSnapshotLocked() 413 413 + e.snapshotMu.Unlock() 414 414 + } 415 415 + 416 416 + func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) { 417 417 + e.mu.Lock() 418 418 + res := e.active[cl.GetLeaseId()] 419 419 + if res == nil { 420 420 + e.mu.Unlock() 421 421 + e.sendReject(cl.GetLeaseId(), "reservation missing or expired", millv1.RejectClass_REJECT_CLASS_TRANSIENT) 422 422 + return 423 423 + } 424 424 + if res.committed { 425 425 + e.mu.Unlock() 426 426 + e.sendCommitted(cl.GetLeaseId()) 427 427 + return 428 428 + } 429 429 + res.committed = true 430 430 + if res.ttlTimer != nil { 431 431 + res.ttlTimer.Stop() 432 432 + } 433 433 + 434 434 + jobCtx, cancel := context.WithCancel(e.lifecycleCtx) 435 435 + res.cancel = cancel 436 436 + e.mu.Unlock() 437 437 + 438 438 + vault := newMemVault(cl.GetSecrets()) 439 439 + re := newReservedEngine(res.realEngine, res.slot) 440 440 + pipeline := &models.Pipeline{ 441 441 + RepoDid: res.repoDid, 442 442 + Workflows: map[models.Engine][]models.Workflow{re: {*res.wf}}, 443 443 + TrustedSource: true, 444 444 + } 445 445 + 446 446 + e.startTail(res) 447 447 + 448 448 + e.jobsWG.Add(1) 449 449 + go func() { 450 450 + defer e.jobsWG.Done() 451 451 + engine.StartWorkflows(e.l, vault, e.cfg, nil, e.db, e.n, jobCtx, pipeline, res.wid.PipelineId) 452 452 + }() 453 453 + 454 454 + e.sendCommitted(cl.GetLeaseId()) 455 455 + } 456 456 + 457 457 + func (e *Executor) handleRelease(leaseID string) { 458 458 + cleanup, ok := e.takeUncommittedReservation(leaseID, true) 459 459 + if !ok { 460 460 + return 461 461 + } 462 462 + e.releaseReservation(cleanup) 463 463 + } 464 464 + 465 465 + func (e *Executor) handleCancel(leaseID string) { 466 466 + e.mu.Lock() 467 467 + res := e.active[leaseID] 468 468 + if res == nil { 469 469 + e.mu.Unlock() 470 470 + if err := e.appendTerminal(leaseID, string(models.StatusKindCancelled), nil); err != nil { 471 471 + e.l.Error("persist cancelled reservation terminal", "lease", leaseID, "err", err) 472 472 + return 473 473 + } 474 474 + e.sendCancelAck(leaseID) 475 475 + return 476 476 + } 477 477 + res.cancelled = true 478 478 + cancel := res.cancel 479 479 + committed := res.committed 480 480 + var cleanup func() 481 481 + if !committed { 482 482 + cleanup = e.removeReservationLocked(res, true) 483 483 + } 484 484 + e.mu.Unlock() 485 485 + 486 486 + if !committed { 487 487 + if err := e.appendTerminal(leaseID, string(models.StatusKindCancelled), nil); err != nil { 488 488 + e.l.Error("persist cancelled reservation terminal", "lease", leaseID, "err", err) 489 489 + e.releaseReservation(cleanup) 490 490 + return 491 491 + } 492 492 + e.sendCancelAck(leaseID) 493 493 + e.releaseReservation(cleanup) 494 494 + return 495 495 + } 496 496 + 497 497 + e.sendCancelAck(leaseID) 498 498 + if cancel != nil { 499 499 + cancel() 500 500 + } 501 501 + } 502 502 + 503 503 + func (e *Executor) expireReservation(leaseID string) { 504 504 + cleanup, ok := e.takeUncommittedReservation(leaseID, true) 505 505 + if !ok { 506 506 + return 507 507 + } 508 508 + e.releaseReservation(cleanup) 509 509 + } 510 510 + 511 511 + func (e *Executor) takeUncommittedReservation(leaseID string, releaseSlot bool) (func(), bool) { 512 512 + e.mu.Lock() 513 513 + defer e.mu.Unlock() 514 514 + res := e.active[leaseID] 515 515 + if res == nil || res.committed { 516 516 + return nil, false 517 517 + } 518 518 + if res.ttlTimer != nil { 519 519 + res.ttlTimer.Stop() 520 520 + } 521 521 + return e.removeReservationLocked(res, releaseSlot), true 522 522 + } 523 523 + 524 524 + func (e *Executor) removeReservationLocked(res *reservation, releaseSlot bool) func() { 525 525 + delete(e.active, res.leaseID) 526 526 + slot := res.slot 527 527 + return func() { 528 528 + if releaseSlot && slot != nil { 529 529 + slot.Release() 530 530 + } 531 531 + } 532 532 + } 533 533 + 534 534 + func (e *Executor) snapshotLoop(ctx context.Context, enc *millproto.Encoder) { 535 535 + ticker := time.NewTicker(snapshotEvery) 536 536 + defer ticker.Stop() 537 537 + 538 538 + for { 539 539 + select { 540 540 + case <-ctx.Done(): 541 541 + return 542 542 + case <-ticker.C: 543 543 + e.pushSnapshot() 544 544 + } 545 545 + } 546 546 + } 547 547 + 548 548 + func (e *Executor) pushSnapshot() { 549 549 + e.snapshotMu.Lock() 550 550 + defer e.snapshotMu.Unlock() 551 551 + e.pushSnapshotLocked() 552 552 + } 553 553 + 554 554 + func (e *Executor) pushSnapshotLocked() { 555 555 + e.mu.Lock() 556 556 + activeLeases := make([]string, 0, len(e.active)) 557 557 + for leaseID := range e.active { 558 558 + activeLeases = append(activeLeases, leaseID) 559 559 + } 560 560 + e.mu.Unlock() 561 561 + 562 562 + avail := make(map[string]*millv1.EngineAvailability) 563 563 + for name, eng := range e.engines { 564 564 + a := &millv1.EngineAvailability{Available: true} 565 565 + if getter, ok := eng.(interface{ Load() map[string]float64 }); ok { 566 566 + a.Load = getter.Load() 567 567 + } 568 568 + avail[name] = a 569 569 + } 570 570 + 571 571 + e.nextSeqno++ 572 572 + snap := &millproto.Message{ 573 573 + NodeSnapshot: &millv1.NodeSnapshot{ 574 574 + Seqno: e.nextSeqno, 575 575 + Engines: avail, 576 576 + ActiveLeaseIds: activeLeases, 577 577 + }, 578 578 + } 579 579 + e.send(snap) 580 580 + } 581 581 + 582 582 + func (e *Executor) Drain() { 583 583 + e.mu.Lock() 584 584 + e.draining = true 585 585 + e.mu.Unlock() 586 586 + e.pushSnapshot() 587 587 + } 588 588 + 589 589 + func ttlDuration(secs uint32) time.Duration { 590 590 + if secs == 0 { 591 591 + return defaultReservationTTL 592 592 + } 593 593 + return time.Duration(secs) * time.Second 594 594 + } 595 595 + 596 596 + const defaultReservationTTL = 60 * time.Second 597 597 + 598 598 + func normalizeLabels(labels []string) []string { 599 599 + seen := make(map[string]struct{}, len(labels)) 600 600 + out := make([]string, 0, len(labels)) 601 601 + for _, label := range labels { 602 602 + label = strings.TrimSpace(label) 603 603 + if label == "" { 604 604 + continue 605 605 + } 606 606 + if _, ok := seen[label]; ok { 607 607 + continue 608 608 + } 609 609 + seen[label] = struct{}{} 610 610 + out = append(out, label) 611 611 + } 612 612 + return out 613 613 + }

+251

spindle/mill/executor/observe.go

Reviewed

··· 1 1 + package executor 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "crypto/sha256" 6 6 + "encoding/hex" 7 7 + "fmt" 8 8 + "io" 9 9 + "os" 10 10 + "strings" 11 11 + "sync" 12 12 + "time" 13 13 + 14 14 + "github.com/hpcloud/tail" 15 15 + "tangled.org/core/api/tangled" 16 16 + millproto "tangled.org/core/spindle/mill/proto" 17 17 + millv1 "tangled.org/core/spindle/mill/proto/gen" 18 18 + "tangled.org/core/spindle/models" 19 19 + "tangled.org/core/spindle/secrets" 20 20 + ) 21 21 + 22 22 + func (e *Executor) observeLoop(ctx context.Context, sub <-chan struct{}, cursor int64) { 23 23 + ticker := time.NewTicker(5 * time.Second) 24 24 + defer ticker.Stop() 25 25 + 26 26 + for { 27 27 + select { 28 28 + case <-ctx.Done(): 29 29 + return 30 30 + case <-sub: 31 31 + case <-ticker.C: 32 32 + } 33 33 + e.drainEvents(&cursor) 34 34 + } 35 35 + } 36 36 + 37 37 + func (e *Executor) drainEvents(cursor *int64) { 38 38 + events, err := e.db.GetEvents(*cursor, 128) 39 39 + if err != nil { 40 40 + e.l.Error("drain status events failed", "err", err) 41 41 + return 42 42 + } 43 43 + for _, ev := range events { 44 44 + if ev.Created > *cursor { 45 45 + *cursor = ev.Created 46 46 + } 47 47 + st, ok := parseStatus(ev.EventJson) 48 48 + if !ok { 49 49 + continue 50 50 + } 51 51 + if err := e.onStatusRow(st); err != nil { 52 52 + e.l.Error("process status row failed", "err", err) 53 53 + } 54 54 + } 55 55 + } 56 56 + 57 57 + func (e *Executor) onStatusRow(st *tangled.PipelineStatus) error { 58 58 + res := e.reservationFor(st.Pipeline, st.Workflow) 59 59 + if res == nil { 60 60 + return nil 61 61 + } 62 62 + 63 63 + if models.StatusKind(st.Status).IsFinish() { 64 64 + return e.finishJob(res, st) 65 65 + } 66 66 + 67 67 + return e.appendStatus(res.leaseID, st) 68 68 + } 69 69 + 70 70 + func (e *Executor) finishJob(res *reservation, st *tangled.PipelineStatus) error { 71 71 + e.mu.Lock() 72 72 + if e.active[res.leaseID] != res { 73 73 + e.mu.Unlock() 74 74 + return nil 75 75 + } 76 76 + cancelled := res.cancelled 77 77 + e.mu.Unlock() 78 78 + 79 79 + // 1. Finalize log tail first so all log lines precede terminal event 80 80 + if res.stopTail != nil { 81 81 + res.stopTail() 82 82 + } 83 83 + 84 84 + terminalStatus := st.Status 85 85 + if cancelled { 86 86 + terminalStatus = string(models.StatusKindCancelled) 87 87 + } 88 88 + var logDir string 89 89 + if e.cfg != nil { 90 90 + logDir = e.cfg.Server.LogDir 91 91 + } 92 92 + logPath := models.LogFilePath(logDir, res.wid) 93 93 + 94 94 + var errStr string 95 95 + if st != nil && st.Error != nil { 96 96 + errStr = *st.Error 97 97 + } 98 98 + var exitCode int64 99 99 + if st != nil && st.ExitCode != nil { 100 100 + exitCode = *st.ExitCode 101 101 + } 102 102 + 103 103 + // Calculate SHA256 of log file 104 104 + hash := "" 105 105 + if f, err := os.Open(logPath); err == nil { 106 106 + h := sha256.New() 107 107 + if _, err := io.Copy(h, f); err == nil { 108 108 + hash = "sha256:" + hex.EncodeToString(h.Sum(nil)) 109 109 + } 110 110 + _ = f.Close() 111 111 + } 112 112 + 113 113 + // refs are opaque keys interpreted by the configured artifact store 114 114 + ref := "logs/" + res.leaseID + ".log" 115 115 + 116 116 + // 2. Persist restart-retryable pending artifact state 117 117 + if e.db != nil { 118 118 + _ = e.db.SavePendingArtifact(res.leaseID, res.wid.Name, terminalStatus, errStr, exitCode, ref, hash) 119 119 + } 120 120 + 121 121 + // 3. Upload artifact with non-cancelled cleanup context 122 122 + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(context.Background()), 2*time.Minute) 123 123 + defer cancel() 124 124 + 125 125 + if e.writer != nil { 126 126 + if f, err := os.Open(logPath); err == nil { 127 127 + defer f.Close() 128 128 + if uploadErr := e.writer.Put(cleanupCtx, ref, f); uploadErr != nil { 129 129 + e.l.Error("artifact upload failed", "lease", res.leaseID, "err", uploadErr) 130 130 + return fmt.Errorf("artifact upload: %w", uploadErr) 131 131 + } 132 132 + } 133 133 + } 134 134 + 135 135 + // 4. Append terminal event (with LogArtifact) to outbox 136 136 + if err := e.appendTerminalWithArtifact(res.leaseID, terminalStatus, st, ref, hash); err != nil { 137 137 + return err 138 138 + } 139 139 + 140 140 + // 5. Remove pending artifact state after terminal outbox append succeeds 141 141 + if e.db != nil { 142 142 + _ = e.db.RemovePendingArtifact(res.leaseID) 143 143 + } 144 144 + e.mu.Lock() 145 145 + cleanup := e.removeReservationLocked(res, false) 146 146 + e.mu.Unlock() 147 147 + 148 148 + cleanup() 149 149 + 150 150 + e.pushSnapshot() 151 151 + return nil 152 152 + } 153 153 + 154 154 + func (e *Executor) reservationFor(pipelineAturi, workflow string) *reservation { 155 155 + e.mu.Lock() 156 156 + defer e.mu.Unlock() 157 157 + for _, res := range e.active { 158 158 + if string(res.wid.PipelineId.AtUri()) == pipelineAturi && res.wid.Name == workflow { 159 159 + return res 160 160 + } 161 161 + } 162 162 + return nil 163 163 + } 164 164 + 165 165 + func (e *Executor) maskSecrets(res *reservation, text string) string { 166 166 + if res == nil || res.vault == nil { 167 167 + return text 168 168 + } 169 169 + for _, s := range res.vault.secrets { 170 170 + if s.Value != "" { 171 171 + text = strings.ReplaceAll(text, s.Value, "***") 172 172 + } 173 173 + } 174 174 + return text 175 175 + } 176 176 + 177 177 + func (e *Executor) SendLiveLog(leaseID string, raw []byte) error { 178 178 + e.connMu.Lock() 179 179 + enc := e.enc 180 180 + e.connMu.Unlock() 181 181 + if enc == nil { 182 182 + return nil 183 183 + } 184 184 + return enc.Encode(&millproto.Message{ 185 185 + LiveLog: &millv1.LiveLog{ 186 186 + LeaseId: leaseID, 187 187 + RawJson: raw, 188 188 + }, 189 189 + }) 190 190 + } 191 191 + 192 192 + func (e *Executor) startTail(res *reservation) { 193 193 + path := models.LogFilePath(e.cfg.Server.LogDir, res.wid) 194 194 + t, err := tail.TailFile(path, tail.Config{ 195 195 + Follow: true, 196 196 + ReOpen: true, 197 197 + MustExist: false, 198 198 + Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}, 199 199 + Logger: tail.DiscardingLogger, 200 200 + }) 201 201 + if err != nil { 202 202 + e.l.Error("tail log file failed", "wid", res.wid, "err", err) 203 203 + return 204 204 + } 205 205 + 206 206 + done := make(chan struct{}) 207 207 + go func() { 208 208 + defer close(done) 209 209 + for line := range t.Lines { 210 210 + if line == nil || line.Err != nil { 211 211 + continue 212 212 + } 213 213 + masked := e.maskSecrets(res, line.Text) 214 214 + _ = e.SendLiveLog(res.leaseID, []byte(masked+"\n")) 215 215 + } 216 216 + }() 217 217 + 218 218 + var once sync.Once 219 219 + res.stopTail = func() { 220 220 + once.Do(func() { 221 221 + _ = t.StopAtEOF() 222 222 + <-done 223 223 + }) 224 224 + } 225 225 + } 226 226 + 227 227 + type memVault struct { 228 228 + secrets []secrets.UnlockedSecret 229 229 + } 230 230 + 231 231 + func newMemVault(pb []*millv1.Secret) *memVault { 232 232 + v := &memVault{secrets: make([]secrets.UnlockedSecret, 0, len(pb))} 233 233 + for _, s := range pb { 234 234 + v.secrets = append(v.secrets, secrets.UnlockedSecret{ 235 235 + Key: s.Key, 236 236 + Value: s.Value, 237 237 + }) 238 238 + } 239 239 + return v 240 240 + } 241 241 + 242 242 + func (v *memVault) GetSecretsUnlocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.UnlockedSecret, error) { 243 243 + return v.secrets, nil 244 244 + } 245 245 + func (v *memVault) GetSecretsLocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.LockedSecret, error) { 246 246 + return nil, nil 247 247 + } 248 248 + func (v *memVault) AddSecret(ctx context.Context, s secrets.UnlockedSecret) error { return nil } 249 249 + func (v *memVault) RemoveSecret(ctx context.Context, s secrets.Secret[any]) error { return nil } 250 250 + 251 251 + var _ secrets.Manager = (*memVault)(nil)

+344

spindle/mill/executor/outbox.go

Reviewed

··· 1 1 + package executor 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "crypto/rand" 6 6 + "encoding/hex" 7 7 + "encoding/json" 8 8 + "fmt" 9 9 + "os" 10 10 + "time" 11 11 + 12 12 + "google.golang.org/protobuf/proto" 13 13 + "tangled.org/core/api/tangled" 14 14 + "tangled.org/core/spindle/db" 15 15 + millproto "tangled.org/core/spindle/mill/proto" 16 16 + millv1 "tangled.org/core/spindle/mill/proto/gen" 17 17 + "tangled.org/core/spindle/models" 18 18 + ) 19 19 + 20 20 + const ( 21 21 + maxBatchBytes = 4 * 1024 * 1024 22 22 + maxBatchEvents = 128 23 23 + ) 24 24 + 25 25 + func generateEpoch() string { 26 26 + var b [8]byte 27 27 + _, _ = rand.Read(b[:]) 28 28 + return hex.EncodeToString(b[:]) 29 29 + } 30 30 + 31 31 + func (e *Executor) initOutbox() error { 32 32 + e.eventMu.Lock() 33 33 + defer e.eventMu.Unlock() 34 34 + 35 35 + epoch, _, err := e.db.GetOutboxState() 36 36 + if err != nil { 37 37 + return err 38 38 + } 39 39 + if epoch != "" { 40 40 + e.epoch = epoch 41 41 + } else { 42 42 + e.epoch = generateEpoch() 43 43 + if err := e.db.SetOutboxEpoch(e.epoch); err != nil { 44 44 + return fmt.Errorf("set outbox epoch: %w", err) 45 45 + } 46 46 + } 47 47 + 48 48 + rows, err := e.db.ListOutboxRows() 49 49 + if err != nil { 50 50 + return fmt.Errorf("list outbox rows: %w", err) 51 51 + } 52 52 + 53 53 + for _, r := range rows { 54 54 + e.outboxBytes += r.ByteSize 55 55 + } 56 56 + _ = e.recoverPendingArtifacts() 57 57 + return nil 58 58 + } 59 59 + 60 60 + func (e *Executor) appendAndSend(leaseID string, payload any, control bool) error { 61 61 + entry := &millv1.Event{LeaseId: leaseID} 62 62 + switch payload := payload.(type) { 63 63 + case *millv1.Event_StatusEvent: 64 64 + entry.Payload = payload 65 65 + case *millv1.Event_AttemptResult: 66 66 + entry.Payload = payload 67 67 + default: 68 68 + return fmt.Errorf("unsupported stream payload %T", payload) 69 69 + } 70 70 + isTerminal := false 71 71 + if _, ok := payload.(*millv1.Event_AttemptResult); ok { 72 72 + isTerminal = true 73 73 + } 74 74 + entry.Seqno = ^uint64(0) 75 75 + wireSize := proto.Size(&millproto.Message{EventBatch: &millv1.EventBatch{ 76 76 + Epoch: e.epoch, 77 77 + Events: []*millv1.Event{entry}, 78 78 + }}) 79 79 + entry.Seqno = 0 80 80 + if wireSize > maxBatchBytes { 81 81 + return fmt.Errorf("control stream entry exceeds wire limit: %d > %d", wireSize, maxBatchBytes) 82 82 + } 83 83 + 84 84 + encoded, err := proto.Marshal(entry) 85 85 + if err != nil { 86 86 + return fmt.Errorf("marshal stream entry: %w", err) 87 87 + } 88 88 + 89 89 + e.eventMu.Lock() 90 90 + if control && !isTerminal && e.maxOutboxBytes > 0 && e.outboxBytes+int64(len(encoded)) > e.maxOutboxBytes { 91 91 + e.l.Warn("outbox reserve exhausted; dropping nonterminal status", "cap", e.maxOutboxBytes) 92 92 + e.eventMu.Unlock() 93 93 + return nil 94 94 + } 95 95 + if _, err := e.db.AppendOutboxRow(encoded, control); err != nil { 96 96 + e.eventMu.Unlock() 97 97 + return fmt.Errorf("append outbox row: %w", err) 98 98 + } 99 99 + e.outboxBytes += int64(len(encoded)) 100 100 + e.eventMu.Unlock() 101 101 + 102 102 + e.sendPending() 103 103 + return nil 104 104 + } 105 105 + 106 106 + func (e *Executor) appendStatus(leaseID string, st *tangled.PipelineStatus) error { 107 107 + if st.Status != string(models.StatusKindRunning) { 108 108 + return fmt.Errorf("unsupported nonterminal status %q", st.Status) 109 109 + } 110 110 + exit, errStr := parseStatusExitAndError(st) 111 111 + payload := &millv1.Event_StatusEvent{StatusEvent: &millv1.StatusEvent{ 112 112 + Status: millv1.NonterminalStatus_RUNNING, 113 113 + Error: errStr, 114 114 + ExitCode: exit, 115 115 + }} 116 116 + return e.appendAndSend(leaseID, payload, true) 117 117 + } 118 118 + 119 119 + func (e *Executor) appendTerminal(leaseID, status string, st *tangled.PipelineStatus) error { 120 120 + return e.appendTerminalWithArtifact(leaseID, status, st, "", "") 121 121 + } 122 122 + 123 123 + func (e *Executor) appendTerminalWithArtifact(leaseID, status string, st *tangled.PipelineStatus, ref, hash string) error { 124 124 + var terminalStatus millv1.TerminalStatus 125 125 + switch status { 126 126 + case string(models.StatusKindSuccess): 127 127 + terminalStatus = millv1.TerminalStatus_SUCCESS 128 128 + case string(models.StatusKindFailed): 129 129 + terminalStatus = millv1.TerminalStatus_FAILED 130 130 + case string(models.StatusKindTimeout): 131 131 + terminalStatus = millv1.TerminalStatus_TIMEOUT 132 132 + case string(models.StatusKindCancelled): 133 133 + terminalStatus = millv1.TerminalStatus_CANCELLED 134 134 + default: 135 135 + return fmt.Errorf("unsupported terminal status %q", status) 136 136 + } 137 137 + 138 138 + exit, errStr := parseStatusExitAndError(st) 139 139 + var logArtifact *millv1.LogArtifact 140 140 + if ref != "" { 141 141 + logArtifact = &millv1.LogArtifact{ 142 142 + Ref: ref, 143 143 + Hash: hash, 144 144 + } 145 145 + } 146 146 + payload := &millv1.Event_AttemptResult{AttemptResult: &millv1.AttemptResult{ 147 147 + Status: terminalStatus, 148 148 + Error: errStr, 149 149 + ExitCode: exit, 150 150 + LogArtifact: logArtifact, 151 151 + }} 152 152 + return e.appendAndSend(leaseID, payload, true) 153 153 + } 154 154 + 155 155 + func (e *Executor) recoverPendingArtifacts() error { 156 156 + if e.db == nil { 157 157 + return nil 158 158 + } 159 159 + pending, err := e.db.ListPendingArtifacts() 160 160 + if err != nil || len(pending) == 0 { 161 161 + return err 162 162 + } 163 163 + for _, p := range pending { 164 164 + if p.Ref != "" { 165 165 + if e.writer == nil { 166 166 + continue 167 167 + } 168 168 + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) 169 169 + var logDir string 170 170 + if e.cfg != nil { 171 171 + logDir = e.cfg.Server.LogDir 172 172 + } 173 173 + logPath := models.LogFilePath(logDir, models.WorkflowId{Name: p.Workflow}) 174 174 + f, openErr := os.Open(logPath) 175 175 + if openErr != nil { 176 176 + cancel() 177 177 + continue 178 178 + } 179 179 + uploadErr := e.writer.Put(ctx, p.Ref, f) 180 180 + _ = f.Close() 181 181 + cancel() 182 182 + if uploadErr != nil { 183 183 + continue 184 184 + } 185 185 + } 186 186 + 187 187 + st := &tangled.PipelineStatus{ 188 188 + Status: p.Status, 189 189 + Error: &p.Error, 190 190 + ExitCode: &p.ExitCode, 191 191 + } 192 192 + if err := e.appendTerminalWithArtifact(p.LeaseID, p.Status, st, p.Ref, p.Hash); err == nil { 193 193 + _ = e.db.RemovePendingArtifact(p.LeaseID) 194 194 + } 195 195 + } 196 196 + return nil 197 197 + } 198 198 + 199 199 + func truncateEventString(value string) string { 200 200 + if len(value) > 65536 { 201 201 + return value[:65536] 202 202 + } 203 203 + return value 204 204 + } 205 205 + 206 206 + func (e *Executor) sendPending() { 207 207 + e.connMu.Lock() 208 208 + enc := e.enc 209 209 + e.connMu.Unlock() 210 210 + 211 211 + if enc == nil { 212 212 + return 213 213 + } 214 214 + 215 215 + e.flushMu.Lock() 216 216 + defer e.flushMu.Unlock() 217 217 + 218 218 + if err := e.sendPendingLocked(enc); err != nil { 219 219 + e.l.Error("send pending events failed", "err", err) 220 220 + } 221 221 + } 222 222 + 223 223 + func (e *Executor) sendPendingLocked(enc messageEncoder) error { 224 224 + for { 225 225 + rows, err := e.db.ListOutboxRowsAfter(e.sentSeqno, maxBatchEvents) 226 226 + if err != nil { 227 227 + return fmt.Errorf("list outbox rows after %d: %w", e.sentSeqno, err) 228 228 + } 229 229 + if len(rows) == 0 { 230 230 + return nil 231 231 + } 232 232 + if err := e.sendRows(enc, rows); err != nil { 233 233 + return err 234 234 + } 235 235 + } 236 236 + } 237 237 + 238 238 + func (e *Executor) sendRows(enc messageEncoder, rows []db.OutboxRow) error { 239 239 + events := make([]*millv1.Event, 0, len(rows)) 240 240 + var lastSeqno uint64 241 241 + 242 242 + for _, r := range rows { 243 243 + ev, err := decodeEvent(r.Payload, r.Seqno) 244 244 + if err != nil { 245 245 + return fmt.Errorf("decode event at seqno %d: %w", r.Seqno, err) 246 246 + } 247 247 + events = append(events, ev) 248 248 + lastSeqno = r.Seqno 249 249 + } 250 250 + 251 251 + batch := &millv1.EventBatch{ 252 252 + Epoch: e.epoch, 253 253 + Events: events, 254 254 + } 255 255 + 256 256 + e.sendMu.Lock() 257 257 + defer e.sendMu.Unlock() 258 258 + 259 259 + if err := enc.Encode(&millproto.Message{EventBatch: batch}); err != nil { 260 260 + return fmt.Errorf("encode event batch (seqno %d..%d): %w", rows[0].Seqno, lastSeqno, err) 261 261 + } 262 262 + e.sentSeqno = lastSeqno 263 263 + return nil 264 264 + } 265 265 + 266 266 + func (e *Executor) replay(ackSeqno uint64) error { 267 267 + e.connMu.Lock() 268 268 + enc := e.enc 269 269 + e.connMu.Unlock() 270 270 + if enc == nil { 271 271 + return nil 272 272 + } 273 273 + 274 274 + e.flushMu.Lock() 275 275 + defer e.flushMu.Unlock() 276 276 + 277 277 + e.sendMu.Lock() 278 278 + e.sentSeqno = ackSeqno 279 279 + e.sendMu.Unlock() 280 280 + 281 281 + return e.sendPendingLocked(enc) 282 282 + } 283 283 + 284 284 + func (e *Executor) handleAck(ack *millv1.Ack) { 285 285 + if ack == nil || ack.GetEpoch() != e.epoch { 286 286 + return 287 287 + } 288 288 + 289 289 + upTo := ack.GetUpToSeqno() 290 290 + if upTo == 0 { 291 291 + return 292 292 + } 293 293 + 294 294 + if err := e.deleteOutboxPrefix(upTo); err != nil { 295 295 + e.l.Error("delete outbox prefix failed", "upTo", upTo, "err", err) 296 296 + } 297 297 + } 298 298 + 299 299 + func (e *Executor) subtractOutboxBytes(deleted db.OutboxDeletion) { 300 300 + e.outboxBytes = max(0, e.outboxBytes-deleted.Bytes) 301 301 + } 302 302 + 303 303 + func parseStatus(raw json.RawMessage) (*tangled.PipelineStatus, bool) { 304 304 + var st tangled.PipelineStatus 305 305 + if err := json.Unmarshal(raw, &st); err != nil { 306 306 + return nil, false 307 307 + } 308 308 + return &st, true 309 309 + } 310 310 + 311 311 + func (e *Executor) deleteOutboxPrefix(upTo uint64) error { 312 312 + e.eventMu.Lock() 313 313 + defer e.eventMu.Unlock() 314 314 + 315 315 + deleted, err := e.db.DeleteOutboxPrefix(upTo) 316 316 + if err == nil { 317 317 + e.subtractOutboxBytes(deleted) 318 318 + } 319 319 + return err 320 320 + } 321 321 + 322 322 + func parseStatusExitAndError(st *tangled.PipelineStatus) (int64, string) { 323 323 + if st == nil { 324 324 + return 0, "" 325 325 + } 326 326 + var exit int64 327 327 + if st.ExitCode != nil { 328 328 + exit = *st.ExitCode 329 329 + } 330 330 + var errStr string 331 331 + if st.Error != nil { 332 332 + errStr = truncateEventString(*st.Error) 333 333 + } 334 334 + return exit, errStr 335 335 + } 336 336 + 337 337 + func decodeEvent(payload []byte, seqno uint64) (*millv1.Event, error) { 338 338 + var ev millv1.Event 339 339 + if err := proto.Unmarshal(payload, &ev); err != nil { 340 340 + return nil, err 341 341 + } 342 342 + ev.Seqno = seqno 343 343 + return &ev, nil 344 344 + }

+37

spindle/mill/executor/reserved.go

Reviewed

··· 1 1 + package executor 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "fmt" 6 6 + "sync" 7 7 + 8 8 + "tangled.org/core/spindle/engine" 9 9 + "tangled.org/core/spindle/models" 10 10 + ) 11 11 + 12 12 + // wraps a real engine so StartWorkflows gets the slot ReserveSeat already 13 13 + // acquired, not a second one. everything else delegates, the execution 14 14 + // path runs exactly like standalone 15 15 + type reservedEngine struct { 16 16 + models.Engine 17 17 + slot engine.WorkflowSlot 18 18 + once sync.Once 19 19 + } 20 20 + 21 21 + func newReservedEngine(inner models.Engine, slot engine.WorkflowSlot) models.Engine { 22 22 + return &reservedEngine{Engine: inner, slot: slot} 23 23 + } 24 24 + 25 25 + // hands back the pre-acquired slot exactly once, a second acquire would 26 26 + // double-count it 27 27 + func (e *reservedEngine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) { 28 28 + var slot engine.WorkflowSlot 29 29 + e.once.Do(func() { 30 30 + slot = e.slot 31 31 + e.slot = nil 32 32 + }) 33 33 + if slot == nil { 34 34 + return nil, fmt.Errorf("reserved slot already consumed") 35 35 + } 36 36 + return slot, nil 37 37 + }

+546

spindle/mill/executor/reserved_test.go

Reviewed

··· 1 1 + package executor 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "encoding/json" 6 6 + "github.com/bluesky-social/indigo/atproto/syntax" 7 7 + "github.com/gorilla/websocket" 8 8 + "google.golang.org/protobuf/proto" 9 9 + "io" 10 10 + "log/slog" 11 11 + "net/http" 12 12 + "net/http/httptest" 13 13 + "path/filepath" 14 14 + "strings" 15 15 + "testing" 16 16 + "time" 17 17 + 18 18 + "tangled.org/core/api/tangled" 19 19 + "tangled.org/core/notifier" 20 20 + "tangled.org/core/spindle/config" 21 21 + "tangled.org/core/spindle/db" 22 22 + "tangled.org/core/spindle/engine" 23 23 + millproto "tangled.org/core/spindle/mill/proto" 24 24 + millv1 "tangled.org/core/spindle/mill/proto/gen" 25 25 + "tangled.org/core/spindle/models" 26 26 + "tangled.org/core/spindle/secrets" 27 27 + ) 28 28 + 29 29 + type captureEncoder struct { 30 30 + messages chan *millproto.Message 31 31 + } 32 32 + 33 33 + func newCaptureEncoder() *captureEncoder { 34 34 + return &captureEncoder{messages: make(chan *millproto.Message, 4)} 35 35 + } 36 36 + 37 37 + func (e *captureEncoder) Encode(msg *millproto.Message) error { 38 38 + e.messages <- msg 39 39 + return nil 40 40 + } 41 41 + 42 42 + type fakeSlot struct{ released int } 43 43 + 44 44 + func (s *fakeSlot) Release() { s.released++ } 45 45 + 46 46 + type fakeEngine struct { 47 47 + setupCalled bool 48 48 + runCalled bool 49 49 + destroyCalled bool 50 50 + acquireCalled bool 51 51 + secrets chan []secrets.UnlockedSecret 52 52 + done chan struct{} 53 53 + } 54 54 + 55 55 + func (e *fakeEngine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { 56 56 + return &models.Workflow{Name: twf.Name}, nil 57 57 + } 58 58 + func (e *fakeEngine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, l models.WorkflowLogger) error { 59 59 + e.setupCalled = true 60 60 + return nil 61 61 + } 62 62 + func (e *fakeEngine) WorkflowTimeout() time.Duration { return 7 * time.Minute } 63 63 + func (e *fakeEngine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { 64 64 + e.destroyCalled = true 65 65 + return nil 66 66 + } 67 67 + func (e *fakeEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, s []secrets.UnlockedSecret, l models.WorkflowLogger) error { 68 68 + e.runCalled = true 69 69 + if e.secrets != nil { 70 70 + e.secrets <- s 71 71 + } 72 72 + if e.done != nil { 73 73 + close(e.done) 74 74 + } 75 75 + return nil 76 76 + } 77 77 + func (e *fakeEngine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode engine.AcquireMode) (engine.WorkflowSlot, error) { 78 78 + e.acquireCalled = true 79 79 + return &fakeSlot{}, nil 80 80 + } 81 81 + 82 82 + type fakeStep struct{} 83 83 + 84 84 + func (fakeStep) Name() string { return "test" } 85 85 + func (fakeStep) Command() string { return "true" } 86 86 + func (fakeStep) Kind() models.StepKind { return models.StepKindUser } 87 87 + 88 88 + func TestNewFailsWhenOutboxCannotInitialize(t *testing.T) { 89 89 + d := testDB(t) 90 90 + if err := d.Close(); err != nil { 91 91 + t.Fatal(err) 92 92 + } 93 93 + n := notifier.New() 94 94 + cfg := &config.Config{} 95 95 + if _, err := New(cfg, nil, d, &n, slog.New(slog.NewTextHandler(io.Discard, nil))); err == nil { 96 96 + t.Fatal("New succeeded with an unavailable outbox database") 97 97 + } 98 98 + } 99 99 + 100 100 + func TestReservedEngineHandsBackHeldSlotOnce(t *testing.T) { 101 101 + inner := &fakeEngine{} 102 102 + slot := &fakeSlot{} 103 103 + re := newReservedEngine(inner, slot) 104 104 + 105 105 + got, err := re.(engine.WorkflowSlotter).AcquireWorkflowSlot(context.Background(), models.WorkflowId{}, nil, engine.Wait) 106 106 + if err != nil { 107 107 + t.Fatal(err) 108 108 + } 109 109 + if got != engine.WorkflowSlot(slot) { 110 110 + t.Fatal("AcquireWorkflowSlot() did not return the held slot") 111 111 + } 112 112 + if inner.acquireCalled { 113 113 + t.Fatal("wrapper must not call the inner engine's AcquireWorkflowSlot") 114 114 + } 115 115 + 116 116 + if _, err := re.(engine.WorkflowSlotter).AcquireWorkflowSlot(context.Background(), models.WorkflowId{}, nil, engine.Wait); err == nil { 117 117 + t.Fatal("second AcquireWorkflowSlot() should error") 118 118 + } 119 119 + } 120 120 + 121 121 + func TestHandleCommitIsIdempotent(t *testing.T) { 122 122 + enc := newCaptureEncoder() 123 123 + e := testExecutor(t) 124 124 + e.enc = enc 125 125 + e.active["lease-1"] = &reservation{leaseID: "lease-1", committed: true} 126 126 + 127 127 + e.handleCommit(context.Background(), &millv1.CommitLease{LeaseId: "lease-1"}) 128 128 + msg := <-enc.messages 129 129 + if got := msg.GetCommitted().GetLeaseId(); got != "lease-1" { 130 130 + t.Fatalf("Committed lease = %q, want lease-1", got) 131 131 + } 132 132 + } 133 133 + 134 134 + func TestHandleCommitRejectsMissingReservation(t *testing.T) { 135 135 + enc := newCaptureEncoder() 136 136 + e := testExecutor(t) 137 137 + e.enc = enc 138 138 + 139 139 + e.handleCommit(context.Background(), &millv1.CommitLease{LeaseId: "expired"}) 140 140 + result := (<-enc.messages).GetReserveResult() 141 141 + if result == nil { 142 142 + t.Fatal("missing reservation commit did not receive a ReserveResult") 143 143 + } 144 144 + if result.GetLeaseId() != "expired" || result.GetAccepted() { 145 145 + t.Fatalf("ReserveResult = %+v, want correlated rejection", result) 146 146 + } 147 147 + } 148 148 + 149 149 + func TestHandleCancelFinalizesExpiredReservation(t *testing.T) { 150 150 + enc := newCaptureEncoder() 151 151 + e := testExecutor(t) 152 152 + e.enc = enc 153 153 + 154 154 + e.handleCancel("lease-expired") 155 155 + var ack *millv1.CancelAck 156 156 + for ack == nil { 157 157 + select { 158 158 + case msg := <-enc.messages: 159 159 + ack = msg.GetCancelAck() 160 160 + case <-time.After(time.Second): 161 161 + t.Fatal("cancel acknowledgement timed out") 162 162 + } 163 163 + } 164 164 + if ack.GetLeaseId() != "lease-expired" { 165 165 + t.Fatalf("CancelAck = %+v, want lease-expired", ack) 166 166 + } 167 167 + rows, err := e.db.ListOutboxRows() 168 168 + if err != nil { 169 169 + t.Fatal(err) 170 170 + } 171 171 + if len(rows) != 1 { 172 172 + t.Fatalf("cancel terminal outbox rows = %d, want 1", len(rows)) 173 173 + } 174 174 + var entry millv1.Event 175 175 + if err := proto.Unmarshal(rows[0].Payload, &entry); err != nil { 176 176 + t.Fatal(err) 177 177 + } 178 178 + if got := entry.GetAttemptResult().GetStatus(); got != millv1.TerminalStatus_CANCELLED { 179 179 + t.Fatalf("cancel terminal = %v, want CANCELLED", got) 180 180 + } 181 181 + } 182 182 + 183 183 + func TestHandleCommitPreservesPreauthorizedSecrets(t *testing.T) { 184 184 + d := testDB(t) 185 185 + n := notifier.New() 186 186 + enc := newCaptureEncoder() 187 187 + e := &Executor{ 188 188 + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 189 189 + db: d, 190 190 + n: &n, 191 191 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 192 192 + active: make(map[string]*reservation), 193 193 + maxOutboxBytes: 10 * 1024 * 1024, 194 194 + enc: enc, 195 195 + } 196 196 + if err := e.initOutbox(); err != nil { 197 197 + t.Fatal(err) 198 198 + } 199 199 + e.lifecycleCtx = context.Background() 200 200 + 201 201 + inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})} 202 202 + slot := &fakeSlot{} 203 203 + repoDid, err := syntax.ParseDID("did:web:example.com") 204 204 + if err != nil { 205 205 + t.Fatal(err) 206 206 + } 207 207 + res := &reservation{ 208 208 + leaseID: "lease-1", 209 209 + wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, 210 210 + realEngine: inner, 211 211 + slot: slot, 212 212 + wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, 213 213 + repoDid: repoDid, 214 214 + } 215 215 + e.active[res.leaseID] = res 216 216 + 217 217 + e.handleCommit(context.Background(), &millv1.CommitLease{ 218 218 + LeaseId: res.leaseID, 219 219 + Secrets: []*millv1.Secret{{Key: "TOKEN", Value: "secret-value"}}, 220 220 + }) 221 221 + if got := (<-enc.messages).GetCommitted().GetLeaseId(); got != res.leaseID { 222 222 + t.Fatalf("Committed lease = %q, want %q", got, res.leaseID) 223 223 + } 224 224 + select { 225 225 + case got := <-inner.secrets: 226 226 + if len(got) != 1 || got[0].Key != "TOKEN" || got[0].Value != "secret-value" { 227 227 + t.Fatalf("RunStep secrets = %+v", got) 228 228 + } 229 229 + case <-time.After(2 * time.Second): 230 230 + t.Fatal("RunStep did not receive CommitLease secrets") 231 231 + } 232 232 + select { 233 233 + case <-inner.done: 234 234 + case <-time.After(2 * time.Second): 235 235 + t.Fatal("workflow did not finish") 236 236 + } 237 237 + if res.stopTail != nil { 238 238 + res.stopTail() 239 239 + } 240 240 + e.jobsWG.Wait() 241 241 + } 242 242 + 243 243 + func TestRunSessionCancellationClosesStalledWebsocket(t *testing.T) { 244 244 + connected := make(chan struct{}) 245 245 + release := make(chan struct{}) 246 246 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 247 247 + conn, err := websocket.Upgrade(w, r, nil, 1024, 1024) 248 248 + if err != nil { 249 249 + return 250 250 + } 251 251 + defer conn.Close() 252 252 + close(connected) 253 253 + <-release 254 254 + })) 255 255 + t.Cleanup(func() { 256 256 + close(release) 257 257 + srv.Close() 258 258 + }) 259 259 + 260 260 + e := testSessionExecutor(t, "ws"+strings.TrimPrefix(srv.URL, "http")) 261 261 + ctx, cancel := context.WithCancel(context.Background()) 262 262 + done := make(chan error, 1) 263 263 + go func() { done <- e.runSession(ctx) }() 264 264 + <-connected 265 265 + cancel() 266 266 + 267 267 + select { 268 268 + case <-done: 269 269 + case <-time.After(2 * time.Second): 270 270 + t.Fatal("runSession did not return after context cancellation") 271 271 + } 272 272 + } 273 273 + 274 274 + func testSessionExecutor(t *testing.T, url string) *Executor { 275 275 + d := testDB(t) 276 276 + e := &Executor{ 277 277 + millURL: url, 278 278 + seats: 1, 279 279 + engines: make(map[string]models.Engine), 280 280 + db: d, 281 281 + cfg: &config.Config{Server: config.Server{Dev: true}}, 282 282 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 283 283 + active: make(map[string]*reservation), 284 284 + maxOutboxBytes: 10 * 1024 * 1024, 285 285 + } 286 286 + if err := e.initOutbox(); err != nil { 287 287 + t.Fatal(err) 288 288 + } 289 289 + return e 290 290 + } 291 291 + 292 292 + func testExecutor(t *testing.T) *Executor { 293 293 + d := testDB(t) 294 294 + e := &Executor{ 295 295 + db: d, 296 296 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 297 297 + active: make(map[string]*reservation), 298 298 + maxOutboxBytes: 10 * 1024 * 1024, 299 299 + } 300 300 + if err := e.initOutbox(); err != nil { 301 301 + t.Fatal(err) 302 302 + } 303 303 + return e 304 304 + } 305 305 + 306 306 + func testDB(t *testing.T) *db.DB { 307 307 + d, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "spindle.db")) 308 308 + if err != nil { 309 309 + t.Fatal(err) 310 310 + } 311 311 + return d 312 312 + } 313 313 + 314 314 + func TestFinishJobReportsCancelledReservationAsCancelled(t *testing.T) { 315 315 + d := testDB(t) 316 316 + res := &reservation{ 317 317 + leaseID: "lease-1", 318 318 + wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, 319 319 + cancelled: true, 320 320 + } 321 321 + e := &Executor{ 322 322 + db: d, 323 323 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 324 324 + active: map[string]*reservation{res.leaseID: res}, 325 325 + maxOutboxBytes: 10 * 1024 * 1024, 326 326 + } 327 327 + if err := e.initOutbox(); err != nil { 328 328 + t.Fatal(err) 329 329 + } 330 330 + 331 331 + e.finishJob(res, &tangled.PipelineStatus{ 332 332 + Pipeline: string(res.wid.PipelineId.AtUri()), 333 333 + Workflow: res.wid.Name, 334 334 + Status: string(models.StatusKindFailed), 335 335 + }) 336 336 + 337 337 + rows, err := d.ListOutboxRows() 338 338 + if err != nil { 339 339 + t.Fatal(err) 340 340 + } 341 341 + if len(rows) != 1 { 342 342 + t.Fatalf("outbox rows = %d, want 1", len(rows)) 343 343 + } 344 344 + 345 345 + var entry millv1.Event 346 346 + if err := proto.Unmarshal(rows[0].Payload, &entry); err != nil { 347 347 + t.Fatal(err) 348 348 + } 349 349 + got := entry.GetAttemptResult().GetStatus() 350 350 + if got != millv1.TerminalStatus_CANCELLED { 351 351 + t.Fatalf("terminal status = %v, want CANCELLED", got) 352 352 + } 353 353 + } 354 354 + 355 355 + func TestReplayRejectsMalformedOutboxRow(t *testing.T) { 356 356 + d := testDB(t) 357 357 + e := &Executor{ 358 358 + db: d, 359 359 + enc: newCaptureEncoder(), 360 360 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 361 361 + } 362 362 + if err := e.initOutbox(); err != nil { 363 363 + t.Fatal(err) 364 364 + } 365 365 + if _, err := d.AppendOutboxRow([]byte("not protobuf"), true); err != nil { 366 366 + t.Fatal(err) 367 367 + } 368 368 + if err := e.replay(0); err == nil { 369 369 + t.Fatal("replay accepted a malformed row and would leave a permanent seqno gap") 370 370 + } 371 371 + } 372 372 + 373 373 + func TestSocketCancellationIndependence(t *testing.T) { 374 374 + d := testDB(t) 375 375 + n := notifier.New() 376 376 + e := &Executor{ 377 377 + db: d, 378 378 + n: &n, 379 379 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 380 380 + active: make(map[string]*reservation), 381 381 + maxOutboxBytes: 10 * 1024 * 1024, 382 382 + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 383 383 + } 384 384 + if err := e.initOutbox(); err != nil { 385 385 + t.Fatal(err) 386 386 + } 387 387 + 388 388 + lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background()) 389 389 + defer cancelLifecycle() 390 390 + e.lifecycleCtx = lifecycleCtx 391 391 + 392 392 + inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})} 393 393 + slot := &fakeSlot{} 394 394 + repoDid, err := syntax.ParseDID("did:web:example.com") 395 395 + if err != nil { 396 396 + t.Fatal(err) 397 397 + } 398 398 + res := &reservation{ 399 399 + leaseID: "lease-1", 400 400 + wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, 401 401 + realEngine: inner, 402 402 + slot: slot, 403 403 + wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, 404 404 + repoDid: repoDid, 405 405 + } 406 406 + e.active[res.leaseID] = res 407 407 + 408 408 + sessionCtx, cancelSession := context.WithCancel(lifecycleCtx) 409 409 + 410 410 + e.handleCommit(sessionCtx, &millv1.CommitLease{ 411 411 + LeaseId: "lease-1", 412 412 + }) 413 413 + 414 414 + cancelSession() 415 415 + 416 416 + // session disconnect must not cancel the running job 417 417 + select { 418 418 + case <-inner.done: 419 419 + case <-time.After(2 * time.Second): 420 420 + t.Fatal("workflow did not complete even though websocket session was cancelled") 421 421 + } 422 422 + 423 423 + e.jobsWG.Wait() 424 424 + } 425 425 + 426 426 + func TestMonotonicSnapshots(t *testing.T) { 427 427 + enc := newCaptureEncoder() 428 428 + e := &Executor{ 429 429 + enc: enc, 430 430 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 431 431 + active: make(map[string]*reservation), 432 432 + maxOutboxBytes: 10 * 1024 * 1024, 433 433 + } 434 434 + 435 435 + e.pushSnapshot() 436 436 + msg1 := <-enc.messages 437 437 + seq1 := msg1.GetNodeSnapshot().GetSeqno() 438 438 + if seq1 != 1 { 439 439 + t.Fatalf("first seq = %d, want 1", seq1) 440 440 + } 441 441 + 442 442 + e.pushSnapshot() 443 443 + msg2 := <-enc.messages 444 444 + seq2 := msg2.GetNodeSnapshot().GetSeqno() 445 445 + if seq2 != 2 { 446 446 + t.Fatalf("second seq = %d, want 2", seq2) 447 447 + } 448 448 + } 449 449 + 450 450 + func TestTimerRace(t *testing.T) { 451 451 + d := testDB(t) 452 452 + e := &Executor{ 453 453 + db: d, 454 454 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 455 455 + active: make(map[string]*reservation), 456 456 + maxOutboxBytes: 10 * 1024 * 1024, 457 457 + } 458 458 + if err := e.initOutbox(); err != nil { 459 459 + t.Fatal(err) 460 460 + } 461 461 + 462 462 + twf, _ := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) 463 463 + tpl, _ := json.Marshal(tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) 464 464 + 465 465 + inner := &fakeEngine{} 466 466 + e.engines = map[string]models.Engine{"microvm": inner} 467 467 + 468 468 + e.handleReserve(context.Background(), &millv1.ReserveSeat{ 469 469 + LeaseId: "lease-1", 470 470 + TargetEngine: "microvm", 471 471 + RawWorkflowJson: string(twf), 472 472 + RawPipelineJson: string(tpl), 473 473 + TtlSeconds: 1, 474 474 + }) 475 475 + 476 476 + e.mu.Lock() 477 477 + res := e.active["lease-1"] 478 478 + e.mu.Unlock() 479 479 + 480 480 + if res == nil { 481 481 + t.Fatal("reservation was not added") 482 482 + } 483 483 + 484 484 + deadline := time.Now().Add(5 * time.Second) 485 485 + for { 486 486 + e.mu.Lock() 487 487 + activeLen := len(e.active) 488 488 + e.mu.Unlock() 489 489 + if activeLen == 0 { 490 490 + break 491 491 + } 492 492 + if time.Now().After(deadline) { 493 493 + t.Fatal("reservation was leaked and never expired") 494 494 + } 495 495 + time.Sleep(10 * time.Millisecond) 496 496 + } 497 497 + } 498 498 + 499 499 + func TestStructuredShutdown(t *testing.T) { 500 500 + d := testDB(t) 501 501 + n := notifier.New() 502 502 + e := &Executor{ 503 503 + db: d, 504 504 + n: &n, 505 505 + l: slog.New(slog.NewTextHandler(io.Discard, nil)), 506 506 + active: make(map[string]*reservation), 507 507 + maxOutboxBytes: 10 * 1024 * 1024, 508 508 + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 509 509 + } 510 510 + if err := e.initOutbox(); err != nil { 511 511 + t.Fatal(err) 512 512 + } 513 513 + 514 514 + ctx, cancel := context.WithCancel(context.Background()) 515 515 + e.lifecycleCtx = ctx 516 516 + 517 517 + inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})} 518 518 + slot := &fakeSlot{} 519 519 + repoDid, err := syntax.ParseDID("did:web:example.com") 520 520 + if err != nil { 521 521 + t.Fatal(err) 522 522 + } 523 523 + res := &reservation{ 524 524 + leaseID: "lease-1", 525 525 + wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, 526 526 + realEngine: inner, 527 527 + slot: slot, 528 528 + wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, 529 529 + repoDid: repoDid, 530 530 + } 531 531 + e.active[res.leaseID] = res 532 532 + 533 533 + e.handleCommit(ctx, &millv1.CommitLease{ 534 534 + LeaseId: "lease-1", 535 535 + }) 536 536 + 537 537 + cancel() 538 538 + 539 539 + e.jobsWG.Wait() 540 540 + 541 541 + select { 542 542 + case <-inner.done: 543 543 + default: 544 544 + t.Fatal("shutdown returned but running job did not finish") 545 545 + } 546 546 + }

+183

spindle/mill/handler.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "github.com/gorilla/websocket" 5 5 + "io" 6 6 + "net/http" 7 7 + "slices" 8 8 + "strings" 9 9 + "sync" 10 10 + "time" 11 11 + 12 12 + millproto "tangled.org/core/spindle/mill/proto" 13 13 + millv1 "tangled.org/core/spindle/mill/proto/gen" 14 14 + ) 15 15 + 16 16 + var ( 17 17 + handshakeSem = make(chan struct{}, 16) 18 18 + handshakeMu sync.Mutex 19 19 + inFlightHandshakes = make(map[string]struct{}) 20 20 + ) 21 21 + 22 22 + type livenessReader struct { 23 23 + r io.Reader 24 24 + conn *websocket.Conn 25 25 + readTimeout time.Duration 26 26 + } 27 27 + 28 28 + func (lr *livenessReader) Read(p []byte) (int, error) { 29 29 + if err := lr.conn.SetReadDeadline(time.Now().Add(lr.readTimeout)); err != nil { 30 30 + return 0, err 31 31 + } 32 32 + return lr.r.Read(p) 33 33 + } 34 34 + 35 35 + var upgrader = websocket.Upgrader{ 36 36 + ReadBufferSize: 1024, 37 37 + WriteBufferSize: 1024, 38 38 + } 39 39 + 40 40 + // auth before upgrade, a bad token never opens a socket 41 41 + func (m *Mill) HandleExecutorConn(w http.ResponseWriter, r *http.Request) { 42 42 + name, authorizedLabels, ok := m.authenticate(r) 43 43 + if !ok { 44 44 + http.Error(w, "unauthorized", http.StatusUnauthorized) 45 45 + return 46 46 + } 47 47 + 48 48 + select { 49 49 + case handshakeSem <- struct{}{}: 50 50 + case <-r.Context().Done(): 51 51 + return 52 52 + } 53 53 + handshakeSlotHeld := true 54 54 + defer func() { 55 55 + if handshakeSlotHeld { 56 56 + <-handshakeSem 57 57 + } 58 58 + }() 59 59 + 60 60 + // enforces one in-flight handshake and one live session at a time per identity 61 61 + handshakeMu.Lock() 62 62 + if _, ok := inFlightHandshakes[name]; ok { 63 63 + handshakeMu.Unlock() 64 64 + http.Error(w, "handshake already in progress", http.StatusConflict) 65 65 + return 66 66 + } 67 67 + m.mu.Lock() 68 68 + old, exists := m.sessions[name] 69 69 + isLive := exists && old.live(m.cfg.ReconnectGrace) 70 70 + m.mu.Unlock() 71 71 + if isLive { 72 72 + handshakeMu.Unlock() 73 73 + http.Error(w, "session already active", http.StatusConflict) 74 74 + return 75 75 + } 76 76 + inFlightHandshakes[name] = struct{}{} 77 77 + handshakeMu.Unlock() 78 78 + identityHandshakeHeld := true 79 79 + 80 80 + defer func() { 81 81 + if identityHandshakeHeld { 82 82 + handshakeMu.Lock() 83 83 + delete(inFlightHandshakes, name) 84 84 + handshakeMu.Unlock() 85 85 + } 86 86 + }() 87 87 + 88 88 + conn, err := upgrader.Upgrade(w, r, nil) 89 89 + if err != nil { 90 90 + m.l.Error("fleet ws upgrade failed", "err", err) 91 91 + return 92 92 + } 93 93 + defer conn.Close() 94 94 + 95 95 + if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { 96 96 + m.l.Error("failed to set pre-hello read deadline", "err", err) 97 97 + return 98 98 + } 99 99 + 100 100 + stream := millproto.NewWSStream(conn) 101 101 + enc := millproto.NewEncoder(stream) 102 102 + dec := millproto.NewDecoder(stream) 103 103 + 104 104 + hello, err := dec.Decode() 105 105 + if err != nil { 106 106 + m.l.Error("fleet read hello failed", "err", err) 107 107 + return 108 108 + } 109 109 + h := hello.GetHello() 110 110 + if h == nil { 111 111 + m.l.Error("fleet first frame was not hello") 112 112 + return 113 113 + } 114 114 + if h.GetProtocolVersion() != millproto.ProtocolVersion { 115 115 + m.l.Error("fleet protocol version mismatch", "got", h.GetProtocolVersion(), "want", millproto.ProtocolVersion) 116 116 + return 117 117 + } 118 118 + if h.GetEpoch() == "" { 119 119 + m.l.Error("fleet hello missing epoch") 120 120 + return 121 121 + } 122 122 + 123 123 + for _, l := range h.GetLabels() { 124 124 + if !slices.Contains(authorizedLabels, l) { 125 125 + m.l.Error("executor requested unauthorized label", "label", l, "authorized", authorizedLabels) 126 126 + return 127 127 + } 128 128 + } 129 129 + 130 130 + sess := newSession(name, h.GetEpoch(), authorizedLabels, enc, m.l) 131 131 + sess.closeTransport = conn.Close 132 132 + sess.labels = h.GetLabels() 133 133 + 134 134 + resume, ok := m.attachSession(sess) 135 135 + if !ok { 136 136 + m.l.Warn("rejecting duplicate live executor session", "node", name) 137 137 + return 138 138 + } 139 139 + m.l.Info("executor connected", "node", sess.nodeID, "arch", h.GetArch(), "labels", h.GetLabels(), "resume", resume) 140 140 + 141 141 + if err := sess.send(&millproto.Message{Resume: &millv1.Resume{Epoch: h.GetEpoch(), AckSeqno: resume}}); err != nil { 142 142 + m.l.Error("fleet send resume failed", "err", err) 143 143 + m.detachSession(sess) 144 144 + return 145 145 + } 146 146 + handshakeSlotHeld = false 147 147 + <-handshakeSem 148 148 + handshakeMu.Lock() 149 149 + delete(inFlightHandshakes, name) 150 150 + handshakeMu.Unlock() 151 151 + identityHandshakeHeld = false 152 152 + m.sessionReady(sess) 153 153 + 154 154 + readTimeout := m.cfg.ReconnectGrace 155 155 + if readTimeout <= 0 { 156 156 + readTimeout = 45 * time.Second 157 157 + } 158 158 + liveDec := millproto.NewDecoder(&livenessReader{r: stream, conn: conn, readTimeout: readTimeout}) 159 159 + 160 160 + if err := sess.readLoop(m, liveDec); err != nil { 161 161 + m.l.Debug("session read ended", "node", sess.nodeID, "err", err) 162 162 + } 163 163 + m.detachSession(sess) 164 164 + } 165 165 + 166 166 + // identity comes from the token hash. unknown or missing token fails closed 167 167 + func (m *Mill) authenticate(r *http.Request) (string, []string, bool) { 168 168 + const prefix = "Bearer " 169 169 + h := r.Header.Get("Authorization") 170 170 + if !strings.HasPrefix(h, prefix) { 171 171 + return "", nil, false 172 172 + } 173 173 + token := strings.TrimPrefix(h, prefix) 174 174 + if token == "" || m.db == nil { 175 175 + return "", nil, false 176 176 + } 177 177 + name, labels, ok, err := m.db.ResolveExecutorToken(HashToken(token)) 178 178 + if err != nil { 179 179 + m.l.Error("executor token lookup failed", "err", err) 180 180 + return "", nil, false 181 181 + } 182 182 + return name, labels, ok 183 183 + }

+345

spindle/mill/integration_test.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "encoding/json" 6 6 + "io" 7 7 + "log/slog" 8 8 + "net/http" 9 9 + "net/http/httptest" 10 10 + "os" 11 11 + "path/filepath" 12 12 + "strings" 13 13 + "testing" 14 14 + "time" 15 15 + 16 16 + "tangled.org/core/api/tangled" 17 17 + "tangled.org/core/notifier" 18 18 + "tangled.org/core/spindle/config" 19 19 + "tangled.org/core/spindle/db" 20 20 + "tangled.org/core/spindle/engines/dummy" 21 21 + "tangled.org/core/spindle/mill/executor" 22 22 + "tangled.org/core/spindle/models" 23 23 + ) 24 24 + 25 25 + func TestEndToEndDummyJob(t *testing.T) { 26 26 + ctx, cancel := context.WithCancel(context.Background()) 27 27 + defer cancel() 28 28 + l := slog.New(slog.NewTextHandler(io.Discard, nil)) 29 29 + 30 30 + millDir := t.TempDir() 31 31 + bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db")) 32 32 + if err != nil { 33 33 + t.Fatalf("mill db: %v", err) 34 34 + } 35 35 + bn := notifier.New() 36 36 + mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second}) 37 37 + mill.Attach(bdb, &bn) 38 38 + if err := bdb.AddExecutorToken("exec-1", HashToken("test-token"), nil, nil); err != nil { 39 39 + t.Fatalf("register executor token: %v", err) 40 40 + } 41 41 + 42 42 + srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn)) 43 43 + defer srv.Close() 44 44 + wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") 45 45 + 46 46 + execDir := t.TempDir() 47 47 + edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db")) 48 48 + if err != nil { 49 49 + t.Fatalf("exec db: %v", err) 50 50 + } 51 51 + en := notifier.New() 52 52 + cfg := &config.Config{} 53 53 + cfg.Server.Dev = true 54 54 + cfg.Server.LogDir = execDir 55 55 + cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") 56 56 + cfg.Server.Hostname = "exec-1" 57 57 + cfg.Mill.URL = wsURL 58 58 + cfg.Mill.Seats = 2 59 59 + cfg.Mill.SharedSecret = "test-token" 60 60 + 61 61 + dummyEng := dummy.New(l) 62 62 + dummyEng.StepDelay = 50 * time.Millisecond 63 63 + engines := map[string]models.Engine{"dummy": dummyEng} 64 64 + exec, err := executor.New(cfg, engines, edb, &en, l) 65 65 + if err != nil { 66 66 + t.Fatalf("executor.New: %v", err) 67 67 + } 68 68 + go exec.Connect(ctx) 69 69 + 70 70 + be := NewEngine("dummy", mill) 71 71 + twf := tangled.Pipeline_Workflow{ 72 72 + Name: "build", 73 73 + Raw: "steps:\n - name: hello\n command: echo hi\n", 74 74 + } 75 75 + wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) 76 76 + if err != nil { 77 77 + t.Fatalf("InitWorkflow: %v", err) 78 78 + } 79 79 + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} 80 80 + 81 81 + placeCtx, placeCancel := context.WithTimeout(ctx, 10*time.Second) 82 82 + defer placeCancel() 83 83 + 84 84 + slot, err := mill.place(placeCtx, "dummy", wid, wf) 85 85 + if err != nil { 86 86 + t.Fatalf("place: %v", err) 87 87 + } 88 88 + defer slot.Release() 89 89 + 90 90 + logPath := models.LogFilePath(millDir, wid) 91 91 + ch := bn.Subscribe() 92 92 + defer bn.Unsubscribe(ch) 93 93 + 94 94 + sawLogContent := make(chan bool, 1) 95 95 + go func() { 96 96 + ticker := time.NewTicker(20 * time.Millisecond) 97 97 + defer ticker.Stop() 98 98 + timeout := time.After(10 * time.Second) 99 99 + for { 100 100 + select { 101 101 + case <-ch: 102 102 + case <-ticker.C: 103 103 + case <-timeout: 104 104 + sawLogContent <- false 105 105 + return 106 106 + } 107 107 + data, err := os.ReadFile(logPath) 108 108 + if err == nil && strings.Contains(string(data), "echo hi") { 109 109 + sawLogContent <- true 110 110 + return 111 111 + } 112 112 + } 113 113 + }() 114 114 + 115 115 + if err := mill.commitAndWait(placeCtx, wf, nil); err != nil { 116 116 + t.Fatalf("commitAndWait: %v, want success", err) 117 117 + } 118 118 + 119 119 + if !<-sawLogContent { 120 120 + t.Fatal("mill live log file never received expected content while job was running") 121 121 + } 122 122 + 123 123 + if !waitForFileRemoval(t, logPath) { 124 124 + t.Fatalf("mill live log file was not removed after terminal artifact was recorded") 125 125 + } 126 126 + 127 127 + if !waitForStatus(t, bdb, wid, "running") { 128 128 + events, _ := bdb.GetEvents(0, 1000) 129 129 + t.Logf("mill events after completion: %+v", events) 130 130 + t.Fatal("mill never saw streamed running status") 131 131 + } 132 132 + } 133 133 + 134 134 + func TestExecutorConfiguredLabelsAreStoredOnSession(t *testing.T) { 135 135 + ctx, cancel := context.WithCancel(context.Background()) 136 136 + defer cancel() 137 137 + l := slog.New(slog.NewTextHandler(io.Discard, nil)) 138 138 + 139 139 + millDir := t.TempDir() 140 140 + bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db")) 141 141 + if err != nil { 142 142 + t.Fatalf("mill db: %v", err) 143 143 + } 144 144 + bn := notifier.New() 145 145 + mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second}) 146 146 + mill.Attach(bdb, &bn) 147 147 + if err := bdb.AddExecutorToken("exec-labels", HashToken("test-token"), nil, []string{"linux", "arm64", "gpu"}); err != nil { 148 148 + t.Fatalf("register executor token: %v", err) 149 149 + } 150 150 + 151 151 + srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn)) 152 152 + defer srv.Close() 153 153 + wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") 154 154 + 155 155 + execDir := t.TempDir() 156 156 + edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db")) 157 157 + if err != nil { 158 158 + t.Fatalf("exec db: %v", err) 159 159 + } 160 160 + en := notifier.New() 161 161 + cfg := &config.Config{} 162 162 + cfg.Server.Dev = true 163 163 + cfg.Server.LogDir = execDir 164 164 + cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") 165 165 + cfg.Server.Hostname = "exec-labels" 166 166 + cfg.Mill.URL = wsURL 167 167 + cfg.Mill.Seats = 2 168 168 + cfg.Mill.SharedSecret = "test-token" 169 169 + cfg.Mill.Labels = []string{"linux", "arm64", "gpu"} 170 170 + 171 171 + engines := map[string]models.Engine{"dummy": dummy.New(l)} 172 172 + exec, err := executor.New(cfg, engines, edb, &en, l) 173 173 + if err != nil { 174 174 + t.Fatalf("executor.New: %v", err) 175 175 + } 176 176 + go exec.Connect(ctx) 177 177 + 178 178 + if !waitForSessionLabels(t, mill, "exec-labels", []string{"linux", "arm64", "gpu"}) { 179 179 + t.Fatal("mill session never stored executor labels from hello") 180 180 + } 181 181 + } 182 182 + 183 183 + func TestEndToEndDummyJobUsesRequiredLabelsAcrossExecutors(t *testing.T) { 184 184 + ctx, cancel := context.WithCancel(context.Background()) 185 185 + defer cancel() 186 186 + l := slog.New(slog.NewTextHandler(io.Discard, nil)) 187 187 + 188 188 + millDir := t.TempDir() 189 189 + bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db")) 190 190 + if err != nil { 191 191 + t.Fatalf("mill db: %v", err) 192 192 + } 193 193 + bn := notifier.New() 194 194 + mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second}) 195 195 + mill.Attach(bdb, &bn) 196 196 + if err := bdb.AddExecutorToken("exec-x86", HashToken("token-x86"), nil, []string{"linux/amd64", "kvm"}); err != nil { 197 197 + t.Fatalf("register x86 executor token: %v", err) 198 198 + } 199 199 + if err := bdb.AddExecutorToken("exec-arm", HashToken("token-arm"), nil, []string{"linux/arm64", "kvm"}); err != nil { 200 200 + t.Fatalf("register arm executor token: %v", err) 201 201 + } 202 202 + 203 203 + srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn)) 204 204 + defer srv.Close() 205 205 + wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") 206 206 + 207 207 + startExecutor := func(name, token string, labels []string) { 208 208 + t.Helper() 209 209 + execDir := t.TempDir() 210 210 + edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db")) 211 211 + if err != nil { 212 212 + t.Fatalf("%s exec db: %v", name, err) 213 213 + } 214 214 + en := notifier.New() 215 215 + cfg := &config.Config{} 216 216 + cfg.Server.Dev = true 217 217 + cfg.Server.LogDir = execDir 218 218 + cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") 219 219 + cfg.Server.Hostname = name 220 220 + cfg.Mill.URL = wsURL 221 221 + cfg.Mill.Seats = 1 222 222 + cfg.Mill.SharedSecret = token 223 223 + cfg.Mill.Labels = labels 224 224 + 225 225 + engines := map[string]models.Engine{"dummy": dummy.New(l)} 226 226 + exec, err := executor.New(cfg, engines, edb, &en, l) 227 227 + if err != nil { 228 228 + t.Fatalf("executor.New: %v", err) 229 229 + } 230 230 + go exec.Connect(ctx) 231 231 + } 232 232 + startExecutor("exec-x86", "token-x86", []string{"linux/amd64", "kvm"}) 233 233 + startExecutor("exec-arm", "token-arm", []string{"linux/arm64", "kvm"}) 234 234 + 235 235 + if !waitForSessionLabels(t, mill, "exec-x86", []string{"linux/amd64", "kvm"}) { 236 236 + t.Fatal("x86 executor did not connect with labels") 237 237 + } 238 238 + if !waitForSessionLabels(t, mill, "exec-arm", []string{"linux/arm64", "kvm"}) { 239 239 + t.Fatal("arm executor did not connect with labels") 240 240 + } 241 241 + 242 242 + be := NewEngine("dummy", mill) 243 243 + twf := tangled.Pipeline_Workflow{ 244 244 + Name: "build-arm", 245 245 + RunsOn: []string{"linux/arm64"}, 246 246 + Raw: "steps:\n - name: hello\n command: echo hi\n", 247 247 + } 248 248 + wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) 249 249 + if err != nil { 250 250 + t.Fatalf("InitWorkflow: %v", err) 251 251 + } 252 252 + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey-arm"}, Name: "build-arm"} 253 253 + 254 254 + placeCtx, placeCancel := context.WithTimeout(ctx, 10*time.Second) 255 255 + defer placeCancel() 256 256 + 257 257 + slot, err := mill.place(placeCtx, "dummy", wid, wf) 258 258 + if err != nil { 259 259 + t.Fatalf("place: %v", err) 260 260 + } 261 261 + defer slot.Release() 262 262 + 263 263 + lease := wf.Data.(*millWorkflowState).Lease 264 264 + if lease == nil { 265 265 + t.Fatal("place did not attach lease to workflow state") 266 266 + } 267 267 + if lease.nodeID != "exec-arm" { 268 268 + t.Fatalf("placed on %q, want exec-arm", lease.nodeID) 269 269 + } 270 270 + 271 271 + if err := mill.commitAndWait(placeCtx, wf, nil); err != nil { 272 272 + t.Fatalf("commitAndWait: %v, want success", err) 273 273 + } 274 274 + } 275 275 + 276 276 + func waitForSessionLabels(t *testing.T, m *Mill, nodeID string, want []string) bool { 277 277 + t.Helper() 278 278 + deadline := time.Now().Add(3 * time.Second) 279 279 + for time.Now().Before(deadline) { 280 280 + m.mu.Lock() 281 281 + sess := m.sessions[nodeID] 282 282 + var got []string 283 283 + if sess != nil { 284 284 + got = append([]string(nil), sess.labels...) 285 285 + } 286 286 + m.mu.Unlock() 287 287 + if sameStringMultiset(got, want) { 288 288 + return true 289 289 + } 290 290 + time.Sleep(10 * time.Millisecond) 291 291 + } 292 292 + return false 293 293 + } 294 294 + 295 295 + func waitForStatus(t *testing.T, d *db.DB, wid models.WorkflowId, want string) bool { 296 296 + t.Helper() 297 297 + deadline := time.Now().Add(3 * time.Second) 298 298 + aturi := string(wid.PipelineId.AtUri()) 299 299 + for time.Now().Before(deadline) { 300 300 + evs, err := d.GetEvents(0, 1000) 301 301 + if err != nil { 302 302 + t.Fatalf("GetEvents: %v", err) 303 303 + } 304 304 + for _, ev := range evs { 305 305 + if ev.Nsid != tangled.PipelineStatusNSID { 306 306 + continue 307 307 + } 308 308 + var st tangled.PipelineStatus 309 309 + if err := json.Unmarshal(ev.EventJson, &st); err != nil { 310 310 + continue 311 311 + } 312 312 + if st.Pipeline == aturi && st.Workflow == wid.Name && st.Status == want { 313 313 + return true 314 314 + } 315 315 + } 316 316 + time.Sleep(50 * time.Millisecond) 317 317 + } 318 318 + return false 319 319 + } 320 320 + 321 321 + func waitForLogFileContent(t *testing.T, path, want string) bool { 322 322 + t.Helper() 323 323 + deadline := time.Now().Add(5 * time.Second) 324 324 + for time.Now().Before(deadline) { 325 325 + data, err := os.ReadFile(path) 326 326 + if err == nil && strings.Contains(string(data), want) { 327 327 + return true 328 328 + } 329 329 + time.Sleep(20 * time.Millisecond) 330 330 + } 331 331 + return false 332 332 + } 333 333 + 334 334 + func waitForFileRemoval(t *testing.T, path string) bool { 335 335 + t.Helper() 336 336 + deadline := time.Now().Add(5 * time.Second) 337 337 + for time.Now().Before(deadline) { 338 338 + _, err := os.Stat(path) 339 339 + if os.IsNotExist(err) { 340 340 + return true 341 341 + } 342 342 + time.Sleep(20 * time.Millisecond) 343 343 + } 344 344 + return false 345 345 + }

+186

spindle/mill/lease.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "sync" 5 5 + 6 6 + "tangled.org/core/spindle/models" 7 7 + 8 8 + millv1 "tangled.org/core/spindle/mill/proto/gen" 9 9 + ) 10 10 + 11 11 + // the mill's view of a remote attempt 12 12 + type leaseState int32 13 13 + 14 14 + const ( 15 15 + // won a bid, executor is holding a seat, not yet committed 16 16 + leaseReserved leaseState = iota 17 17 + // CommitLease sent. the executor may already be running, but the mill 18 18 + // may not have seen Committed yet 19 19 + leaseCommitting 20 20 + // CommitLease sent and acked. job is running on the executor 21 21 + leaseRunning 22 22 + // terminal result arrived or we gave up, no further action 23 23 + leaseDone 24 24 + ) 25 25 + 26 26 + type cancelAction int 27 27 + 28 28 + const ( 29 29 + cancelNoop cancelAction = iota 30 30 + cancelLocal 31 31 + cancelRemote 32 32 + ) 33 33 + 34 34 + // mill-side fencing token for one placed job 35 35 + type RemoteLease struct { 36 36 + id string 37 37 + nodeID string 38 38 + epoch string 39 39 + engine string 40 40 + wid models.WorkflowId // job this lease carries, set once placed 41 41 + // restored after a mill restart. no RunStep waits on it, so terminals 42 42 + // and death are authored directly. set before publication, never mutated 43 43 + orphaned bool 44 44 + claimed bool 45 45 + cancelAcked bool 46 46 + cleanedUp bool 47 47 + cleanupRetry bool 48 48 + 49 49 + mu sync.Mutex 50 50 + state leaseState 51 51 + cancel bool 52 52 + reason string 53 53 + released bool 54 54 + terminal chan *millv1.AttemptResult // buffered(1), RunStep waits here 55 55 + 56 56 + finishMu sync.Mutex 57 57 + } 58 58 + 59 59 + func newLease(id, nodeID, epoch, engine string) *RemoteLease { 60 60 + return &RemoteLease{ 61 61 + id: id, 62 62 + nodeID: nodeID, 63 63 + epoch: epoch, 64 64 + engine: engine, 65 65 + state: leaseReserved, 66 66 + terminal: make(chan *millv1.AttemptResult, 1), 67 67 + } 68 68 + } 69 69 + 70 70 + func (l *RemoteLease) setState(s leaseState) { 71 71 + l.mu.Lock() 72 72 + l.state = s 73 73 + l.mu.Unlock() 74 74 + } 75 75 + 76 76 + func (l *RemoteLease) markCommitting() bool { 77 77 + l.mu.Lock() 78 78 + defer l.mu.Unlock() 79 79 + if l.state == leaseDone { 80 80 + return false 81 81 + } 82 82 + if l.state == leaseReserved { 83 83 + l.state = leaseCommitting 84 84 + } 85 85 + return true 86 86 + } 87 87 + 88 88 + func (l *RemoteLease) markRunning() { 89 89 + l.mu.Lock() 90 90 + defer l.mu.Unlock() 91 91 + if l.state != leaseDone { 92 92 + l.state = leaseRunning 93 93 + } 94 94 + } 95 95 + 96 96 + func (l *RemoteLease) getState() leaseState { 97 97 + l.mu.Lock() 98 98 + defer l.mu.Unlock() 99 99 + return l.state 100 100 + } 101 101 + 102 102 + // only one caller gets to mark it done 103 103 + func (l *RemoteLease) markDone() bool { 104 104 + l.mu.Lock() 105 105 + defer l.mu.Unlock() 106 106 + if l.state == leaseDone { 107 107 + return false 108 108 + } 109 109 + l.state = leaseDone 110 110 + return true 111 111 + } 112 112 + 113 113 + func (l *RemoteLease) requestCancel(reason string) cancelAction { 114 114 + l.mu.Lock() 115 115 + defer l.mu.Unlock() 116 116 + if l.state == leaseDone { 117 117 + return cancelNoop 118 118 + } 119 119 + l.cancel = true 120 120 + l.reason = reason 121 121 + if l.state == leaseReserved { 122 122 + return cancelLocal 123 123 + } 124 124 + return cancelRemote 125 125 + } 126 126 + 127 127 + func (l *RemoteLease) cancelRequested() (bool, string) { 128 128 + l.mu.Lock() 129 129 + defer l.mu.Unlock() 130 130 + return l.cancel, l.reason 131 131 + } 132 132 + 133 133 + func (l *RemoteLease) releaseState() (leaseState, bool) { 134 134 + l.mu.Lock() 135 135 + defer l.mu.Unlock() 136 136 + l.released = true 137 137 + return l.state, l.cancel 138 138 + } 139 139 + 140 140 + func (l *RemoteLease) cleanupReady() bool { 141 141 + l.mu.Lock() 142 142 + defer l.mu.Unlock() 143 143 + return l.released && l.state == leaseDone 144 144 + } 145 145 + 146 146 + func (l *RemoteLease) deliverCancelled(reason string) { 147 147 + l.deliverTerminal(&millv1.AttemptResult{ 148 148 + Status: millv1.TerminalStatus_CANCELLED, 149 149 + Error: reason, 150 150 + }) 151 151 + } 152 152 + 153 153 + // hands the terminal to a waiting RunStep without blocking. duplicates 154 154 + // (eg. reconnect replays) just drop, the channel holds one and the lease 155 155 + // is already done 156 156 + func (l *RemoteLease) deliverTerminal(res *millv1.AttemptResult) { 157 157 + if !l.markDone() { 158 158 + return 159 159 + } 160 160 + select { 161 161 + case l.terminal <- res: 162 162 + default: 163 163 + } 164 164 + } 165 165 + 166 166 + // mill's synthetic single step. real steps run on the executor and the 167 167 + // mill never mirrors them 168 168 + type remoteStep struct{} 169 169 + 170 170 + func (remoteStep) Name() string { return "remote execution" } 171 171 + func (remoteStep) Command() string { return "" } 172 172 + func (remoteStep) Kind() models.StepKind { return models.StepKindSystem } 173 173 + 174 174 + // what AcquireWorkflowSlot returns. Release unwinds placement 175 175 + type millSlot struct { 176 176 + fleet *Mill 177 177 + lease *RemoteLease 178 178 + once sync.Once 179 179 + } 180 180 + 181 181 + func (s *millSlot) Release() { 182 182 + if s == nil { 183 183 + return 184 184 + } 185 185 + s.once.Do(func() { s.fleet.releaseSlot(s) }) 186 186 + }

+1119

spindle/mill/mill.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "encoding/json" 6 6 + "errors" 7 7 + "fmt" 8 8 + "log/slog" 9 9 + "os" 10 10 + "path/filepath" 11 11 + "slices" 12 12 + "strings" 13 13 + "sync" 14 14 + "time" 15 15 + 16 16 + "tangled.org/core/notifier" 17 17 + "tangled.org/core/spindle/db" 18 18 + "tangled.org/core/spindle/engine" 19 19 + "tangled.org/core/spindle/models" 20 20 + "tangled.org/core/spindle/secrets" 21 21 + "tangled.org/core/tid" 22 22 + 23 23 + millproto "tangled.org/core/spindle/mill/proto" 24 24 + millv1 "tangled.org/core/spindle/mill/proto/gen" 25 25 + ) 26 26 + 27 27 + const ( 28 28 + defaultReconnectGrace = 45 * time.Second 29 29 + defaultJobTimeout = 24 * time.Hour 30 30 + defaultBidTimeout = 5 * time.Second 31 31 + defaultTopK = 3 32 32 + defaultMaxPending = 100 33 33 + ) 34 34 + 35 35 + type Config struct { 36 36 + // mill appends live-tailed executor lines here so logview can follow running remote jobs. 37 37 + LogDir string 38 38 + MaxPending int 39 39 + ReconnectGrace time.Duration 40 40 + JobTimeout time.Duration 41 41 + BidTimeout time.Duration 42 42 + TopK int 43 43 + CancelTimeout time.Duration 44 44 + } 45 45 + 46 46 + type Mill struct { 47 47 + l *slog.Logger 48 48 + cfg Config 49 49 + 50 50 + db *db.DB 51 51 + n *notifier.Notifier 52 52 + 53 53 + mu sync.Mutex 54 54 + sessions map[string]*millSession 55 55 + leases map[string]*RemoteLease 56 56 + reservations map[string]*RemoteLease 57 57 + nodeSeqno map[string]uint64 58 58 + pending int 59 59 + changeCh chan struct{} // closed and replaced to wake placement waiters 60 60 + 61 61 + leaseSeq uint64 62 62 + } 63 63 + 64 64 + func New(l *slog.Logger, cfg Config) *Mill { 65 65 + if cfg.ReconnectGrace <= 0 { 66 66 + cfg.ReconnectGrace = defaultReconnectGrace 67 67 + } 68 68 + if cfg.JobTimeout <= 0 { 69 69 + cfg.JobTimeout = defaultJobTimeout 70 70 + } 71 71 + if cfg.BidTimeout <= 0 { 72 72 + cfg.BidTimeout = defaultBidTimeout 73 73 + } 74 74 + if cfg.TopK <= 0 { 75 75 + cfg.TopK = defaultTopK 76 76 + } 77 77 + if cfg.MaxPending <= 0 { 78 78 + cfg.MaxPending = defaultMaxPending 79 79 + } 80 80 + if cfg.CancelTimeout <= 0 { 81 81 + cfg.CancelTimeout = 10 * time.Second 82 82 + } 83 83 + return &Mill{ 84 84 + l: l, 85 85 + cfg: cfg, 86 86 + sessions: make(map[string]*millSession), 87 87 + leases: make(map[string]*RemoteLease), 88 88 + reservations: make(map[string]*RemoteLease), 89 89 + nodeSeqno: make(map[string]uint64), 90 90 + changeCh: make(chan struct{}), 91 91 + } 92 92 + } 93 93 + 94 94 + func (m *Mill) Attach(d *db.DB, n *notifier.Notifier) { 95 95 + m.mu.Lock() 96 96 + m.db = d 97 97 + m.n = n 98 98 + m.mu.Unlock() 99 99 + } 100 100 + 101 101 + func (m *Mill) nextLeaseID() string { 102 102 + m.mu.Lock() 103 103 + m.leaseSeq++ 104 104 + seq := m.leaseSeq 105 105 + m.mu.Unlock() 106 106 + return fmt.Sprintf("%s-%d", tid.TID(), seq) 107 107 + } 108 108 + 109 109 + func (m *Mill) dropReservation(id string) { 110 110 + m.mu.Lock() 111 111 + delete(m.reservations, id) 112 112 + m.mu.Unlock() 113 113 + } 114 114 + 115 115 + func (m *Mill) notifyChange() { 116 116 + m.mu.Lock() 117 117 + m.notifyChangeLocked() 118 118 + m.mu.Unlock() 119 119 + } 120 120 + 121 121 + func (m *Mill) notifyChangeLocked() { 122 122 + close(m.changeCh) 123 123 + m.changeCh = make(chan struct{}) 124 124 + } 125 125 + 126 126 + func (m *Mill) currentChangeCh() <-chan struct{} { 127 127 + m.mu.Lock() 128 128 + defer m.mu.Unlock() 129 129 + return m.changeCh 130 130 + } 131 131 + 132 132 + func (m *Mill) attachSession(sess *millSession) (uint64, bool) { 133 133 + m.mu.Lock() 134 134 + defer m.mu.Unlock() 135 135 + 136 136 + if old := m.sessions[sess.nodeID]; old != nil { 137 137 + if old.live(m.cfg.ReconnectGrace) { 138 138 + // a second live session for the same identity is a hijack 139 139 + // attempt, reject it 140 140 + return 0, false 141 141 + } 142 142 + if old.graceTimer != nil { 143 143 + old.graceTimer.Stop() 144 144 + } 145 145 + old.close() 146 146 + if old.disconnected { 147 147 + m.l.Info("executor reconnected", "node", sess.nodeID) 148 148 + } else { 149 149 + m.l.Warn("replacing silent executor session", "node", sess.nodeID) 150 150 + } 151 151 + } 152 152 + m.sessions[sess.nodeID] = sess 153 153 + // wakes commit retries waiting out reconnect grace 154 154 + m.notifyChangeLocked() 155 155 + return m.nodeSeqno[sess.nodeID+"/"+sess.epoch], true 156 156 + } 157 157 + 158 158 + func (m *Mill) touchSession(sess *millSession) bool { 159 159 + m.mu.Lock() 160 160 + defer m.mu.Unlock() 161 161 + if m.sessions[sess.nodeID] != sess || sess.disconnected { 162 162 + return false 163 163 + } 164 164 + sess.lastSeen = time.Now() 165 165 + return true 166 166 + } 167 167 + 168 168 + func (m *Mill) detachSession(sess *millSession) { 169 169 + m.mu.Lock() 170 170 + if m.sessions[sess.nodeID] != sess { 171 171 + // already replaced by a reconnect 172 172 + m.mu.Unlock() 173 173 + sess.close() 174 174 + return 175 175 + } 176 176 + sess.disconnected = true 177 177 + sess.graceTimer = time.AfterFunc(m.cfg.ReconnectGrace, func() { m.failLeasesAfterGrace(sess) }) 178 178 + m.mu.Unlock() 179 179 + 180 180 + sess.close() 181 181 + m.l.Warn("executor session lost; entering reconnect grace", "node", sess.nodeID, "grace", m.cfg.ReconnectGrace) 182 182 + m.notifyChange() 183 183 + } 184 184 + 185 185 + func (m *Mill) sessionReady(sess *millSession) { 186 186 + for _, lease := range m.cancelledLeasesForNode(sess.nodeID) { 187 187 + _, reason := lease.cancelRequested() 188 188 + m.sendCancel(sess, lease, reason) 189 189 + } 190 190 + m.notifyChange() 191 191 + } 192 192 + 193 193 + func (m *Mill) cancelledLeasesForNode(nodeID string) []*RemoteLease { 194 194 + m.mu.Lock() 195 195 + var candidates []*RemoteLease 196 196 + for _, lease := range m.leases { 197 197 + if lease.nodeID == nodeID { 198 198 + candidates = append(candidates, lease) 199 199 + } 200 200 + } 201 201 + m.mu.Unlock() 202 202 + 203 203 + var leases []*RemoteLease 204 204 + for _, lease := range candidates { 205 205 + if cancelled, _ := lease.cancelRequested(); cancelled && lease.getState() != leaseDone { 206 206 + leases = append(leases, lease) 207 207 + } 208 208 + } 209 209 + return leases 210 210 + } 211 211 + 212 212 + // reconnect grace ran out without the executor coming back. fails every 213 213 + // lease the node held, and reschedules itself if any failure didn't stick 214 214 + func (m *Mill) failLeasesAfterGrace(sess *millSession) { 215 215 + m.mu.Lock() 216 216 + if m.sessions[sess.nodeID] != sess || !sess.disconnected { 217 217 + // reconnected in the meantime 218 218 + m.mu.Unlock() 219 219 + return 220 220 + } 221 221 + var dead []*RemoteLease 222 222 + for _, lease := range m.leases { 223 223 + if lease.nodeID == sess.nodeID { 224 224 + dead = append(dead, lease) 225 225 + } 226 226 + } 227 227 + m.mu.Unlock() 228 228 + 229 229 + m.l.Warn("executor declared dead; failing its in-flight jobs", "node", sess.nodeID, "jobs", len(dead)) 230 230 + deadReason := "executor lost" 231 231 + success := true 232 232 + for _, lease := range dead { 233 233 + switch { 234 234 + case lease.orphaned: 235 235 + // restored leases have no RunStep waiting, fail them straight 236 236 + // into the event stream 237 237 + if err := m.finishOrphan(lease, string(models.StatusKindFailed), &deadReason, nil); err != nil { 238 238 + m.l.Error("finish orphan after executor loss failed, will retry", "lease", lease.id, "err", err) 239 239 + success = false 240 240 + } 241 241 + default: 242 242 + // live leases have a blocked RunStep, keep a pending cancellation 243 243 + // as the terminal reason 244 244 + status := string(models.StatusKindFailed) 245 245 + reason := deadReason 246 246 + if cancelled, cancelReason := lease.cancelRequested(); cancelled { 247 247 + status = string(models.StatusKindCancelled) 248 248 + reason = cancelReason 249 249 + } 250 250 + if err := m.finishLiveLease(lease, status, reason); err != nil { 251 251 + m.l.Error("finish lease after executor loss failed, will retry", "lease", lease.id, "err", err) 252 252 + success = false 253 253 + } 254 254 + } 255 255 + } 256 256 + 257 257 + m.mu.Lock() 258 258 + if !success { 259 259 + // something didn't finish cleanly, try again shortly. finished 260 260 + // leases skip themselves on the next pass 261 261 + sess.graceTimer = time.AfterFunc(5*time.Second, func() { m.failLeasesAfterGrace(sess) }) 262 262 + m.mu.Unlock() 263 263 + return 264 264 + } 265 265 + 266 266 + // everything failed cleanly. forget the node and wake placement 267 267 + if m.sessions[sess.nodeID] == sess { 268 268 + delete(m.sessions, sess.nodeID) 269 269 + } 270 270 + m.mu.Unlock() 271 271 + m.notifyChange() 272 272 + m.sweepUnclaimedOrphans() 273 273 + } 274 274 + 275 275 + func (m *Mill) place(ctx context.Context, engineName string, wid models.WorkflowId, wf *models.Workflow) (engine.WorkflowSlot, error) { 276 276 + m.mu.Lock() 277 277 + if m.cfg.MaxPending > 0 && m.pending >= m.cfg.MaxPending { 278 278 + max := m.cfg.MaxPending 279 279 + cur := m.pending 280 280 + m.mu.Unlock() 281 281 + return nil, fmt.Errorf("%w: mill has %d pending jobs (max %d)", engine.ErrNoWorkflowSlots, cur, max) 282 282 + } 283 283 + m.pending++ 284 284 + m.mu.Unlock() 285 285 + defer func() { 286 286 + m.mu.Lock() 287 287 + m.pending-- 288 288 + m.mu.Unlock() 289 289 + }() 290 290 + 291 291 + for { 292 292 + if err := ctx.Err(); err != nil { 293 293 + return nil, err 294 294 + } 295 295 + 296 296 + // grab the channel before bidding. a change mid-bid closes it, so 297 297 + // the wait below re-bids right away 298 298 + ch := m.currentChangeCh() 299 299 + 300 300 + lease, err := m.bid(ctx, engineName, wid, wf) 301 301 + if err != nil { 302 302 + return nil, err 303 303 + } 304 304 + if lease != nil { 305 305 + lease.wid = wid 306 306 + if err := m.persistLease(lease, leaseRowReserved); err != nil { 307 307 + m.releaseRemote(lease) 308 308 + m.mu.Lock() 309 309 + delete(m.reservations, lease.id) 310 310 + m.mu.Unlock() 311 311 + return nil, fmt.Errorf("persist reserved mill lease: %w", err) 312 312 + } 313 313 + m.mu.Lock() 314 314 + delete(m.reservations, lease.id) 315 315 + m.leases[lease.id] = lease 316 316 + if st, ok := wf.Data.(*millWorkflowState); ok && st != nil { 317 317 + st.Lease = lease 318 318 + } 319 319 + m.mu.Unlock() 320 320 + return &millSlot{fleet: m, lease: lease}, nil 321 321 + } 322 322 + 323 323 + // no executor available. wait for a change or ctx 324 324 + select { 325 325 + case <-ctx.Done(): 326 326 + return nil, ctx.Err() 327 327 + case <-ch: 328 328 + } 329 329 + } 330 330 + } 331 331 + 332 332 + func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId, wf *models.Workflow) (*RemoteLease, error) { 333 333 + rawPipeline, rawWorkflow, err := marshalJob(wf) 334 334 + if err != nil { 335 335 + return nil, err 336 336 + } 337 337 + 338 338 + requiredLabels := requiredLabels(wf) 339 339 + candidates := m.rankCandidates(engineName, requiredLabels) 340 340 + if len(candidates) == 0 { 341 341 + return nil, nil 342 342 + } 343 343 + 344 344 + type bidResult struct { 345 345 + sess *millSession 346 346 + lease *RemoteLease 347 347 + rank int 348 348 + incompatible bool 349 349 + reason string 350 350 + } 351 351 + limit := m.cfg.TopK 352 352 + if limit <= 0 { 353 353 + limit = len(candidates) 354 354 + } 355 355 + if len(candidates) > limit { 356 356 + candidates = candidates[:limit] 357 357 + } 358 358 + results := make(chan bidResult, limit) 359 359 + ask := func(rank int, sess *millSession) { 360 360 + bidCtx, cancel := context.WithTimeout(ctx, m.cfg.BidTimeout) 361 361 + defer cancel() 362 362 + leaseID := m.nextLeaseID() 363 363 + lease := newLease(leaseID, sess.nodeID, sess.epoch, engineName) 364 364 + m.mu.Lock() 365 365 + m.reservations[leaseID] = lease 366 366 + m.mu.Unlock() 367 367 + msg := &millproto.Message{ReserveSeat: &millv1.ReserveSeat{ 368 368 + LeaseId: leaseID, 369 369 + TargetEngine: engineName, 370 370 + RawPipelineJson: rawPipeline, 371 371 + RawWorkflowJson: rawWorkflow, 372 372 + Knot: wid.Knot, 373 373 + Rkey: wid.Rkey, 374 374 + TtlSeconds: uint32(m.cfg.ReconnectGrace / time.Second), 375 375 + }} 376 376 + resp, err := sess.request(bidCtx, leaseID, msg) 377 377 + if err != nil { 378 378 + m.dropReservation(leaseID) 379 379 + results <- bidResult{} 380 380 + return 381 381 + } 382 382 + rr := resp.GetReserveResult() 383 383 + if rr == nil { 384 384 + m.dropReservation(leaseID) 385 385 + results <- bidResult{} 386 386 + return 387 387 + } 388 388 + if !rr.GetAccepted() { 389 389 + m.dropReservation(leaseID) 390 390 + if rr.GetRejectClass() == millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE { 391 391 + results <- bidResult{sess: sess, rank: rank, incompatible: true, reason: rr.GetRejectReason()} 392 392 + return 393 393 + } 394 394 + results <- bidResult{} 395 395 + return 396 396 + } 397 397 + results <- bidResult{sess: sess, lease: lease, rank: rank} 398 398 + } 399 399 + next := 0 400 400 + inFlight := 0 401 401 + for next < len(candidates) && inFlight < limit { 402 402 + inFlight++ 403 403 + go ask(next, candidates[next]) 404 404 + next++ 405 405 + } 406 406 + 407 407 + var winner *bidResult 408 408 + var losers []*RemoteLease 409 409 + var incompatible []string 410 410 + // any soft reject (transient or timeout) means the fleet was just 411 411 + // busy, so an all-incompatible outcome isn't a hard placement error 412 412 + softReject := false 413 413 + for inFlight > 0 { 414 414 + r := <-results 415 415 + inFlight-- 416 416 + // incompatible rejects get reported to the user, any other failure 417 417 + // just means the fleet is busy 418 418 + if r.incompatible { 419 419 + if r.reason != "" { 420 420 + incompatible = append(incompatible, r.reason) 421 421 + } 422 422 + } else if r.lease == nil { 423 423 + softReject = true 424 424 + } 425 425 + // a failed bid means ask the next candidate, unless someone won 426 426 + if r.lease == nil { 427 427 + for winner == nil && next < len(candidates) && inFlight < limit { 428 428 + inFlight++ 429 429 + go ask(next, candidates[next]) 430 430 + next++ 431 431 + } 432 432 + continue 433 433 + } 434 434 + // if this bid is worse than the winner it goes to the losers pile 435 435 + if winner != nil && r.rank >= winner.rank { 436 436 + losers = append(losers, r.lease) 437 437 + continue 438 438 + } 439 439 + // otherwise it's the new best and the old winner joins the losers 440 440 + if winner != nil { 441 441 + losers = append(losers, winner.lease) 442 442 + } 443 443 + winner = &r 444 444 + } 445 445 + 446 446 + // let the losers go so they free their seats right away 447 447 + for _, l := range losers { 448 448 + m.dropReservation(l.id) 449 449 + m.releaseRemote(l) 450 450 + } 451 451 + 452 452 + if winner == nil { 453 453 + if len(incompatible) > 0 && !softReject { 454 454 + return nil, fmt.Errorf("no compatible executor for %s: %s", engineName, strings.Join(incompatible, "; ")) 455 455 + } 456 456 + return nil, nil 457 457 + } 458 458 + return winner.lease, nil 459 459 + } 460 460 + 461 461 + // ranks nodes that are least busy first. if a resource is used a lot 462 462 + // then that node will lose to one that is more even across the board. 463 463 + func (m *Mill) rankCandidates(engineName string, requiredLabels []string) []*millSession { 464 464 + m.mu.Lock() 465 465 + defer m.mu.Unlock() 466 466 + 467 467 + type ranked struct { 468 468 + sess *millSession 469 469 + worst float64 470 470 + sum float64 471 471 + } 472 472 + var rs []ranked 473 473 + for _, s := range m.sessions { 474 474 + // only live, reporting sessions can take work 475 475 + if s.disconnected { 476 476 + continue 477 477 + } 478 478 + if s.snapshot == nil { 479 479 + continue 480 480 + } 481 481 + // the engine has to exist and have room right now 482 482 + ea, ok := s.snapshot.GetEngines()[engineName] 483 483 + if !ok || !ea.GetAvailable() { 484 484 + continue 485 485 + } 486 486 + // and satisfy the wf's label requirements 487 487 + if !hasLabels(s.labels, requiredLabels) { 488 488 + continue 489 489 + } 490 490 + worst, sum := loadScore(ea.GetLoad()) 491 491 + rs = append(rs, ranked{sess: s, worst: worst, sum: sum}) 492 492 + } 493 493 + slices.SortStableFunc(rs, func(a, b ranked) int { 494 494 + if a.worst < b.worst { 495 495 + return -1 496 496 + } 497 497 + if a.worst > b.worst { 498 498 + return 1 499 499 + } 500 500 + if a.sum < b.sum { 501 501 + return -1 502 502 + } 503 503 + if a.sum > b.sum { 504 504 + return 1 505 505 + } 506 506 + return 0 507 507 + }) 508 508 + 509 509 + out := make([]*millSession, len(rs)) 510 510 + for i := range rs { 511 511 + out[i] = rs[i].sess 512 512 + } 513 513 + return out 514 514 + } 515 515 + 516 516 + func loadScore(load map[string]float64) (worst, sum float64) { 517 517 + for _, v := range load { 518 518 + if v > worst { 519 519 + worst = v 520 520 + } 521 521 + sum += v 522 522 + } 523 523 + return worst, sum 524 524 + } 525 525 + 526 526 + func requiredLabels(wf *models.Workflow) []string { 527 527 + st, ok := wf.Data.(*millWorkflowState) 528 528 + if !ok || st == nil { 529 529 + return nil 530 530 + } 531 531 + return st.RawWorkflow.RunsOn 532 532 + } 533 533 + 534 534 + func hasLabels(labels []string, required []string) bool { 535 535 + for _, want := range required { 536 536 + if !slices.Contains(labels, want) { 537 537 + return false 538 538 + } 539 539 + } 540 540 + return true 541 541 + } 542 542 + 543 543 + func (m *Mill) commitAndWait(ctx context.Context, wf *models.Workflow, unlocked []secrets.UnlockedSecret) error { 544 544 + st, ok := wf.Data.(*millWorkflowState) 545 545 + if !ok || st == nil || st.Lease == nil { 546 546 + return fmt.Errorf("mill workflow state missing lease") 547 547 + } 548 548 + lease := st.Lease 549 549 + 550 550 + pbSecrets := make([]*millv1.Secret, len(unlocked)) 551 551 + for i, s := range unlocked { 552 552 + pbSecrets[i] = &millv1.Secret{Key: s.Key, Value: s.Value} 553 553 + } 554 554 + 555 555 + commit := &millproto.Message{CommitLease: &millv1.CommitLease{ 556 556 + LeaseId: lease.id, 557 557 + Secrets: pbSecrets, 558 558 + }} 559 559 + 560 560 + // commit retries ride reconnects, a reservation outlives one 561 561 + // disconnect. lost session or slow executor just means wait and retry, 562 562 + // only job timeout or a dead lease stops the loop 563 563 + for { 564 564 + if res, ok := pollTerminal(lease); ok { 565 565 + return terminalError(res.Status) 566 566 + } 567 567 + if !lease.markCommitting() { 568 568 + return engine.ErrWorkflowFailed 569 569 + } 570 570 + 571 571 + sess := m.sessionForNode(lease.nodeID) 572 572 + if sess == nil { 573 573 + if done, err := m.waitCommitRetry(ctx, lease); done || err != nil { 574 574 + return err 575 575 + } 576 576 + continue 577 577 + } 578 578 + 579 579 + reqCtx, cancel := context.WithTimeout(ctx, m.cfg.BidTimeout) 580 580 + resp, err := sess.request(reqCtx, lease.id, commit) 581 581 + cancel() 582 582 + if err != nil { 583 583 + switch { 584 584 + case errors.Is(err, errSessionClosed): 585 585 + // session died mid-request. wait out the grace, then retry on 586 586 + // the new one 587 587 + if done, err := m.waitCommitRetry(ctx, lease); done || err != nil { 588 588 + return err 589 589 + } 590 590 + continue 591 591 + case errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil: 592 592 + // executor didn't answer in time, but its seat is still held so 593 593 + // retrying is safe 594 594 + continue 595 595 + case errors.Is(err, context.DeadlineExceeded): 596 596 + // the job ctx itself ran out, a real timeout 597 597 + return engine.ErrTimedOut 598 598 + case errors.Is(err, context.Canceled): 599 599 + return err 600 600 + default: 601 601 + m.l.Warn("commit lease send failed; waiting for reconnect", "lease", lease.id, "node", lease.nodeID, "err", err) 602 602 + if done, err := m.waitCommitRetry(ctx, lease); done || err != nil { 603 603 + return err 604 604 + } 605 605 + continue 606 606 + } 607 607 + } 608 608 + if resp.GetCommitted() == nil { 609 609 + return engine.ErrWorkflowFailed 610 610 + } 611 611 + lease.markRunning() 612 612 + if err := m.persistLease(lease, leaseRowRunning); err != nil { 613 613 + m.l.Error("persist running mill lease", "lease", lease.id, "err", err) 614 614 + } 615 615 + if cancelled, reason := lease.cancelRequested(); cancelled { 616 616 + m.sendCancel(sess, lease, reason) 617 617 + } 618 618 + break 619 619 + } 620 620 + 621 621 + select { 622 622 + case res := <-lease.terminal: 623 623 + return terminalError(res.Status) 624 624 + case <-ctx.Done(): 625 625 + if ctx.Err() == context.DeadlineExceeded { 626 626 + return engine.ErrTimedOut 627 627 + } 628 628 + return ctx.Err() 629 629 + } 630 630 + } 631 631 + 632 632 + func (m *Mill) waitCommitRetry(ctx context.Context, lease *RemoteLease) (bool, error) { 633 633 + // grab the channel before checking for a live session again 634 634 + // a reconnect will still close the channel if it happens in between 635 635 + ch := m.currentChangeCh() 636 636 + if m.sessionForNode(lease.nodeID) != nil { 637 637 + return false, nil 638 638 + } 639 639 + select { 640 640 + case res := <-lease.terminal: 641 641 + return true, terminalError(res.Status) 642 642 + case <-ctx.Done(): 643 643 + if ctx.Err() == context.DeadlineExceeded { 644 644 + return true, engine.ErrTimedOut 645 645 + } 646 646 + return true, ctx.Err() 647 647 + case <-ch: 648 648 + return false, nil 649 649 + } 650 650 + } 651 651 + 652 652 + func terminalError(status millv1.TerminalStatus) error { 653 653 + switch status { 654 654 + case millv1.TerminalStatus_SUCCESS: 655 655 + return nil 656 656 + case millv1.TerminalStatus_TIMEOUT: 657 657 + return engine.ErrTimedOut 658 658 + case millv1.TerminalStatus_CANCELLED: 659 659 + return engine.ErrWorkflowCanceled 660 660 + default: 661 661 + return engine.ErrWorkflowFailed 662 662 + } 663 663 + } 664 664 + 665 665 + func pollTerminal(lease *RemoteLease) (*millv1.AttemptResult, bool) { 666 666 + select { 667 667 + case res := <-lease.terminal: 668 668 + return res, true 669 669 + default: 670 670 + return nil, false 671 671 + } 672 672 + } 673 673 + 674 674 + func (m *Mill) destroy(wid models.WorkflowId) { 675 675 + m.mu.Lock() 676 676 + var lease *RemoteLease 677 677 + for _, l := range m.leases { 678 678 + if l.wid == wid { 679 679 + lease = l 680 680 + break 681 681 + } 682 682 + } 683 683 + m.mu.Unlock() 684 684 + if lease == nil { 685 685 + return 686 686 + } 687 687 + reason := "workflow destroyed" 688 688 + switch lease.requestCancel(reason) { 689 689 + case cancelLocal: 690 690 + if sess := m.sessionForNode(lease.nodeID); sess != nil { 691 691 + _ = sess.send(&millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: lease.id}}) 692 692 + } 693 693 + lease.deliverCancelled(reason) 694 694 + case cancelRemote: 695 695 + if sess := m.sessionForNode(lease.nodeID); sess != nil { 696 696 + m.sendCancel(sess, lease, reason) 697 697 + } 698 698 + } 699 699 + } 700 700 + 701 701 + func (m *Mill) releaseSlot(s *millSlot) { 702 702 + lease := s.lease 703 703 + state, cancelled := lease.releaseState() 704 704 + if state == leaseReserved { 705 705 + m.releaseRemote(lease) 706 706 + } else if cancelled && state != leaseDone { 707 707 + return 708 708 + } 709 709 + if err := m.cleanupLease(lease); err != nil { 710 710 + m.l.Error("releaseSlot cleanupLease failed", "lease", lease.id, "err", err) 711 711 + } 712 712 + } 713 713 + func (m *Mill) cleanupLease(lease *RemoteLease) error { 714 714 + lease.finishMu.Lock() 715 715 + defer lease.finishMu.Unlock() 716 716 + return m.cleanupLeaseLocked(lease) 717 717 + } 718 718 + 719 719 + func (m *Mill) cleanupLeaseLocked(lease *RemoteLease) error { 720 720 + if lease.cleanedUp { 721 721 + return nil 722 722 + } 723 723 + if m.db != nil { 724 724 + if err := m.db.DeleteMillLease(lease.id); err != nil { 725 725 + m.scheduleCleanupLocked(lease) 726 726 + return err 727 727 + } 728 728 + } 729 729 + m.mu.Lock() 730 730 + delete(m.leases, lease.id) 731 731 + m.mu.Unlock() 732 732 + lease.cleanedUp = true 733 733 + m.notifyChange() 734 734 + return nil 735 735 + } 736 736 + 737 737 + func (m *Mill) scheduleCleanupLocked(lease *RemoteLease) { 738 738 + if lease.cleanedUp || lease.cleanupRetry { 739 739 + return 740 740 + } 741 741 + lease.cleanupRetry = true 742 742 + time.AfterFunc(5*time.Second, func() { 743 743 + lease.finishMu.Lock() 744 744 + lease.cleanupRetry = false 745 745 + err := m.cleanupLeaseLocked(lease) 746 746 + lease.finishMu.Unlock() 747 747 + if err != nil { 748 748 + m.l.Error("retry lease cleanup failed", "lease", lease.id, "err", err) 749 749 + } 750 750 + }) 751 751 + } 752 752 + 753 753 + func (m *Mill) releaseRemote(lease *RemoteLease) { 754 754 + lease.setState(leaseDone) 755 755 + if sess := m.sessionForNode(lease.nodeID); sess != nil { 756 756 + _ = sess.send(&millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: lease.id}}) 757 757 + } 758 758 + } 759 759 + 760 760 + func (m *Mill) sendCancel(sess *millSession, lease *RemoteLease, reason string) { 761 761 + if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{ 762 762 + LeaseId: lease.id, 763 763 + Reason: reason, 764 764 + }}); err != nil { 765 765 + return 766 766 + } 767 767 + time.AfterFunc(m.cfg.CancelTimeout, func() { m.checkCancelDeadline(lease) }) 768 768 + } 769 769 + 770 770 + func (m *Mill) sessionForNode(nodeID string) *millSession { 771 771 + m.mu.Lock() 772 772 + defer m.mu.Unlock() 773 773 + sess := m.sessions[nodeID] 774 774 + if sess == nil || sess.disconnected { 775 775 + return nil 776 776 + } 777 777 + return sess 778 778 + } 779 779 + 780 780 + func (m *Mill) onSnapshot(sess *millSession, snap *millv1.NodeSnapshot) error { 781 781 + m.mu.Lock() 782 782 + if sess.snapshot != nil && snap.Seqno <= sess.snapshot.Seqno { 783 783 + m.mu.Unlock() 784 784 + return fmt.Errorf("protocol error: snapshot seqno regression. Got %d, last seen %d", snap.Seqno, sess.snapshot.Seqno) 785 785 + } 786 786 + sess.snapshot = snap 787 787 + m.mu.Unlock() 788 788 + 789 789 + if err := m.reconcileLeases(sess, snap.GetActiveLeaseIds()); err != nil { 790 790 + return err 791 791 + } 792 792 + m.mu.Lock() 793 793 + for _, id := range snap.GetActiveLeaseIds() { 794 794 + if lease := m.leases[id]; lease != nil && lease.nodeID == sess.nodeID && lease.epoch == sess.epoch { 795 795 + lease.claimed = true 796 796 + } 797 797 + } 798 798 + m.mu.Unlock() 799 799 + m.notifyChange() 800 800 + return nil 801 801 + } 802 802 + 803 803 + func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { 804 804 + if batch == nil { 805 805 + return nil 806 806 + } 807 807 + 808 808 + if batch.Epoch != sess.epoch { 809 809 + return fmt.Errorf("protocol error: batch epoch %q does not match session %q", batch.Epoch, sess.epoch) 810 810 + } 811 811 + 812 812 + m.mu.Lock() 813 813 + currentKey := sess.nodeID + "/" + sess.epoch 814 814 + current := m.nodeSeqno[currentKey] 815 815 + m.mu.Unlock() 816 816 + 817 817 + expected := current + 1 818 818 + var newEntries []*millv1.Event 819 819 + for _, entry := range batch.Events { 820 820 + // reconnect replays old seqnos, drop those 821 821 + if entry.Seqno <= current { 822 822 + continue 823 823 + } 824 824 + // gap means executor lost rows. so dont apply a partial batch 825 825 + if entry.Seqno != expected { 826 826 + return fmt.Errorf("protocol error: gap in stream seqnos. Expected %d, got %d", expected, entry.Seqno) 827 827 + } 828 828 + newEntries = append(newEntries, entry) 829 829 + expected++ 830 830 + } 831 831 + 832 832 + // all replays, still ack so the executor can trim its outbox 833 833 + if len(newEntries) == 0 { 834 834 + return m.sendAck(sess, current) 835 835 + } 836 836 + 837 837 + type pendingTerminal struct { 838 838 + lease *RemoteLease 839 839 + ar *millv1.AttemptResult 840 840 + } 841 841 + var pendingTerminals []pendingTerminal 842 842 + var artifactLeases []*RemoteLease 843 843 + finishedInBatch := make(map[string]struct{}) 844 844 + 845 845 + var highestSeqno uint64 = current 846 846 + 847 847 + applyFunc := func(tx *db.EventBatchTx) error { 848 848 + for _, entry := range newEntries { 849 849 + m.mu.Lock() 850 850 + lease := m.leases[entry.LeaseId] 851 851 + m.mu.Unlock() 852 852 + 853 853 + // events for leases this node doesn't own are skipped but still 854 854 + // count as processed 855 855 + if lease == nil || lease.nodeID != sess.nodeID { 856 856 + highestSeqno = entry.Seqno 857 857 + continue 858 858 + } 859 859 + 860 860 + // events arriving for a different epoch are invalid (different session) 861 861 + if lease.epoch != "" && lease.epoch != sess.epoch { 862 862 + return fmt.Errorf("protocol error: lease %q epoch %q does not match session %q", lease.id, lease.epoch, sess.epoch) 863 863 + } 864 864 + 865 865 + lease.mu.Lock() 866 866 + state := lease.state 867 867 + lease.mu.Unlock() 868 868 + 869 869 + // done leases can replay terminals on reconnect, skip them 870 870 + if state == leaseDone { 871 871 + highestSeqno = entry.Seqno 872 872 + continue 873 873 + } 874 874 + 875 875 + if _, finished := finishedInBatch[lease.id]; finished { 876 876 + return fmt.Errorf("protocol error: stream entry follows terminal for lease %q", lease.id) 877 877 + } 878 878 + 879 879 + switch { 880 880 + case entry.GetStatusEvent() != nil: 881 881 + ev := entry.GetStatusEvent() 882 882 + statusStr := string(models.StatusKindRunning) 883 883 + var errMsg *string 884 884 + if e := ev.GetError(); e != "" { 885 885 + errMsg = &e 886 886 + } 887 887 + var exitCode *int64 888 888 + if c := ev.GetExitCode(); c != 0 { 889 889 + exitCode = &c 890 890 + } 891 891 + pipelineAtUri := string(lease.wid.PipelineId.AtUri()) 892 892 + if tx != nil { 893 893 + if err := tx.InsertStatusEvent(pipelineAtUri, lease.wid.Name, statusStr, errMsg, exitCode); err != nil { 894 894 + return err 895 895 + } 896 896 + } 897 897 + 898 898 + case entry.GetAttemptResult() != nil: 899 899 + ar := entry.GetAttemptResult() 900 900 + statusStr := "success" 901 901 + switch ar.Status { 902 902 + case millv1.TerminalStatus_SUCCESS: 903 903 + statusStr = "success" 904 904 + case millv1.TerminalStatus_FAILED: 905 905 + statusStr = "failed" 906 906 + case millv1.TerminalStatus_TIMEOUT: 907 907 + statusStr = "timeout" 908 908 + case millv1.TerminalStatus_CANCELLED: 909 909 + statusStr = "cancelled" 910 910 + default: 911 911 + return fmt.Errorf("protocol error: unsupported terminal status %v", ar.Status) 912 912 + } 913 913 + var errMsg *string 914 914 + if e := ar.GetError(); e != "" { 915 915 + errMsg = &e 916 916 + } 917 917 + var exitCode *int64 918 918 + if c := ar.GetExitCode(); c != 0 { 919 919 + exitCode = &c 920 920 + } 921 921 + finishedInBatch[lease.id] = struct{}{} 922 922 + pipelineAtUri := string(lease.wid.PipelineId.AtUri()) 923 923 + if tx != nil { 924 924 + if err := tx.InsertStatusEvent(pipelineAtUri, lease.wid.Name, statusStr, errMsg, exitCode); err != nil { 925 925 + return err 926 926 + } 927 927 + if err := tx.DeleteLease(lease.id); err != nil { 928 928 + return err 929 929 + } 930 930 + if a := ar.GetLogArtifact(); a != nil { 931 931 + if a.GetRef() == "" { 932 932 + return fmt.Errorf("protocol error: empty log artifact ref") 933 933 + } 934 934 + if !strings.HasPrefix(a.GetHash(), "sha256:") { 935 935 + return fmt.Errorf("protocol error: invalid log artifact hash %q", a.GetHash()) 936 936 + } 937 937 + if tx != nil { 938 938 + if err := tx.InsertArtifactRef(lease.id, lease.wid.Name, a.GetRef(), a.GetHash()); err != nil { 939 939 + return err 940 940 + } 941 941 + } 942 942 + artifactLeases = append(artifactLeases, lease) 943 943 + } 944 944 + } 945 945 + pendingTerminals = append(pendingTerminals, pendingTerminal{ 946 946 + lease: lease, 947 947 + ar: ar, 948 948 + }) 949 949 + } 950 950 + 951 951 + highestSeqno = entry.Seqno 952 952 + } 953 953 + 954 954 + if tx != nil { 955 955 + return tx.AdvanceCursor(sess.nodeID, sess.epoch, highestSeqno) 956 956 + } 957 957 + return nil 958 958 + } 959 959 + 960 960 + var err error 961 961 + if m.db != nil { 962 962 + err = m.db.ApplyEventBatch(m.n, applyFunc) 963 963 + } else { 964 964 + err = applyFunc(nil) 965 965 + } 966 966 + 967 967 + if err != nil { 968 968 + return err 969 969 + } 970 970 + 971 971 + if m.cfg.LogDir != "" { 972 972 + for _, lease := range artifactLeases { 973 973 + path := models.LogFilePath(m.cfg.LogDir, lease.wid) 974 974 + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { 975 975 + m.l.Warn("failed to remove live log file after artifact recorded", "path", path, "err", err) 976 976 + } 977 977 + } 978 978 + } 979 979 + 980 980 + m.mu.Lock() 981 981 + if highestSeqno > m.nodeSeqno[currentKey] { 982 982 + m.nodeSeqno[currentKey] = highestSeqno 983 983 + } 984 984 + m.mu.Unlock() 985 985 + 986 986 + for _, pt := range pendingTerminals { 987 987 + // orphans have no waiting RunStep, just mark and clean up 988 988 + if pt.lease.orphaned { 989 989 + pt.lease.markDone() 990 990 + _ = m.cleanupLease(pt.lease) 991 991 + continue 992 992 + } 993 993 + pt.lease.deliverTerminal(pt.ar) 994 994 + if pt.lease.cleanupReady() { 995 995 + _ = m.cleanupLease(pt.lease) 996 996 + } 997 997 + } 998 998 + 999 999 + return m.sendAck(sess, highestSeqno) 1000 1000 + } 1001 1001 + 1002 1002 + func (m *Mill) sendAck(sess *millSession, seqno uint64) error { 1003 1003 + msg := &millproto.Message{Ack: &millv1.Ack{ 1004 1004 + Epoch: sess.epoch, 1005 1005 + UpToSeqno: seqno, 1006 1006 + }} 1007 1007 + if err := sess.send(msg); err != nil { 1008 1008 + return fmt.Errorf("send ack message: %w", err) 1009 1009 + } 1010 1010 + return nil 1011 1011 + } 1012 1012 + func (m *Mill) onLiveLog(sess *millSession, ll *millv1.LiveLog) error { 1013 1013 + if ll == nil || ll.GetLeaseId() == "" { 1014 1014 + return nil 1015 1015 + } 1016 1016 + m.mu.Lock() 1017 1017 + lease := m.leases[ll.GetLeaseId()] 1018 1018 + m.mu.Unlock() 1019 1019 + if lease == nil || lease.nodeID != sess.nodeID { 1020 1020 + return nil 1021 1021 + } 1022 1022 + raw := ll.GetRawJson() 1023 1023 + if m.cfg.LogDir == "" || len(raw) == 0 { 1024 1024 + if m.n != nil { 1025 1025 + m.n.NotifyAll() 1026 1026 + } 1027 1027 + return nil 1028 1028 + } 1029 1029 + lease.mu.Lock() 1030 1030 + isDone := (lease.state == leaseDone) 1031 1031 + lease.mu.Unlock() 1032 1032 + if isDone { 1033 1033 + if m.n != nil { 1034 1034 + m.n.NotifyAll() 1035 1035 + } 1036 1036 + return nil 1037 1037 + } 1038 1038 + 1039 1039 + logPath := models.LogFilePath(m.cfg.LogDir, lease.wid) 1040 1040 + if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { 1041 1041 + m.l.Warn("failed to create log dir", "path", filepath.Dir(logPath), "err", err) 1042 1042 + if m.n != nil { 1043 1043 + m.n.NotifyAll() 1044 1044 + } 1045 1045 + return nil 1046 1046 + } 1047 1047 + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) 1048 1048 + if err != nil { 1049 1049 + m.l.Warn("failed to open log file", "path", logPath, "err", err) 1050 1050 + if m.n != nil { 1051 1051 + m.n.NotifyAll() 1052 1052 + } 1053 1053 + return nil 1054 1054 + } 1055 1055 + if _, err := f.Write(raw); err != nil { 1056 1056 + m.l.Warn("failed to write log file", "path", logPath, "err", err) 1057 1057 + } 1058 1058 + _ = f.Close() 1059 1059 + 1060 1060 + if m.n != nil { 1061 1061 + m.n.NotifyAll() 1062 1062 + } 1063 1063 + return nil 1064 1064 + } 1065 1065 + 1066 1066 + func (m *Mill) onCancelAck(sess *millSession, ca *millv1.CancelAck) { 1067 1067 + m.mu.Lock() 1068 1068 + lease := m.leases[ca.GetLeaseId()] 1069 1069 + m.mu.Unlock() 1070 1070 + if lease == nil || lease.nodeID != sess.nodeID { 1071 1071 + return 1072 1072 + } 1073 1073 + lease.mu.Lock() 1074 1074 + lease.cancelAcked = true 1075 1075 + lease.mu.Unlock() 1076 1076 + } 1077 1077 + 1078 1078 + func (m *Mill) checkCancelDeadline(lease *RemoteLease) { 1079 1079 + lease.mu.Lock() 1080 1080 + isDone := (lease.state == leaseDone) 1081 1081 + isAcked := lease.cancelAcked 1082 1082 + lease.mu.Unlock() 1083 1083 + 1084 1084 + if isDone && isAcked { 1085 1085 + return 1086 1086 + } 1087 1087 + 1088 1088 + m.l.Warn("node failed to comply with cancel request within deadline, quarantining", "node", lease.nodeID, "lease", lease.id, "done", isDone, "acked", isAcked) 1089 1089 + 1090 1090 + reason := fmt.Sprintf("cancel noncompliance for lease %s (done: %t, acked: %t)", lease.id, isDone, isAcked) 1091 1091 + if m.db != nil { 1092 1092 + if err := m.db.QuarantineExecutor(lease.nodeID, reason); err != nil { 1093 1093 + m.l.Error("failed to quarantine executor", "node", lease.nodeID, "err", err) 1094 1094 + } 1095 1095 + } 1096 1096 + 1097 1097 + m.mu.Lock() 1098 1098 + sess := m.sessions[lease.nodeID] 1099 1099 + m.mu.Unlock() 1100 1100 + if sess != nil { 1101 1101 + sess.close() 1102 1102 + } 1103 1103 + } 1104 1104 + 1105 1105 + func marshalJob(wf *models.Workflow) (pipeline string, workflow string, err error) { 1106 1106 + st, ok := wf.Data.(*millWorkflowState) 1107 1107 + if !ok || st == nil { 1108 1108 + return "", "", fmt.Errorf("mill workflow state missing") 1109 1109 + } 1110 1110 + p, err := json.Marshal(st.RawPipeline) 1111 1111 + if err != nil { 1112 1112 + return "", "", fmt.Errorf("marshal pipeline: %w", err) 1113 1113 + } 1114 1114 + w, err := json.Marshal(st.RawWorkflow) 1115 1115 + if err != nil { 1116 1116 + return "", "", fmt.Errorf("marshal workflow: %w", err) 1117 1117 + } 1118 1118 + return string(p), string(w), nil 1119 1119 + }

+873

spindle/mill/mill_test.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "errors" 6 6 + "io" 7 7 + "log/slog" 8 8 + "sync" 9 9 + "testing" 10 10 + "time" 11 11 + 12 12 + "tangled.org/core/api/tangled" 13 13 + "tangled.org/core/spindle/engine" 14 14 + "tangled.org/core/spindle/models" 15 15 + 16 16 + millproto "tangled.org/core/spindle/mill/proto" 17 17 + millv1 "tangled.org/core/spindle/mill/proto/gen" 18 18 + ) 19 19 + 20 20 + type scriptedEncoder func(*millproto.Message) error 21 21 + 22 22 + func (e scriptedEncoder) Encode(msg *millproto.Message) error { return e(msg) } 23 23 + 24 24 + func testWorkflow(name string) *models.Workflow { 25 25 + return &models.Workflow{ 26 26 + Name: name, 27 27 + Environment: map[string]string{}, 28 28 + Steps: []models.Step{remoteStep{}}, 29 29 + Data: &millWorkflowState{ 30 30 + RawWorkflow: tangled.Pipeline_Workflow{Name: name}, 31 31 + RawPipeline: tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}, 32 32 + }, 33 33 + } 34 34 + } 35 35 + 36 36 + func testWorkflowWithRunsOn(name string, runsOn []string) *models.Workflow { 37 37 + wf := testWorkflow(name) 38 38 + wf.Data.(*millWorkflowState).RawWorkflow.RunsOn = runsOn 39 39 + return wf 40 40 + } 41 41 + 42 42 + func addCandidateSession(t *testing.T, m *Mill, nodeID string, labels []string, load float64, enc messageEncoder) *millSession { 43 43 + t.Helper() 44 44 + if enc == nil { 45 45 + enc = scriptedEncoder(func(*millproto.Message) error { return nil }) 46 46 + } 47 47 + sess := newSession(nodeID, "inc-"+nodeID, labels, enc, slog.New(slog.NewTextHandler(io.Discard, nil))) 48 48 + sess.snapshot = &millv1.NodeSnapshot{ 49 49 + Seqno: 1, 50 50 + Engines: map[string]*millv1.EngineAvailability{ 51 51 + "dummy": {Available: load < 1.0, Load: map[string]float64{"slots": load}}, 52 52 + }, 53 53 + } 54 54 + m.mu.Lock() 55 55 + m.sessions[nodeID] = sess 56 56 + m.mu.Unlock() 57 57 + return sess 58 58 + } 59 59 + 60 60 + func assertRankedNodes(t *testing.T, got []*millSession, want []string) { 61 61 + t.Helper() 62 62 + if len(got) != len(want) { 63 63 + t.Fatalf("rankCandidates() returned %d candidates, want %d: got %v want %v", len(got), len(want), sessionIDs(got), want) 64 64 + } 65 65 + for i := range want { 66 66 + if got[i].nodeID != want[i] { 67 67 + t.Fatalf("rankCandidates()[%d] = %q, want %q; full order got %v want %v", i, got[i].nodeID, want[i], sessionIDs(got), want) 68 68 + } 69 69 + } 70 70 + } 71 71 + 72 72 + func sessionIDs(sessions []*millSession) []string { 73 73 + out := make([]string, len(sessions)) 74 74 + for i, sess := range sessions { 75 75 + out[i] = sess.nodeID 76 76 + } 77 77 + return out 78 78 + } 79 79 + 80 80 + func sameStringMultiset(a, b []string) bool { 81 81 + if len(a) != len(b) { 82 82 + return false 83 83 + } 84 84 + counts := make(map[string]int, len(a)) 85 85 + for _, s := range a { 86 86 + counts[s]++ 87 87 + } 88 88 + for _, s := range b { 89 89 + if counts[s] == 0 { 90 90 + return false 91 91 + } 92 92 + counts[s]-- 93 93 + } 94 94 + return true 95 95 + } 96 96 + 97 97 + type reserveReply struct { 98 98 + accepted bool 99 99 + rejectClass millv1.RejectClass 100 100 + reason string 101 101 + } 102 102 + 103 103 + func addReplyingCandidateSession(t *testing.T, m *Mill, nodeID string, labels []string, load float64, asked chan<- string, reply reserveReply) *millSession { 104 104 + t.Helper() 105 105 + var sess *millSession 106 106 + sess = addCandidateSession(t, m, nodeID, labels, load, scriptedEncoder(func(msg *millproto.Message) error { 107 107 + rs := msg.GetReserveSeat() 108 108 + if rs == nil { 109 109 + return nil 110 110 + } 111 111 + if asked != nil { 112 112 + asked <- nodeID 113 113 + } 114 114 + sess.deliver(rs.GetLeaseId(), &millproto.Message{ReserveResult: &millv1.ReserveResult{ 115 115 + LeaseId: rs.GetLeaseId(), 116 116 + Accepted: reply.accepted, 117 117 + RejectReason: reply.reason, 118 118 + RejectClass: reply.rejectClass, 119 119 + }}) 120 120 + return nil 121 121 + })) 122 122 + return sess 123 123 + } 124 124 + 125 125 + func drainAsked(ch <-chan string) []string { 126 126 + var out []string 127 127 + for { 128 128 + select { 129 129 + case nodeID := <-ch: 130 130 + out = append(out, nodeID) 131 131 + default: 132 132 + return out 133 133 + } 134 134 + } 135 135 + } 136 136 + 137 137 + func TestCommitRetriesAfterSessionCloseBeforeCommitted(t *testing.T) { 138 138 + l := slog.New(slog.NewTextHandler(io.Discard, nil)) 139 139 + m := New(l, Config{BidTimeout: 25 * time.Millisecond, ReconnectGrace: time.Second}) 140 140 + wf := testWorkflow("build") 141 141 + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 142 142 + lease := newLease("lease-1", "node-1", "inc-1", "dummy") 143 143 + lease.wid = wid 144 144 + wf.Data.(*millWorkflowState).Lease = lease 145 145 + 146 146 + m.mu.Lock() 147 147 + m.leases[lease.id] = lease 148 148 + m.mu.Unlock() 149 149 + 150 150 + var sess1 *millSession 151 151 + firstCommit := make(chan struct{}) 152 152 + sess1 = newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 153 153 + if msg.GetCommitLease() != nil { 154 154 + close(firstCommit) 155 155 + m.detachSession(sess1) 156 156 + } 157 157 + return nil 158 158 + }), l) 159 159 + m.attachSession(sess1) 160 160 + 161 161 + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) 162 162 + defer cancel() 163 163 + done := make(chan error, 1) 164 164 + go func() { done <- m.commitAndWait(ctx, wf, nil) }() 165 165 + 166 166 + select { 167 167 + case <-firstCommit: 168 168 + case <-ctx.Done(): 169 169 + t.Fatal("first commit was not sent") 170 170 + } 171 171 + 172 172 + var sess2 *millSession 173 173 + sess2 = newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 174 174 + if msg.GetCommitLease() == nil { 175 175 + return nil 176 176 + } 177 177 + leaseID := msg.GetCommitLease().GetLeaseId() 178 178 + sess2.deliver(leaseID, &millproto.Message{Committed: &millv1.Committed{LeaseId: leaseID}}) 179 179 + _ = m.onEventBatch(sess2, &millv1.EventBatch{ 180 180 + Epoch: sess2.epoch, 181 181 + Events: []*millv1.Event{ 182 182 + { 183 183 + Seqno: 1, 184 184 + LeaseId: leaseID, 185 185 + Payload: &millv1.Event_AttemptResult{ 186 186 + AttemptResult: &millv1.AttemptResult{ 187 187 + Status: millv1.TerminalStatus_SUCCESS, 188 188 + }, 189 189 + }, 190 190 + }, 191 191 + }, 192 192 + }) 193 193 + return nil 194 194 + }), l) 195 195 + m.attachSession(sess2) 196 196 + m.sessionReady(sess2) 197 197 + 198 198 + select { 199 199 + case err := <-done: 200 200 + if err != nil { 201 201 + t.Fatalf("commitAndWait() error = %v, want success after reconnect", err) 202 202 + } 203 203 + case <-ctx.Done(): 204 204 + t.Fatal("commitAndWait() did not finish after reconnect") 205 205 + } 206 206 + } 207 207 + 208 208 + func TestDestroyRunningLeaseDoesNotDropCancelledTerminal(t *testing.T) { 209 209 + l := slog.New(slog.NewTextHandler(io.Discard, nil)) 210 210 + m := New(l, Config{}) 211 211 + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 212 212 + lease := newLease("lease-1", "node-1", "inc-1", "dummy") 213 213 + lease.wid = wid 214 214 + lease.setState(leaseRunning) 215 215 + 216 216 + m.mu.Lock() 217 217 + m.leases[lease.id] = lease 218 218 + m.mu.Unlock() 219 219 + 220 220 + m.destroy(wid) 221 221 + if lease.getState() == leaseDone { 222 222 + t.Fatal("destroy sealed the lease before the terminal result") 223 223 + } 224 224 + 225 225 + lease.deliverTerminal(&millv1.AttemptResult{ 226 226 + Status: millv1.TerminalStatus_CANCELLED, 227 227 + }) 228 228 + res := <-lease.terminal 229 229 + if err := terminalError(res.Status); !errors.Is(err, engine.ErrWorkflowCanceled) { 230 230 + t.Fatalf("terminalError() = %v, want ErrCancelled", err) 231 231 + } 232 232 + } 233 233 + 234 234 + func TestPlaceBlocksWhenNoCapacity(t *testing.T) { 235 235 + l := slog.New(slog.NewTextHandler(io.Discard, nil)) 236 236 + m := New(l, Config{}) 237 237 + wf := testWorkflow("build") 238 238 + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 239 239 + 240 240 + // no executors at all: place must block until ctx expires (user sees pending) 241 241 + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) 242 242 + defer cancel() 243 243 + 244 244 + _, err := m.place(ctx, "dummy", wid, wf) 245 245 + if err != context.DeadlineExceeded { 246 246 + t.Fatalf("place() error = %v, want DeadlineExceeded", err) 247 247 + } 248 248 + } 249 249 + 250 250 + func TestRankCandidatesFiltersRequiredLabelsWithANDSemantics(t *testing.T) { 251 251 + m := New(slog.New(slog.NewTextHandler(io.Discard, nil)), Config{}) 252 252 + addCandidateSession(t, m, "linux-high", []string{"linux"}, 0.0, nil) 253 253 + addCandidateSession(t, m, "linux-arm", []string{"linux", "arm64"}, 0.25, nil) 254 254 + addCandidateSession(t, m, "unlabeled", nil, 0.5, nil) 255 255 + addCandidateSession(t, m, "linux-arm-gpu", []string{"linux", "arm64", "gpu"}, 0.75, nil) 256 256 + addCandidateSession(t, m, "linux-arm-full", []string{"linux", "arm64"}, 1.0, nil) 257 257 + 258 258 + tests := []struct { 259 259 + name string 260 260 + requiredLabels []string 261 261 + want []string 262 262 + }{ 263 263 + { 264 264 + name: "no required labels keeps old capacity ranking", 265 265 + want: []string{"linux-high", "linux-arm", "unlabeled", "linux-arm-gpu"}, 266 266 + }, 267 267 + { 268 268 + name: "single required label includes every candidate carrying it", 269 269 + requiredLabels: []string{"linux"}, 270 270 + want: []string{"linux-high", "linux-arm", "linux-arm-gpu"}, 271 271 + }, 272 272 + { 273 273 + name: "all required labels must be present", 274 274 + requiredLabels: []string{"linux", "arm64"}, 275 275 + want: []string{"linux-arm", "linux-arm-gpu"}, 276 276 + }, 277 277 + { 278 278 + name: "one missing required label excludes the candidate", 279 279 + requiredLabels: []string{"linux", "arm64", "gpu"}, 280 280 + want: []string{"linux-arm-gpu"}, 281 281 + }, 282 282 + { 283 283 + name: "unknown required label leaves no candidate", 284 284 + requiredLabels: []string{"linux", "arm64", "metal"}, 285 285 + }, 286 286 + } 287 287 + 288 288 + for _, tt := range tests { 289 289 + t.Run(tt.name, func(t *testing.T) { 290 290 + assertRankedNodes(t, m.rankCandidates("dummy", tt.requiredLabels), tt.want) 291 291 + }) 292 292 + } 293 293 + } 294 294 + 295 295 + func TestPlaceWithMissingRequiredLabelsStaysPendingWithoutReserve(t *testing.T) { 296 296 + l := slog.New(slog.NewTextHandler(io.Discard, nil)) 297 297 + m := New(l, Config{BidTimeout: 10 * time.Millisecond}) 298 298 + reserveSent := make(chan struct{}, 1) 299 299 + addCandidateSession(t, m, "linux-only", []string{"linux"}, 0.75, scriptedEncoder(func(msg *millproto.Message) error { 300 300 + if msg.GetReserveSeat() != nil { 301 301 + select { 302 302 + case reserveSent <- struct{}{}: 303 303 + default: 304 304 + } 305 305 + } 306 306 + return nil 307 307 + })) 308 308 + wf := testWorkflowWithRunsOn("build", []string{"linux", "arm64"}) 309 309 + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 310 310 + 311 311 + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond) 312 312 + defer cancel() 313 313 + _, err := m.place(ctx, "dummy", wid, wf) 314 314 + if err != context.DeadlineExceeded { 315 315 + t.Fatalf("place() error = %v, want DeadlineExceeded while job remains pending", err) 316 316 + } 317 317 + select { 318 318 + case <-reserveSent: 319 319 + t.Fatal("place() sent ReserveSeat to executor missing a required label") 320 320 + default: 321 321 + } 322 322 + } 323 323 + 324 324 + func TestMaxPendingRejects(t *testing.T) { 325 325 + l := slog.New(slog.NewTextHandler(io.Discard, nil)) 326 326 + m := New(l, Config{MaxPending: 1}) 327 327 + 328 328 + m.mu.Lock() 329 329 + m.pending = 1 330 330 + m.mu.Unlock() 331 331 + 332 332 + wf2 := testWorkflow("b") 333 333 + _, err := m.place(context.Background(), "dummy", models.WorkflowId{Name: "b"}, wf2) 334 334 + if err == nil { 335 335 + t.Fatal("place() past maxPending should error") 336 336 + } 337 337 + } 338 338 + 339 339 + func TestCancelledRunningLeaseSurvivesReleaseForReconnectReplay(t *testing.T) { 340 340 + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 341 341 + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 342 342 + lease := newLease("lease-1", "node-1", "inc-1", "dummy") 343 343 + lease.wid = wid 344 344 + lease.setState(leaseRunning) 345 345 + if err := m.persistLease(lease, leaseRowRunning); err != nil { 346 346 + t.Fatalf("persistLease: %v", err) 347 347 + } 348 348 + m.mu.Lock() 349 349 + m.leases[lease.id] = lease 350 350 + m.mu.Unlock() 351 351 + 352 352 + m.destroy(wid) 353 353 + slot := &millSlot{fleet: m, lease: lease} 354 354 + slot.Release() 355 355 + 356 356 + m.mu.Lock() 357 357 + _, retained := m.leases[lease.id] 358 358 + m.mu.Unlock() 359 359 + if !retained { 360 360 + t.Fatal("slot release removed a cancellation-requested running lease before its terminal result") 361 361 + } 362 362 + if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 1 { 363 363 + t.Fatalf("durable leases after slot release = %+v, err = %v; want retained lease", rows, err) 364 364 + } 365 365 + 366 366 + var sentMu sync.Mutex 367 367 + var sent []*millproto.Message 368 368 + sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 369 369 + sentMu.Lock() 370 370 + sent = append(sent, msg) 371 371 + sentMu.Unlock() 372 372 + return nil 373 373 + }), discardLogger()) 374 374 + if _, ok := m.attachSession(sess); !ok { 375 375 + t.Fatal("attachSession rejected reconnect") 376 376 + } 377 377 + m.sessionReady(sess) 378 378 + sentMu.Lock() 379 379 + var replayed bool 380 380 + for _, msg := range sent { 381 381 + if cancel := msg.GetCancelAttempt(); cancel != nil && cancel.GetLeaseId() == lease.id { 382 382 + replayed = true 383 383 + } 384 384 + } 385 385 + sentMu.Unlock() 386 386 + if !replayed { 387 387 + t.Fatal("reconnect did not replay CancelAttempt for retained lease") 388 388 + } 389 389 + 390 390 + if err := m.onEventBatch(sess, &millv1.EventBatch{ 391 391 + Epoch: sess.epoch, 392 392 + Events: []*millv1.Event{ 393 393 + { 394 394 + Seqno: 1, 395 395 + LeaseId: lease.id, 396 396 + Payload: &millv1.Event_AttemptResult{ 397 397 + AttemptResult: &millv1.AttemptResult{ 398 398 + Status: millv1.TerminalStatus_CANCELLED, 399 399 + }, 400 400 + }, 401 401 + }, 402 402 + }, 403 403 + }); err != nil { 404 404 + t.Fatalf("onEventBatch: %v", err) 405 405 + } 406 406 + m.mu.Lock() 407 407 + _, retained = m.leases[lease.id] 408 408 + m.mu.Unlock() 409 409 + if retained { 410 410 + t.Fatal("terminal result did not clean retained cancelled lease") 411 411 + } 412 412 + if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 0 { 413 413 + t.Fatalf("durable leases after terminal = %+v, err = %v; want none", rows, err) 414 414 + } 415 415 + slot.Release() 416 416 + } 417 417 + 418 418 + func TestSessionRequestCancelledBeforeRegistrationOrSend(t *testing.T) { 419 419 + sent := 0 420 420 + sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(*millproto.Message) error { 421 421 + sent++ 422 422 + return nil 423 423 + }), discardLogger()) 424 424 + ctx, cancel := context.WithCancel(context.Background()) 425 425 + cancel() 426 426 + _, err := sess.request(ctx, "lease-1", &millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: "lease-1"}}) 427 427 + if !errors.Is(err, context.Canceled) { 428 428 + t.Fatalf("request error = %v, want context.Canceled", err) 429 429 + } 430 430 + if sent != 0 { 431 431 + t.Fatalf("request sent %d messages for an already-cancelled context, want 0", sent) 432 432 + } 433 433 + sess.mu.Lock() 434 434 + pending := len(sess.pending) 435 435 + sess.mu.Unlock() 436 436 + if pending != 0 { 437 437 + t.Fatalf("request left %d pending waiters, want 0", pending) 438 438 + } 439 439 + } 440 440 + 441 441 + func TestAttachSessionReplacesSilentIncumbentButRejectsActiveDuplicate(t *testing.T) { 442 442 + m := New(discardLogger(), Config{ReconnectGrace: time.Minute}) 443 443 + old := newSession("node-1", "inc-old", nil, nopEncoder(), discardLogger()) 444 444 + transportClosed := make(chan struct{}) 445 445 + old.closeTransport = func() error { 446 446 + close(transportClosed) 447 447 + return nil 448 448 + } 449 449 + if _, ok := m.attachSession(old); !ok { 450 450 + t.Fatal("first attach rejected") 451 451 + } 452 452 + m.mu.Lock() 453 453 + old.lastSeen = time.Now().Add(-2 * m.cfg.ReconnectGrace) 454 454 + m.mu.Unlock() 455 455 + 456 456 + replacement := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) 457 457 + if _, ok := m.attachSession(replacement); !ok { 458 458 + t.Fatal("silent incumbent blocked authenticated replacement") 459 459 + } 460 460 + select { 461 461 + case <-transportClosed: 462 462 + default: 463 463 + t.Fatal("replacing a silent incumbent did not close its transport") 464 464 + } 465 465 + m.mu.Lock() 466 466 + replacement.lastSeen = time.Now().Add(-2 * m.cfg.ReconnectGrace) 467 467 + m.mu.Unlock() 468 468 + if err := replacement.dispatch(m, &millproto.Message{NodeSnapshot: &millv1.NodeSnapshot{Seqno: 1}}); err != nil { 469 469 + t.Fatalf("periodic snapshot dispatch: %v", err) 470 470 + } 471 471 + if _, ok := m.attachSession(newSession("node-1", "inc-dup", nil, nopEncoder(), discardLogger())); ok { 472 472 + t.Fatal("active replacement did not reject a duplicate session") 473 473 + } 474 474 + } 475 475 + 476 476 + func TestPlaceReleasesRemoteReservationWhenInitialPersistenceFails(t *testing.T) { 477 477 + m, bdb := restoreTestMill(t, Config{BidTimeout: time.Second}) 478 478 + if err := bdb.Close(); err != nil { 479 479 + t.Fatalf("close db: %v", err) 480 480 + } 481 481 + released := make(chan string, 1) 482 482 + var sess *millSession 483 483 + sess = addCandidateSession(t, m, "node-1", nil, 0, scriptedEncoder(func(msg *millproto.Message) error { 484 484 + switch { 485 485 + case msg.GetReserveSeat() != nil: 486 486 + leaseID := msg.GetReserveSeat().GetLeaseId() 487 487 + sess.deliver(leaseID, &millproto.Message{ReserveResult: &millv1.ReserveResult{ 488 488 + LeaseId: leaseID, 489 489 + Accepted: true, 490 490 + }}) 491 491 + case msg.GetReleaseLease() != nil: 492 492 + released <- msg.GetReleaseLease().GetLeaseId() 493 493 + } 494 494 + return nil 495 495 + })) 496 496 + 497 497 + ctx, cancel := context.WithTimeout(context.Background(), time.Second) 498 498 + defer cancel() 499 499 + slot, err := m.place( 500 500 + ctx, 501 501 + "dummy", 502 502 + models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, 503 503 + testWorkflow("build"), 504 504 + ) 505 505 + if err == nil { 506 506 + t.Fatal("place succeeded after reserved lease persistence failed") 507 507 + } 508 508 + if slot != nil { 509 509 + t.Fatalf("place returned slot %T after persistence failure", slot) 510 510 + } 511 511 + select { 512 512 + case leaseID := <-released: 513 513 + if leaseID == "" { 514 514 + t.Fatal("ReleaseLease had empty lease id") 515 515 + } 516 516 + case <-time.After(time.Second): 517 517 + t.Fatal("persistence failure did not compensate with ReleaseLease") 518 518 + } 519 519 + m.mu.Lock() 520 520 + leases := len(m.leases) 521 521 + m.mu.Unlock() 522 522 + if leases != 0 { 523 523 + t.Fatalf("mill published %d leases after initial persistence failure, want 0", leases) 524 524 + } 525 525 + } 526 526 + 527 527 + func TestGapsAndDuplicates(t *testing.T) { 528 528 + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 529 529 + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 530 530 + m.attachSession(sess) 531 531 + 532 532 + owned := newLease("lease-1", "node-1", "inc-1", "dummy") 533 533 + owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 534 534 + m.mu.Lock() 535 535 + m.leases[owned.id] = owned 536 536 + m.mu.Unlock() 537 537 + 538 538 + err := m.onEventBatch(sess, &millv1.EventBatch{ 539 539 + Epoch: sess.epoch, 540 540 + Events: []*millv1.Event{ 541 541 + { 542 542 + Seqno: 0, 543 543 + LeaseId: owned.id, 544 544 + Payload: &millv1.Event_StatusEvent{ 545 545 + StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING}, 546 546 + }, 547 547 + }, 548 548 + }, 549 549 + }) 550 550 + if err != nil { 551 551 + t.Fatalf("expected duplicate to be skipped without error, got: %v", err) 552 552 + } 553 553 + 554 554 + err = m.onEventBatch(sess, &millv1.EventBatch{ 555 555 + Epoch: sess.epoch, 556 556 + Events: []*millv1.Event{ 557 557 + { 558 558 + Seqno: 2, 559 559 + LeaseId: owned.id, 560 560 + Payload: &millv1.Event_StatusEvent{ 561 561 + StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING}, 562 562 + }, 563 563 + }, 564 564 + }, 565 565 + }) 566 566 + if err == nil { 567 567 + t.Fatal("expected error due to seqno gap, got nil") 568 568 + } 569 569 + } 570 570 + 571 571 + func TestAtomicBatchRollback(t *testing.T) { 572 572 + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 573 573 + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 574 574 + m.attachSession(sess) 575 575 + 576 576 + owned := newLease("lease-1", "node-1", "inc-1", "dummy") 577 577 + owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 578 578 + m.mu.Lock() 579 579 + m.leases[owned.id] = owned 580 580 + m.mu.Unlock() 581 581 + 582 582 + if _, err := bdb.Exec(` 583 583 + create trigger reject_status_event 584 584 + before insert on events 585 585 + begin 586 586 + select raise(abort, 'forced status event failure'); 587 587 + end 588 588 + `); err != nil { 589 589 + t.Fatalf("failed to create fail trigger: %v", err) 590 590 + } 591 591 + defer bdb.Exec("drop trigger reject_status_event") 592 592 + 593 593 + err := m.onEventBatch(sess, &millv1.EventBatch{ 594 594 + Epoch: sess.epoch, 595 595 + Events: []*millv1.Event{ 596 596 + { 597 597 + Seqno: 1, 598 598 + LeaseId: owned.id, 599 599 + Payload: &millv1.Event_StatusEvent{ 600 600 + StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING}, 601 601 + }, 602 602 + }, 603 603 + }, 604 604 + }) 605 605 + if err == nil { 606 606 + t.Fatal("expected status event insertion to fail due to trigger") 607 607 + } 608 608 + 609 609 + m.mu.Lock() 610 610 + seqno := m.nodeSeqno["node-1/inc-1"] 611 611 + m.mu.Unlock() 612 612 + if seqno != 0 { 613 613 + t.Fatalf("expected seqno 0 due to rollback, got %d", seqno) 614 614 + } 615 615 + } 616 616 + 617 617 + func TestTerminalBeforeACK(t *testing.T) { 618 618 + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 619 619 + 620 620 + ackSent := make(chan struct{}) 621 621 + sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 622 622 + if msg.GetAck() != nil { 623 623 + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 624 624 + st, err := bdb.GetStatus(wid) 625 625 + if err != nil || st.Status != "success" { 626 626 + t.Errorf("expected terminal status success at ACK time, got status: %v, err: %v", st, err) 627 627 + } 628 628 + close(ackSent) 629 629 + } 630 630 + return nil 631 631 + }), discardLogger()) 632 632 + m.attachSession(sess) 633 633 + 634 634 + owned := newLease("lease-1", "node-1", "inc-1", "dummy") 635 635 + owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 636 636 + m.mu.Lock() 637 637 + m.leases[owned.id] = owned 638 638 + m.mu.Unlock() 639 639 + 640 640 + err := m.onEventBatch(sess, &millv1.EventBatch{ 641 641 + Epoch: sess.epoch, 642 642 + Events: []*millv1.Event{ 643 643 + { 644 644 + Seqno: 1, 645 645 + LeaseId: owned.id, 646 646 + Payload: &millv1.Event_AttemptResult{ 647 647 + AttemptResult: &millv1.AttemptResult{Status: millv1.TerminalStatus_SUCCESS}, 648 648 + }, 649 649 + }, 650 650 + }, 651 651 + }) 652 652 + if err != nil { 653 653 + t.Fatalf("onEventBatch: %v", err) 654 654 + } 655 655 + 656 656 + select { 657 657 + case <-ackSent: 658 658 + case <-time.After(2 * time.Second): 659 659 + t.Fatal("ACK was not sent") 660 660 + } 661 661 + } 662 662 + 663 663 + func TestExecutorRestartEmptySnapshot(t *testing.T) { 664 664 + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 665 665 + 666 666 + lease := newLease("lease-1", "node-1", "inc-old", "dummy") 667 667 + lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 668 668 + m.mu.Lock() 669 669 + m.leases[lease.id] = lease 670 670 + m.mu.Unlock() 671 671 + if err := m.persistLease(lease, leaseRowRunning); err != nil { 672 672 + t.Fatalf("persistLease: %v", err) 673 673 + } 674 674 + 675 675 + sess := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) 676 676 + m.attachSession(sess) 677 677 + 678 678 + err := m.onSnapshot(sess, &millv1.NodeSnapshot{ 679 679 + Seqno: 1, 680 680 + ActiveLeaseIds: nil, 681 681 + }) 682 682 + if err != nil { 683 683 + t.Fatalf("onSnapshot: %v", err) 684 684 + } 685 685 + 686 686 + m.mu.Lock() 687 687 + _, stillActive := m.leases["lease-1"] 688 688 + m.mu.Unlock() 689 689 + if stillActive { 690 690 + t.Fatal("expected old epoch lease to be reconciled and failed") 691 691 + } 692 692 + } 693 693 + 694 694 + func TestReplacementLostBeforeSnapshotFailsOldEpochLease(t *testing.T) { 695 695 + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 696 696 + lease := newLease("lease-1", "node-1", "inc-old", "dummy") 697 697 + lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 698 698 + lease.setState(leaseRunning) 699 699 + if err := m.persistLease(lease, leaseRowRunning); err != nil { 700 700 + t.Fatalf("persistLease: %v", err) 701 701 + } 702 702 + m.mu.Lock() 703 703 + m.leases[lease.id] = lease 704 704 + m.mu.Unlock() 705 705 + 706 706 + replacement := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) 707 707 + if _, ok := m.attachSession(replacement); !ok { 708 708 + t.Fatal("attachSession rejected replacement") 709 709 + } 710 710 + replacement.disconnected = true 711 711 + m.failLeasesAfterGrace(replacement) 712 712 + 713 713 + m.mu.Lock() 714 714 + _, stillActive := m.leases[lease.id] 715 715 + m.mu.Unlock() 716 716 + if stillActive { 717 717 + t.Fatal("replacement loss stranded old-epoch lease") 718 718 + } 719 719 + } 720 720 + 721 721 + func TestSeqRegression(t *testing.T) { 722 722 + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 723 723 + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 724 724 + m.attachSession(sess) 725 725 + 726 726 + err := m.onSnapshot(sess, &millv1.NodeSnapshot{ 727 727 + Seqno: 5, 728 728 + }) 729 729 + if err != nil { 730 730 + t.Fatalf("first snapshot: %v", err) 731 731 + } 732 732 + 733 733 + err = m.onSnapshot(sess, &millv1.NodeSnapshot{ 734 734 + Seqno: 4, 735 735 + }) 736 736 + if err == nil { 737 737 + t.Fatal("expected seqno regression to be rejected") 738 738 + } 739 739 + } 740 740 + 741 741 + func TestClaimedSweep(t *testing.T) { 742 742 + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 743 743 + 744 744 + lease := newLease("lease-1", "node-1", "inc-1", "dummy") 745 745 + lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 746 746 + lease.orphaned = true 747 747 + lease.claimed = false 748 748 + m.mu.Lock() 749 749 + m.leases[lease.id] = lease 750 750 + m.mu.Unlock() 751 751 + 752 752 + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 753 753 + m.attachSession(sess) 754 754 + err := m.onSnapshot(sess, &millv1.NodeSnapshot{ 755 755 + Seqno: 1, 756 756 + ActiveLeaseIds: []string{"lease-1"}, 757 757 + }) 758 758 + if err != nil { 759 759 + t.Fatalf("onSnapshot: %v", err) 760 760 + } 761 761 + 762 762 + m.detachSession(sess) 763 763 + 764 764 + m.sweepUnclaimedOrphans() 765 765 + 766 766 + m.mu.Lock() 767 767 + _, stillRunning := m.leases["lease-1"] 768 768 + m.mu.Unlock() 769 769 + if !stillRunning { 770 770 + t.Fatal("claimed lease was incorrectly swept by startup sweep") 771 771 + } 772 772 + } 773 773 + 774 774 + func TestCancelDeadline(t *testing.T) { 775 775 + m, _ := restoreTestMill(t, Config{CancelTimeout: 50 * time.Millisecond}) 776 776 + 777 777 + sessClosed := make(chan struct{}) 778 778 + sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 779 779 + return nil 780 780 + }), discardLogger()) 781 781 + sess.closeTransport = func() error { 782 782 + close(sessClosed) 783 783 + return nil 784 784 + } 785 785 + m.attachSession(sess) 786 786 + 787 787 + lease := newLease("lease-1", "node-1", "inc-1", "dummy") 788 788 + lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 789 789 + lease.setState(leaseRunning) 790 790 + m.mu.Lock() 791 791 + m.leases[lease.id] = lease 792 792 + m.mu.Unlock() 793 793 + 794 794 + m.destroy(lease.wid) 795 795 + 796 796 + select { 797 797 + case <-sessClosed: 798 798 + case <-time.After(1 * time.Second): 799 799 + t.Fatal("session was not closed after cancel deadline expiration") 800 800 + } 801 801 + } 802 802 + 803 803 + func TestCleanupRetry(t *testing.T) { 804 804 + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 805 805 + 806 806 + lease := newLease("lease-1", "node-1", "inc-1", "dummy") 807 807 + lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 808 808 + if err := m.persistLease(lease, leaseRowRunning); err != nil { 809 809 + t.Fatalf("persist lease: %v", err) 810 810 + } 811 811 + m.mu.Lock() 812 812 + m.leases[lease.id] = lease 813 813 + m.mu.Unlock() 814 814 + 815 815 + if _, err := bdb.Exec(` 816 816 + create trigger reject_cleanup_delete 817 817 + before delete on mill_leases 818 818 + begin 819 819 + select raise(abort, 'forced delete failure'); 820 820 + end 821 821 + `); err != nil { 822 822 + t.Fatalf("failed to create fail trigger: %v", err) 823 823 + } 824 824 + 825 825 + err := m.cleanupLease(lease) 826 826 + if err == nil { 827 827 + t.Fatal("expected cleanupLease to fail") 828 828 + } 829 829 + 830 830 + m.mu.Lock() 831 831 + _, stillRunning := m.leases["lease-1"] 832 832 + m.mu.Unlock() 833 833 + if !stillRunning { 834 834 + t.Fatal("lease was removed from memory despite cleanup failure") 835 835 + } 836 836 + 837 837 + if _, err := bdb.Exec("drop trigger reject_cleanup_delete"); err != nil { 838 838 + t.Fatalf("drop trigger: %v", err) 839 839 + } 840 840 + 841 841 + err = m.cleanupLease(lease) 842 842 + if err != nil { 843 843 + t.Fatalf("expected retry cleanup to succeed, got: %v", err) 844 844 + } 845 845 + 846 846 + m.mu.Lock() 847 847 + _, stillRunning = m.leases["lease-1"] 848 848 + m.mu.Unlock() 849 849 + if stillRunning { 850 850 + t.Fatal("lease still in memory after successful cleanup retry") 851 851 + } 852 852 + } 853 853 + 854 854 + func TestBoundedBidding(t *testing.T) { 855 855 + m := New(discardLogger(), Config{TopK: 2, BidTimeout: 10 * time.Millisecond}) 856 856 + 857 857 + addCandidateSession(t, m, "node-1", nil, 0, nil) 858 858 + addCandidateSession(t, m, "node-2", nil, 0, nil) 859 859 + addCandidateSession(t, m, "node-3", nil, 0, nil) 860 860 + addCandidateSession(t, m, "node-4", nil, 0, nil) 861 861 + addCandidateSession(t, m, "node-5", nil, 0, nil) 862 862 + 863 863 + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) 864 864 + defer cancel() 865 865 + 866 866 + lease, err := m.bid(ctx, "dummy", models.WorkflowId{}, testWorkflow("build")) 867 867 + if err != nil { 868 868 + t.Fatalf("bid: %v", err) 869 869 + } 870 870 + if lease != nil { 871 871 + t.Fatalf("did not expect a lease, got %+v", lease) 872 872 + } 873 873 + }

+1679

spindle/mill/proto/gen/mill.pb.go

Reviewed

··· 1 1 + // Code generated by protoc-gen-go. DO NOT EDIT. 2 2 + // versions: 3 3 + // protoc-gen-go v1.36.11 4 4 + // protoc (unknown) 5 5 + // source: spindle/mill/v1/mill.proto 6 6 + 7 7 + package millv1 8 8 + 9 9 + import ( 10 10 + _ "buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go/buf/validate" 11 11 + protoreflect "google.golang.org/protobuf/reflect/protoreflect" 12 12 + protoimpl "google.golang.org/protobuf/runtime/protoimpl" 13 13 + reflect "reflect" 14 14 + sync "sync" 15 15 + unsafe "unsafe" 16 16 + ) 17 17 + 18 18 + const ( 19 19 + // Verify that this generated code is sufficiently up-to-date. 20 20 + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 21 21 + // Verify that runtime/protoimpl is sufficiently up-to-date. 22 22 + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 23 23 + ) 24 24 + 25 25 + type RejectClass int32 26 26 + 27 27 + const ( 28 28 + RejectClass_REJECT_CLASS_UNSPECIFIED RejectClass = 0 29 29 + RejectClass_REJECT_CLASS_TRANSIENT RejectClass = 1 30 30 + RejectClass_REJECT_CLASS_INCOMPATIBLE RejectClass = 2 31 31 + ) 32 32 + 33 33 + // Enum value maps for RejectClass. 34 34 + var ( 35 35 + RejectClass_name = map[int32]string{ 36 36 + 0: "REJECT_CLASS_UNSPECIFIED", 37 37 + 1: "REJECT_CLASS_TRANSIENT", 38 38 + 2: "REJECT_CLASS_INCOMPATIBLE", 39 39 + } 40 40 + RejectClass_value = map[string]int32{ 41 41 + "REJECT_CLASS_UNSPECIFIED": 0, 42 42 + "REJECT_CLASS_TRANSIENT": 1, 43 43 + "REJECT_CLASS_INCOMPATIBLE": 2, 44 44 + } 45 45 + ) 46 46 + 47 47 + func (x RejectClass) Enum() *RejectClass { 48 48 + p := new(RejectClass) 49 49 + *p = x 50 50 + return p 51 51 + } 52 52 + 53 53 + func (x RejectClass) String() string { 54 54 + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 55 55 + } 56 56 + 57 57 + func (RejectClass) Descriptor() protoreflect.EnumDescriptor { 58 58 + return file_spindle_mill_v1_mill_proto_enumTypes[0].Descriptor() 59 59 + } 60 60 + 61 61 + func (RejectClass) Type() protoreflect.EnumType { 62 62 + return &file_spindle_mill_v1_mill_proto_enumTypes[0] 63 63 + } 64 64 + 65 65 + func (x RejectClass) Number() protoreflect.EnumNumber { 66 66 + return protoreflect.EnumNumber(x) 67 67 + } 68 68 + 69 69 + // Deprecated: Use RejectClass.Descriptor instead. 70 70 + func (RejectClass) EnumDescriptor() ([]byte, []int) { 71 71 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{0} 72 72 + } 73 73 + 74 74 + type NonterminalStatus int32 75 75 + 76 76 + const ( 77 77 + NonterminalStatus_NONTERMINAL_STATUS_UNSPECIFIED NonterminalStatus = 0 78 78 + NonterminalStatus_RUNNING NonterminalStatus = 1 79 79 + ) 80 80 + 81 81 + // Enum value maps for NonterminalStatus. 82 82 + var ( 83 83 + NonterminalStatus_name = map[int32]string{ 84 84 + 0: "NONTERMINAL_STATUS_UNSPECIFIED", 85 85 + 1: "RUNNING", 86 86 + } 87 87 + NonterminalStatus_value = map[string]int32{ 88 88 + "NONTERMINAL_STATUS_UNSPECIFIED": 0, 89 89 + "RUNNING": 1, 90 90 + } 91 91 + ) 92 92 + 93 93 + func (x NonterminalStatus) Enum() *NonterminalStatus { 94 94 + p := new(NonterminalStatus) 95 95 + *p = x 96 96 + return p 97 97 + } 98 98 + 99 99 + func (x NonterminalStatus) String() string { 100 100 + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 101 101 + } 102 102 + 103 103 + func (NonterminalStatus) Descriptor() protoreflect.EnumDescriptor { 104 104 + return file_spindle_mill_v1_mill_proto_enumTypes[1].Descriptor() 105 105 + } 106 106 + 107 107 + func (NonterminalStatus) Type() protoreflect.EnumType { 108 108 + return &file_spindle_mill_v1_mill_proto_enumTypes[1] 109 109 + } 110 110 + 111 111 + func (x NonterminalStatus) Number() protoreflect.EnumNumber { 112 112 + return protoreflect.EnumNumber(x) 113 113 + } 114 114 + 115 115 + // Deprecated: Use NonterminalStatus.Descriptor instead. 116 116 + func (NonterminalStatus) EnumDescriptor() ([]byte, []int) { 117 117 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{1} 118 118 + } 119 119 + 120 120 + type TerminalStatus int32 121 121 + 122 122 + const ( 123 123 + TerminalStatus_TERMINAL_STATUS_UNSPECIFIED TerminalStatus = 0 124 124 + TerminalStatus_SUCCESS TerminalStatus = 1 125 125 + TerminalStatus_FAILED TerminalStatus = 2 126 126 + TerminalStatus_TIMEOUT TerminalStatus = 3 127 127 + TerminalStatus_CANCELLED TerminalStatus = 4 128 128 + ) 129 129 + 130 130 + // Enum value maps for TerminalStatus. 131 131 + var ( 132 132 + TerminalStatus_name = map[int32]string{ 133 133 + 0: "TERMINAL_STATUS_UNSPECIFIED", 134 134 + 1: "SUCCESS", 135 135 + 2: "FAILED", 136 136 + 3: "TIMEOUT", 137 137 + 4: "CANCELLED", 138 138 + } 139 139 + TerminalStatus_value = map[string]int32{ 140 140 + "TERMINAL_STATUS_UNSPECIFIED": 0, 141 141 + "SUCCESS": 1, 142 142 + "FAILED": 2, 143 143 + "TIMEOUT": 3, 144 144 + "CANCELLED": 4, 145 145 + } 146 146 + ) 147 147 + 148 148 + func (x TerminalStatus) Enum() *TerminalStatus { 149 149 + p := new(TerminalStatus) 150 150 + *p = x 151 151 + return p 152 152 + } 153 153 + 154 154 + func (x TerminalStatus) String() string { 155 155 + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 156 156 + } 157 157 + 158 158 + func (TerminalStatus) Descriptor() protoreflect.EnumDescriptor { 159 159 + return file_spindle_mill_v1_mill_proto_enumTypes[2].Descriptor() 160 160 + } 161 161 + 162 162 + func (TerminalStatus) Type() protoreflect.EnumType { 163 163 + return &file_spindle_mill_v1_mill_proto_enumTypes[2] 164 164 + } 165 165 + 166 166 + func (x TerminalStatus) Number() protoreflect.EnumNumber { 167 167 + return protoreflect.EnumNumber(x) 168 168 + } 169 169 + 170 170 + // Deprecated: Use TerminalStatus.Descriptor instead. 171 171 + func (TerminalStatus) EnumDescriptor() ([]byte, []int) { 172 172 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{2} 173 173 + } 174 174 + 175 175 + // executor identity, sent on connect 176 176 + type Hello struct { 177 177 + state protoimpl.MessageState `protogen:"open.v1"` 178 178 + ProtocolVersion uint32 `protobuf:"varint,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"` 179 179 + // GOARCH of the node, informational only 180 180 + Arch string `protobuf:"bytes,2,opt,name=arch,proto3" json:"arch,omitempty"` 181 181 + // operator-defined labels, matched against runs_on 182 182 + Labels []string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty"` 183 183 + Epoch string `protobuf:"bytes,4,opt,name=epoch,proto3" json:"epoch,omitempty"` 184 184 + unknownFields protoimpl.UnknownFields 185 185 + sizeCache protoimpl.SizeCache 186 186 + } 187 187 + 188 188 + func (x *Hello) Reset() { 189 189 + *x = Hello{} 190 190 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[0] 191 191 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 192 192 + ms.StoreMessageInfo(mi) 193 193 + } 194 194 + 195 195 + func (x *Hello) String() string { 196 196 + return protoimpl.X.MessageStringOf(x) 197 197 + } 198 198 + 199 199 + func (*Hello) ProtoMessage() {} 200 200 + 201 201 + func (x *Hello) ProtoReflect() protoreflect.Message { 202 202 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[0] 203 203 + if x != nil { 204 204 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 205 205 + if ms.LoadMessageInfo() == nil { 206 206 + ms.StoreMessageInfo(mi) 207 207 + } 208 208 + return ms 209 209 + } 210 210 + return mi.MessageOf(x) 211 211 + } 212 212 + 213 213 + // Deprecated: Use Hello.ProtoReflect.Descriptor instead. 214 214 + func (*Hello) Descriptor() ([]byte, []int) { 215 215 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{0} 216 216 + } 217 217 + 218 218 + func (x *Hello) GetProtocolVersion() uint32 { 219 219 + if x != nil { 220 220 + return x.ProtocolVersion 221 221 + } 222 222 + return 0 223 223 + } 224 224 + 225 225 + func (x *Hello) GetArch() string { 226 226 + if x != nil { 227 227 + return x.Arch 228 228 + } 229 229 + return "" 230 230 + } 231 231 + 232 232 + func (x *Hello) GetLabels() []string { 233 233 + if x != nil { 234 234 + return x.Labels 235 235 + } 236 236 + return nil 237 237 + } 238 238 + 239 239 + func (x *Hello) GetEpoch() string { 240 240 + if x != nil { 241 241 + return x.Epoch 242 242 + } 243 243 + return "" 244 244 + } 245 245 + 246 246 + // reconnect state for an existing epoch 247 247 + type Resume struct { 248 248 + state protoimpl.MessageState `protogen:"open.v1"` 249 249 + Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"` 250 250 + AckSeqno uint64 `protobuf:"varint,2,opt,name=ack_seqno,json=ackSeqno,proto3" json:"ack_seqno,omitempty"` 251 251 + unknownFields protoimpl.UnknownFields 252 252 + sizeCache protoimpl.SizeCache 253 253 + } 254 254 + 255 255 + func (x *Resume) Reset() { 256 256 + *x = Resume{} 257 257 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[1] 258 258 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 259 259 + ms.StoreMessageInfo(mi) 260 260 + } 261 261 + 262 262 + func (x *Resume) String() string { 263 263 + return protoimpl.X.MessageStringOf(x) 264 264 + } 265 265 + 266 266 + func (*Resume) ProtoMessage() {} 267 267 + 268 268 + func (x *Resume) ProtoReflect() protoreflect.Message { 269 269 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[1] 270 270 + if x != nil { 271 271 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 272 272 + if ms.LoadMessageInfo() == nil { 273 273 + ms.StoreMessageInfo(mi) 274 274 + } 275 275 + return ms 276 276 + } 277 277 + return mi.MessageOf(x) 278 278 + } 279 279 + 280 280 + // Deprecated: Use Resume.ProtoReflect.Descriptor instead. 281 281 + func (*Resume) Descriptor() ([]byte, []int) { 282 282 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{1} 283 283 + } 284 284 + 285 285 + func (x *Resume) GetEpoch() string { 286 286 + if x != nil { 287 287 + return x.Epoch 288 288 + } 289 289 + return "" 290 290 + } 291 291 + 292 292 + func (x *Resume) GetAckSeqno() uint64 { 293 293 + if x != nil { 294 294 + return x.AckSeqno 295 295 + } 296 296 + return 0 297 297 + } 298 298 + 299 299 + // per-engine state of a node 300 300 + type EngineAvailability struct { 301 301 + state protoimpl.MessageState `protogen:"open.v1"` 302 302 + Available bool `protobuf:"varint,1,opt,name=available,proto3" json:"available,omitempty"` 303 303 + // opaque engine-defined load metrics, higher means more loaded 304 304 + Load map[string]float64 `protobuf:"bytes,2,rep,name=load,proto3" json:"load,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"fixed64,2,opt,name=value"` 305 305 + unknownFields protoimpl.UnknownFields 306 306 + sizeCache protoimpl.SizeCache 307 307 + } 308 308 + 309 309 + func (x *EngineAvailability) Reset() { 310 310 + *x = EngineAvailability{} 311 311 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[2] 312 312 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 313 313 + ms.StoreMessageInfo(mi) 314 314 + } 315 315 + 316 316 + func (x *EngineAvailability) String() string { 317 317 + return protoimpl.X.MessageStringOf(x) 318 318 + } 319 319 + 320 320 + func (*EngineAvailability) ProtoMessage() {} 321 321 + 322 322 + func (x *EngineAvailability) ProtoReflect() protoreflect.Message { 323 323 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[2] 324 324 + if x != nil { 325 325 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 326 326 + if ms.LoadMessageInfo() == nil { 327 327 + ms.StoreMessageInfo(mi) 328 328 + } 329 329 + return ms 330 330 + } 331 331 + return mi.MessageOf(x) 332 332 + } 333 333 + 334 334 + // Deprecated: Use EngineAvailability.ProtoReflect.Descriptor instead. 335 335 + func (*EngineAvailability) Descriptor() ([]byte, []int) { 336 336 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{2} 337 337 + } 338 338 + 339 339 + func (x *EngineAvailability) GetAvailable() bool { 340 340 + if x != nil { 341 341 + return x.Available 342 342 + } 343 343 + return false 344 344 + } 345 345 + 346 346 + func (x *EngineAvailability) GetLoad() map[string]float64 { 347 347 + if x != nil { 348 348 + return x.Load 349 349 + } 350 350 + return nil 351 351 + } 352 352 + 353 353 + // full node state 354 354 + type NodeSnapshot struct { 355 355 + state protoimpl.MessageState `protogen:"open.v1"` 356 356 + Seqno uint64 `protobuf:"varint,1,opt,name=seqno,proto3" json:"seqno,omitempty"` 357 357 + Engines map[string]*EngineAvailability `protobuf:"bytes,2,rep,name=engines,proto3" json:"engines,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` 358 358 + // every lease the executor currently holds, reserved or running 359 359 + ActiveLeaseIds []string `protobuf:"bytes,3,rep,name=active_lease_ids,json=activeLeaseIds,proto3" json:"active_lease_ids,omitempty"` 360 360 + unknownFields protoimpl.UnknownFields 361 361 + sizeCache protoimpl.SizeCache 362 362 + } 363 363 + 364 364 + func (x *NodeSnapshot) Reset() { 365 365 + *x = NodeSnapshot{} 366 366 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[3] 367 367 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 368 368 + ms.StoreMessageInfo(mi) 369 369 + } 370 370 + 371 371 + func (x *NodeSnapshot) String() string { 372 372 + return protoimpl.X.MessageStringOf(x) 373 373 + } 374 374 + 375 375 + func (*NodeSnapshot) ProtoMessage() {} 376 376 + 377 377 + func (x *NodeSnapshot) ProtoReflect() protoreflect.Message { 378 378 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[3] 379 379 + if x != nil { 380 380 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 381 381 + if ms.LoadMessageInfo() == nil { 382 382 + ms.StoreMessageInfo(mi) 383 383 + } 384 384 + return ms 385 385 + } 386 386 + return mi.MessageOf(x) 387 387 + } 388 388 + 389 389 + // Deprecated: Use NodeSnapshot.ProtoReflect.Descriptor instead. 390 390 + func (*NodeSnapshot) Descriptor() ([]byte, []int) { 391 391 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{3} 392 392 + } 393 393 + 394 394 + func (x *NodeSnapshot) GetSeqno() uint64 { 395 395 + if x != nil { 396 396 + return x.Seqno 397 397 + } 398 398 + return 0 399 399 + } 400 400 + 401 401 + func (x *NodeSnapshot) GetEngines() map[string]*EngineAvailability { 402 402 + if x != nil { 403 403 + return x.Engines 404 404 + } 405 405 + return nil 406 406 + } 407 407 + 408 408 + func (x *NodeSnapshot) GetActiveLeaseIds() []string { 409 409 + if x != nil { 410 410 + return x.ActiveLeaseIds 411 411 + } 412 412 + return nil 413 413 + } 414 414 + 415 415 + // a seat reservation, carries no secrets 416 416 + type ReserveSeat struct { 417 417 + state protoimpl.MessageState `protogen:"open.v1"` 418 418 + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 419 419 + TargetEngine string `protobuf:"bytes,2,opt,name=target_engine,json=targetEngine,proto3" json:"target_engine,omitempty"` 420 420 + RawPipelineJson string `protobuf:"bytes,3,opt,name=raw_pipeline_json,json=rawPipelineJson,proto3" json:"raw_pipeline_json,omitempty"` 421 421 + RawWorkflowJson string `protobuf:"bytes,4,opt,name=raw_workflow_json,json=rawWorkflowJson,proto3" json:"raw_workflow_json,omitempty"` 422 422 + // pipeline id, the executor reconstructs the exact WorkflowId from it 423 423 + Knot string `protobuf:"bytes,5,opt,name=knot,proto3" json:"knot,omitempty"` 424 424 + Rkey string `protobuf:"bytes,6,opt,name=rkey,proto3" json:"rkey,omitempty"` 425 425 + TtlSeconds uint32 `protobuf:"varint,7,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"` 426 426 + unknownFields protoimpl.UnknownFields 427 427 + sizeCache protoimpl.SizeCache 428 428 + } 429 429 + 430 430 + func (x *ReserveSeat) Reset() { 431 431 + *x = ReserveSeat{} 432 432 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[4] 433 433 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 434 434 + ms.StoreMessageInfo(mi) 435 435 + } 436 436 + 437 437 + func (x *ReserveSeat) String() string { 438 438 + return protoimpl.X.MessageStringOf(x) 439 439 + } 440 440 + 441 441 + func (*ReserveSeat) ProtoMessage() {} 442 442 + 443 443 + func (x *ReserveSeat) ProtoReflect() protoreflect.Message { 444 444 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[4] 445 445 + if x != nil { 446 446 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 447 447 + if ms.LoadMessageInfo() == nil { 448 448 + ms.StoreMessageInfo(mi) 449 449 + } 450 450 + return ms 451 451 + } 452 452 + return mi.MessageOf(x) 453 453 + } 454 454 + 455 455 + // Deprecated: Use ReserveSeat.ProtoReflect.Descriptor instead. 456 456 + func (*ReserveSeat) Descriptor() ([]byte, []int) { 457 457 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{4} 458 458 + } 459 459 + 460 460 + func (x *ReserveSeat) GetLeaseId() string { 461 461 + if x != nil { 462 462 + return x.LeaseId 463 463 + } 464 464 + return "" 465 465 + } 466 466 + 467 467 + func (x *ReserveSeat) GetTargetEngine() string { 468 468 + if x != nil { 469 469 + return x.TargetEngine 470 470 + } 471 471 + return "" 472 472 + } 473 473 + 474 474 + func (x *ReserveSeat) GetRawPipelineJson() string { 475 475 + if x != nil { 476 476 + return x.RawPipelineJson 477 477 + } 478 478 + return "" 479 479 + } 480 480 + 481 481 + func (x *ReserveSeat) GetRawWorkflowJson() string { 482 482 + if x != nil { 483 483 + return x.RawWorkflowJson 484 484 + } 485 485 + return "" 486 486 + } 487 487 + 488 488 + func (x *ReserveSeat) GetKnot() string { 489 489 + if x != nil { 490 490 + return x.Knot 491 491 + } 492 492 + return "" 493 493 + } 494 494 + 495 495 + func (x *ReserveSeat) GetRkey() string { 496 496 + if x != nil { 497 497 + return x.Rkey 498 498 + } 499 499 + return "" 500 500 + } 501 501 + 502 502 + func (x *ReserveSeat) GetTtlSeconds() uint32 { 503 503 + if x != nil { 504 504 + return x.TtlSeconds 505 505 + } 506 506 + return 0 507 507 + } 508 508 + 509 509 + type ReserveResult struct { 510 510 + state protoimpl.MessageState `protogen:"open.v1"` 511 511 + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 512 512 + Accepted bool `protobuf:"varint,2,opt,name=accepted,proto3" json:"accepted,omitempty"` 513 513 + RejectReason string `protobuf:"bytes,3,opt,name=reject_reason,json=rejectReason,proto3" json:"reject_reason,omitempty"` 514 514 + RejectClass RejectClass `protobuf:"varint,4,opt,name=reject_class,json=rejectClass,proto3,enum=spindle.mill.v1.RejectClass" json:"reject_class,omitempty"` 515 515 + unknownFields protoimpl.UnknownFields 516 516 + sizeCache protoimpl.SizeCache 517 517 + } 518 518 + 519 519 + func (x *ReserveResult) Reset() { 520 520 + *x = ReserveResult{} 521 521 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[5] 522 522 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 523 523 + ms.StoreMessageInfo(mi) 524 524 + } 525 525 + 526 526 + func (x *ReserveResult) String() string { 527 527 + return protoimpl.X.MessageStringOf(x) 528 528 + } 529 529 + 530 530 + func (*ReserveResult) ProtoMessage() {} 531 531 + 532 532 + func (x *ReserveResult) ProtoReflect() protoreflect.Message { 533 533 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[5] 534 534 + if x != nil { 535 535 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 536 536 + if ms.LoadMessageInfo() == nil { 537 537 + ms.StoreMessageInfo(mi) 538 538 + } 539 539 + return ms 540 540 + } 541 541 + return mi.MessageOf(x) 542 542 + } 543 543 + 544 544 + // Deprecated: Use ReserveResult.ProtoReflect.Descriptor instead. 545 545 + func (*ReserveResult) Descriptor() ([]byte, []int) { 546 546 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{5} 547 547 + } 548 548 + 549 549 + func (x *ReserveResult) GetLeaseId() string { 550 550 + if x != nil { 551 551 + return x.LeaseId 552 552 + } 553 553 + return "" 554 554 + } 555 555 + 556 556 + func (x *ReserveResult) GetAccepted() bool { 557 557 + if x != nil { 558 558 + return x.Accepted 559 559 + } 560 560 + return false 561 561 + } 562 562 + 563 563 + func (x *ReserveResult) GetRejectReason() string { 564 564 + if x != nil { 565 565 + return x.RejectReason 566 566 + } 567 567 + return "" 568 568 + } 569 569 + 570 570 + func (x *ReserveResult) GetRejectClass() RejectClass { 571 571 + if x != nil { 572 572 + return x.RejectClass 573 573 + } 574 574 + return RejectClass_REJECT_CLASS_UNSPECIFIED 575 575 + } 576 576 + 577 577 + // a single unlocked secret 578 578 + type Secret struct { 579 579 + state protoimpl.MessageState `protogen:"open.v1"` 580 580 + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` 581 581 + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` 582 582 + unknownFields protoimpl.UnknownFields 583 583 + sizeCache protoimpl.SizeCache 584 584 + } 585 585 + 586 586 + func (x *Secret) Reset() { 587 587 + *x = Secret{} 588 588 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[6] 589 589 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 590 590 + ms.StoreMessageInfo(mi) 591 591 + } 592 592 + 593 593 + func (x *Secret) String() string { 594 594 + return protoimpl.X.MessageStringOf(x) 595 595 + } 596 596 + 597 597 + func (*Secret) ProtoMessage() {} 598 598 + 599 599 + func (x *Secret) ProtoReflect() protoreflect.Message { 600 600 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[6] 601 601 + if x != nil { 602 602 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 603 603 + if ms.LoadMessageInfo() == nil { 604 604 + ms.StoreMessageInfo(mi) 605 605 + } 606 606 + return ms 607 607 + } 608 608 + return mi.MessageOf(x) 609 609 + } 610 610 + 611 611 + // Deprecated: Use Secret.ProtoReflect.Descriptor instead. 612 612 + func (*Secret) Descriptor() ([]byte, []int) { 613 613 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{6} 614 614 + } 615 615 + 616 616 + func (x *Secret) GetKey() string { 617 617 + if x != nil { 618 618 + return x.Key 619 619 + } 620 620 + return "" 621 621 + } 622 622 + 623 623 + func (x *Secret) GetValue() string { 624 624 + if x != nil { 625 625 + return x.Value 626 626 + } 627 627 + return "" 628 628 + } 629 629 + 630 630 + // promotes a reservation to a running job and hands over the secrets 631 631 + type CommitLease struct { 632 632 + state protoimpl.MessageState `protogen:"open.v1"` 633 633 + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 634 634 + Secrets []*Secret `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty"` 635 635 + unknownFields protoimpl.UnknownFields 636 636 + sizeCache protoimpl.SizeCache 637 637 + } 638 638 + 639 639 + func (x *CommitLease) Reset() { 640 640 + *x = CommitLease{} 641 641 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] 642 642 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 643 643 + ms.StoreMessageInfo(mi) 644 644 + } 645 645 + 646 646 + func (x *CommitLease) String() string { 647 647 + return protoimpl.X.MessageStringOf(x) 648 648 + } 649 649 + 650 650 + func (*CommitLease) ProtoMessage() {} 651 651 + 652 652 + func (x *CommitLease) ProtoReflect() protoreflect.Message { 653 653 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] 654 654 + if x != nil { 655 655 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 656 656 + if ms.LoadMessageInfo() == nil { 657 657 + ms.StoreMessageInfo(mi) 658 658 + } 659 659 + return ms 660 660 + } 661 661 + return mi.MessageOf(x) 662 662 + } 663 663 + 664 664 + // Deprecated: Use CommitLease.ProtoReflect.Descriptor instead. 665 665 + func (*CommitLease) Descriptor() ([]byte, []int) { 666 666 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{7} 667 667 + } 668 668 + 669 669 + func (x *CommitLease) GetLeaseId() string { 670 670 + if x != nil { 671 671 + return x.LeaseId 672 672 + } 673 673 + return "" 674 674 + } 675 675 + 676 676 + func (x *CommitLease) GetSecrets() []*Secret { 677 677 + if x != nil { 678 678 + return x.Secrets 679 679 + } 680 680 + return nil 681 681 + } 682 682 + 683 683 + type Committed struct { 684 684 + state protoimpl.MessageState `protogen:"open.v1"` 685 685 + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 686 686 + unknownFields protoimpl.UnknownFields 687 687 + sizeCache protoimpl.SizeCache 688 688 + } 689 689 + 690 690 + func (x *Committed) Reset() { 691 691 + *x = Committed{} 692 692 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] 693 693 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 694 694 + ms.StoreMessageInfo(mi) 695 695 + } 696 696 + 697 697 + func (x *Committed) String() string { 698 698 + return protoimpl.X.MessageStringOf(x) 699 699 + } 700 700 + 701 701 + func (*Committed) ProtoMessage() {} 702 702 + 703 703 + func (x *Committed) ProtoReflect() protoreflect.Message { 704 704 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] 705 705 + if x != nil { 706 706 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 707 707 + if ms.LoadMessageInfo() == nil { 708 708 + ms.StoreMessageInfo(mi) 709 709 + } 710 710 + return ms 711 711 + } 712 712 + return mi.MessageOf(x) 713 713 + } 714 714 + 715 715 + // Deprecated: Use Committed.ProtoReflect.Descriptor instead. 716 716 + func (*Committed) Descriptor() ([]byte, []int) { 717 717 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{8} 718 718 + } 719 719 + 720 720 + func (x *Committed) GetLeaseId() string { 721 721 + if x != nil { 722 722 + return x.LeaseId 723 723 + } 724 724 + return "" 725 725 + } 726 726 + 727 727 + // drops a reservation that was never committed 728 728 + type ReleaseLease struct { 729 729 + state protoimpl.MessageState `protogen:"open.v1"` 730 730 + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 731 731 + unknownFields protoimpl.UnknownFields 732 732 + sizeCache protoimpl.SizeCache 733 733 + } 734 734 + 735 735 + func (x *ReleaseLease) Reset() { 736 736 + *x = ReleaseLease{} 737 737 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] 738 738 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 739 739 + ms.StoreMessageInfo(mi) 740 740 + } 741 741 + 742 742 + func (x *ReleaseLease) String() string { 743 743 + return protoimpl.X.MessageStringOf(x) 744 744 + } 745 745 + 746 746 + func (*ReleaseLease) ProtoMessage() {} 747 747 + 748 748 + func (x *ReleaseLease) ProtoReflect() protoreflect.Message { 749 749 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] 750 750 + if x != nil { 751 751 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 752 752 + if ms.LoadMessageInfo() == nil { 753 753 + ms.StoreMessageInfo(mi) 754 754 + } 755 755 + return ms 756 756 + } 757 757 + return mi.MessageOf(x) 758 758 + } 759 759 + 760 760 + // Deprecated: Use ReleaseLease.ProtoReflect.Descriptor instead. 761 761 + func (*ReleaseLease) Descriptor() ([]byte, []int) { 762 762 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{9} 763 763 + } 764 764 + 765 765 + func (x *ReleaseLease) GetLeaseId() string { 766 766 + if x != nil { 767 767 + return x.LeaseId 768 768 + } 769 769 + return "" 770 770 + } 771 771 + 772 772 + // cancels a running attempt 773 773 + type CancelAttempt struct { 774 774 + state protoimpl.MessageState `protogen:"open.v1"` 775 775 + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 776 776 + Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` 777 777 + unknownFields protoimpl.UnknownFields 778 778 + sizeCache protoimpl.SizeCache 779 779 + } 780 780 + 781 781 + func (x *CancelAttempt) Reset() { 782 782 + *x = CancelAttempt{} 783 783 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] 784 784 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 785 785 + ms.StoreMessageInfo(mi) 786 786 + } 787 787 + 788 788 + func (x *CancelAttempt) String() string { 789 789 + return protoimpl.X.MessageStringOf(x) 790 790 + } 791 791 + 792 792 + func (*CancelAttempt) ProtoMessage() {} 793 793 + 794 794 + func (x *CancelAttempt) ProtoReflect() protoreflect.Message { 795 795 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] 796 796 + if x != nil { 797 797 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 798 798 + if ms.LoadMessageInfo() == nil { 799 799 + ms.StoreMessageInfo(mi) 800 800 + } 801 801 + return ms 802 802 + } 803 803 + return mi.MessageOf(x) 804 804 + } 805 805 + 806 806 + // Deprecated: Use CancelAttempt.ProtoReflect.Descriptor instead. 807 807 + func (*CancelAttempt) Descriptor() ([]byte, []int) { 808 808 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{10} 809 809 + } 810 810 + 811 811 + func (x *CancelAttempt) GetLeaseId() string { 812 812 + if x != nil { 813 813 + return x.LeaseId 814 814 + } 815 815 + return "" 816 816 + } 817 817 + 818 818 + func (x *CancelAttempt) GetReason() string { 819 819 + if x != nil { 820 820 + return x.Reason 821 821 + } 822 822 + return "" 823 823 + } 824 824 + 825 825 + type CancelAck struct { 826 826 + state protoimpl.MessageState `protogen:"open.v1"` 827 827 + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 828 828 + unknownFields protoimpl.UnknownFields 829 829 + sizeCache protoimpl.SizeCache 830 830 + } 831 831 + 832 832 + func (x *CancelAck) Reset() { 833 833 + *x = CancelAck{} 834 834 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] 835 835 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 836 836 + ms.StoreMessageInfo(mi) 837 837 + } 838 838 + 839 839 + func (x *CancelAck) String() string { 840 840 + return protoimpl.X.MessageStringOf(x) 841 841 + } 842 842 + 843 843 + func (*CancelAck) ProtoMessage() {} 844 844 + 845 845 + func (x *CancelAck) ProtoReflect() protoreflect.Message { 846 846 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] 847 847 + if x != nil { 848 848 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 849 849 + if ms.LoadMessageInfo() == nil { 850 850 + ms.StoreMessageInfo(mi) 851 851 + } 852 852 + return ms 853 853 + } 854 854 + return mi.MessageOf(x) 855 855 + } 856 856 + 857 857 + // Deprecated: Use CancelAck.ProtoReflect.Descriptor instead. 858 858 + func (*CancelAck) Descriptor() ([]byte, []int) { 859 859 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{11} 860 860 + } 861 861 + 862 862 + func (x *CancelAck) GetLeaseId() string { 863 863 + if x != nil { 864 864 + return x.LeaseId 865 865 + } 866 866 + return "" 867 867 + } 868 868 + 869 869 + type StatusEvent struct { 870 870 + state protoimpl.MessageState `protogen:"open.v1"` 871 871 + Status NonterminalStatus `protobuf:"varint,1,opt,name=status,proto3,enum=spindle.mill.v1.NonterminalStatus" json:"status,omitempty"` 872 872 + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` 873 873 + ExitCode int64 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` 874 874 + unknownFields protoimpl.UnknownFields 875 875 + sizeCache protoimpl.SizeCache 876 876 + } 877 877 + 878 878 + func (x *StatusEvent) Reset() { 879 879 + *x = StatusEvent{} 880 880 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] 881 881 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 882 882 + ms.StoreMessageInfo(mi) 883 883 + } 884 884 + 885 885 + func (x *StatusEvent) String() string { 886 886 + return protoimpl.X.MessageStringOf(x) 887 887 + } 888 888 + 889 889 + func (*StatusEvent) ProtoMessage() {} 890 890 + 891 891 + func (x *StatusEvent) ProtoReflect() protoreflect.Message { 892 892 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] 893 893 + if x != nil { 894 894 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 895 895 + if ms.LoadMessageInfo() == nil { 896 896 + ms.StoreMessageInfo(mi) 897 897 + } 898 898 + return ms 899 899 + } 900 900 + return mi.MessageOf(x) 901 901 + } 902 902 + 903 903 + // Deprecated: Use StatusEvent.ProtoReflect.Descriptor instead. 904 904 + func (*StatusEvent) Descriptor() ([]byte, []int) { 905 905 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{12} 906 906 + } 907 907 + 908 908 + func (x *StatusEvent) GetStatus() NonterminalStatus { 909 909 + if x != nil { 910 910 + return x.Status 911 911 + } 912 912 + return NonterminalStatus_NONTERMINAL_STATUS_UNSPECIFIED 913 913 + } 914 914 + 915 915 + func (x *StatusEvent) GetError() string { 916 916 + if x != nil { 917 917 + return x.Error 918 918 + } 919 919 + return "" 920 920 + } 921 921 + 922 922 + func (x *StatusEvent) GetExitCode() int64 { 923 923 + if x != nil { 924 924 + return x.ExitCode 925 925 + } 926 926 + return 0 927 927 + } 928 928 + 929 929 + type LogArtifact struct { 930 930 + state protoimpl.MessageState `protogen:"open.v1"` 931 931 + Ref string `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` 932 932 + Hash string `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"` 933 933 + unknownFields protoimpl.UnknownFields 934 934 + sizeCache protoimpl.SizeCache 935 935 + } 936 936 + 937 937 + func (x *LogArtifact) Reset() { 938 938 + *x = LogArtifact{} 939 939 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] 940 940 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 941 941 + ms.StoreMessageInfo(mi) 942 942 + } 943 943 + 944 944 + func (x *LogArtifact) String() string { 945 945 + return protoimpl.X.MessageStringOf(x) 946 946 + } 947 947 + 948 948 + func (*LogArtifact) ProtoMessage() {} 949 949 + 950 950 + func (x *LogArtifact) ProtoReflect() protoreflect.Message { 951 951 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] 952 952 + if x != nil { 953 953 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 954 954 + if ms.LoadMessageInfo() == nil { 955 955 + ms.StoreMessageInfo(mi) 956 956 + } 957 957 + return ms 958 958 + } 959 959 + return mi.MessageOf(x) 960 960 + } 961 961 + 962 962 + // Deprecated: Use LogArtifact.ProtoReflect.Descriptor instead. 963 963 + func (*LogArtifact) Descriptor() ([]byte, []int) { 964 964 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{13} 965 965 + } 966 966 + 967 967 + func (x *LogArtifact) GetRef() string { 968 968 + if x != nil { 969 969 + return x.Ref 970 970 + } 971 971 + return "" 972 972 + } 973 973 + 974 974 + func (x *LogArtifact) GetHash() string { 975 975 + if x != nil { 976 976 + return x.Hash 977 977 + } 978 978 + return "" 979 979 + } 980 980 + 981 981 + // terminal outcome of an attempt 982 982 + type AttemptResult struct { 983 983 + state protoimpl.MessageState `protogen:"open.v1"` 984 984 + Status TerminalStatus `protobuf:"varint,1,opt,name=status,proto3,enum=spindle.mill.v1.TerminalStatus" json:"status,omitempty"` 985 985 + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` 986 986 + ExitCode int64 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` 987 987 + LogArtifact *LogArtifact `protobuf:"bytes,4,opt,name=log_artifact,json=logArtifact,proto3" json:"log_artifact,omitempty"` 988 988 + unknownFields protoimpl.UnknownFields 989 989 + sizeCache protoimpl.SizeCache 990 990 + } 991 991 + 992 992 + func (x *AttemptResult) Reset() { 993 993 + *x = AttemptResult{} 994 994 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] 995 995 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 996 996 + ms.StoreMessageInfo(mi) 997 997 + } 998 998 + 999 999 + func (x *AttemptResult) String() string { 1000 1000 + return protoimpl.X.MessageStringOf(x) 1001 1001 + } 1002 1002 + 1003 1003 + func (*AttemptResult) ProtoMessage() {} 1004 1004 + 1005 1005 + func (x *AttemptResult) ProtoReflect() protoreflect.Message { 1006 1006 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] 1007 1007 + if x != nil { 1008 1008 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1009 1009 + if ms.LoadMessageInfo() == nil { 1010 1010 + ms.StoreMessageInfo(mi) 1011 1011 + } 1012 1012 + return ms 1013 1013 + } 1014 1014 + return mi.MessageOf(x) 1015 1015 + } 1016 1016 + 1017 1017 + // Deprecated: Use AttemptResult.ProtoReflect.Descriptor instead. 1018 1018 + func (*AttemptResult) Descriptor() ([]byte, []int) { 1019 1019 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{14} 1020 1020 + } 1021 1021 + 1022 1022 + func (x *AttemptResult) GetStatus() TerminalStatus { 1023 1023 + if x != nil { 1024 1024 + return x.Status 1025 1025 + } 1026 1026 + return TerminalStatus_TERMINAL_STATUS_UNSPECIFIED 1027 1027 + } 1028 1028 + 1029 1029 + func (x *AttemptResult) GetError() string { 1030 1030 + if x != nil { 1031 1031 + return x.Error 1032 1032 + } 1033 1033 + return "" 1034 1034 + } 1035 1035 + 1036 1036 + func (x *AttemptResult) GetExitCode() int64 { 1037 1037 + if x != nil { 1038 1038 + return x.ExitCode 1039 1039 + } 1040 1040 + return 0 1041 1041 + } 1042 1042 + 1043 1043 + func (x *AttemptResult) GetLogArtifact() *LogArtifact { 1044 1044 + if x != nil { 1045 1045 + return x.LogArtifact 1046 1046 + } 1047 1047 + return nil 1048 1048 + } 1049 1049 + 1050 1050 + // live non-replay log frame 1051 1051 + type LiveLog struct { 1052 1052 + state protoimpl.MessageState `protogen:"open.v1"` 1053 1053 + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 1054 1054 + RawJson []byte `protobuf:"bytes,2,opt,name=raw_json,json=rawJson,proto3" json:"raw_json,omitempty"` 1055 1055 + unknownFields protoimpl.UnknownFields 1056 1056 + sizeCache protoimpl.SizeCache 1057 1057 + } 1058 1058 + 1059 1059 + func (x *LiveLog) Reset() { 1060 1060 + *x = LiveLog{} 1061 1061 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] 1062 1062 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1063 1063 + ms.StoreMessageInfo(mi) 1064 1064 + } 1065 1065 + 1066 1066 + func (x *LiveLog) String() string { 1067 1067 + return protoimpl.X.MessageStringOf(x) 1068 1068 + } 1069 1069 + 1070 1070 + func (*LiveLog) ProtoMessage() {} 1071 1071 + 1072 1072 + func (x *LiveLog) ProtoReflect() protoreflect.Message { 1073 1073 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] 1074 1074 + if x != nil { 1075 1075 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1076 1076 + if ms.LoadMessageInfo() == nil { 1077 1077 + ms.StoreMessageInfo(mi) 1078 1078 + } 1079 1079 + return ms 1080 1080 + } 1081 1081 + return mi.MessageOf(x) 1082 1082 + } 1083 1083 + 1084 1084 + // Deprecated: Use LiveLog.ProtoReflect.Descriptor instead. 1085 1085 + func (*LiveLog) Descriptor() ([]byte, []int) { 1086 1086 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{15} 1087 1087 + } 1088 1088 + 1089 1089 + func (x *LiveLog) GetLeaseId() string { 1090 1090 + if x != nil { 1091 1091 + return x.LeaseId 1092 1092 + } 1093 1093 + return "" 1094 1094 + } 1095 1095 + 1096 1096 + func (x *LiveLog) GetRawJson() []byte { 1097 1097 + if x != nil { 1098 1098 + return x.RawJson 1099 1099 + } 1100 1100 + return nil 1101 1101 + } 1102 1102 + 1103 1103 + // one event in the executor's stream back to the mill 1104 1104 + type Event struct { 1105 1105 + state protoimpl.MessageState `protogen:"open.v1"` 1106 1106 + Seqno uint64 `protobuf:"varint,1,opt,name=seqno,proto3" json:"seqno,omitempty"` 1107 1107 + LeaseId string `protobuf:"bytes,2,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 1108 1108 + // Types that are valid to be assigned to Payload: 1109 1109 + // 1110 1110 + // *Event_StatusEvent 1111 1111 + // *Event_AttemptResult 1112 1112 + Payload isEvent_Payload `protobuf_oneof:"payload"` 1113 1113 + unknownFields protoimpl.UnknownFields 1114 1114 + sizeCache protoimpl.SizeCache 1115 1115 + } 1116 1116 + 1117 1117 + func (x *Event) Reset() { 1118 1118 + *x = Event{} 1119 1119 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] 1120 1120 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1121 1121 + ms.StoreMessageInfo(mi) 1122 1122 + } 1123 1123 + 1124 1124 + func (x *Event) String() string { 1125 1125 + return protoimpl.X.MessageStringOf(x) 1126 1126 + } 1127 1127 + 1128 1128 + func (*Event) ProtoMessage() {} 1129 1129 + 1130 1130 + func (x *Event) ProtoReflect() protoreflect.Message { 1131 1131 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] 1132 1132 + if x != nil { 1133 1133 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1134 1134 + if ms.LoadMessageInfo() == nil { 1135 1135 + ms.StoreMessageInfo(mi) 1136 1136 + } 1137 1137 + return ms 1138 1138 + } 1139 1139 + return mi.MessageOf(x) 1140 1140 + } 1141 1141 + 1142 1142 + // Deprecated: Use Event.ProtoReflect.Descriptor instead. 1143 1143 + func (*Event) Descriptor() ([]byte, []int) { 1144 1144 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16} 1145 1145 + } 1146 1146 + 1147 1147 + func (x *Event) GetSeqno() uint64 { 1148 1148 + if x != nil { 1149 1149 + return x.Seqno 1150 1150 + } 1151 1151 + return 0 1152 1152 + } 1153 1153 + 1154 1154 + func (x *Event) GetLeaseId() string { 1155 1155 + if x != nil { 1156 1156 + return x.LeaseId 1157 1157 + } 1158 1158 + return "" 1159 1159 + } 1160 1160 + 1161 1161 + func (x *Event) GetPayload() isEvent_Payload { 1162 1162 + if x != nil { 1163 1163 + return x.Payload 1164 1164 + } 1165 1165 + return nil 1166 1166 + } 1167 1167 + 1168 1168 + func (x *Event) GetStatusEvent() *StatusEvent { 1169 1169 + if x != nil { 1170 1170 + if x, ok := x.Payload.(*Event_StatusEvent); ok { 1171 1171 + return x.StatusEvent 1172 1172 + } 1173 1173 + } 1174 1174 + return nil 1175 1175 + } 1176 1176 + 1177 1177 + func (x *Event) GetAttemptResult() *AttemptResult { 1178 1178 + if x != nil { 1179 1179 + if x, ok := x.Payload.(*Event_AttemptResult); ok { 1180 1180 + return x.AttemptResult 1181 1181 + } 1182 1182 + } 1183 1183 + return nil 1184 1184 + } 1185 1185 + 1186 1186 + type isEvent_Payload interface { 1187 1187 + isEvent_Payload() 1188 1188 + } 1189 1189 + 1190 1190 + type Event_StatusEvent struct { 1191 1191 + StatusEvent *StatusEvent `protobuf:"bytes,3,opt,name=status_event,json=statusEvent,proto3,oneof"` 1192 1192 + } 1193 1193 + 1194 1194 + type Event_AttemptResult struct { 1195 1195 + AttemptResult *AttemptResult `protobuf:"bytes,4,opt,name=attempt_result,json=attemptResult,proto3,oneof"` 1196 1196 + } 1197 1197 + 1198 1198 + func (*Event_StatusEvent) isEvent_Payload() {} 1199 1199 + 1200 1200 + func (*Event_AttemptResult) isEvent_Payload() {} 1201 1201 + 1202 1202 + // a flushed bundle of events 1203 1203 + type EventBatch struct { 1204 1204 + state protoimpl.MessageState `protogen:"open.v1"` 1205 1205 + Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"` 1206 1206 + Events []*Event `protobuf:"bytes,2,rep,name=events,proto3" json:"events,omitempty"` 1207 1207 + unknownFields protoimpl.UnknownFields 1208 1208 + sizeCache protoimpl.SizeCache 1209 1209 + } 1210 1210 + 1211 1211 + func (x *EventBatch) Reset() { 1212 1212 + *x = EventBatch{} 1213 1213 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] 1214 1214 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1215 1215 + ms.StoreMessageInfo(mi) 1216 1216 + } 1217 1217 + 1218 1218 + func (x *EventBatch) String() string { 1219 1219 + return protoimpl.X.MessageStringOf(x) 1220 1220 + } 1221 1221 + 1222 1222 + func (*EventBatch) ProtoMessage() {} 1223 1223 + 1224 1224 + func (x *EventBatch) ProtoReflect() protoreflect.Message { 1225 1225 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] 1226 1226 + if x != nil { 1227 1227 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1228 1228 + if ms.LoadMessageInfo() == nil { 1229 1229 + ms.StoreMessageInfo(mi) 1230 1230 + } 1231 1231 + return ms 1232 1232 + } 1233 1233 + return mi.MessageOf(x) 1234 1234 + } 1235 1235 + 1236 1236 + // Deprecated: Use EventBatch.ProtoReflect.Descriptor instead. 1237 1237 + func (*EventBatch) Descriptor() ([]byte, []int) { 1238 1238 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17} 1239 1239 + } 1240 1240 + 1241 1241 + func (x *EventBatch) GetEpoch() string { 1242 1242 + if x != nil { 1243 1243 + return x.Epoch 1244 1244 + } 1245 1245 + return "" 1246 1246 + } 1247 1247 + 1248 1248 + func (x *EventBatch) GetEvents() []*Event { 1249 1249 + if x != nil { 1250 1250 + return x.Events 1251 1251 + } 1252 1252 + return nil 1253 1253 + } 1254 1254 + 1255 1255 + // all events up to and including up_to_seqno are durably processed 1256 1256 + type Ack struct { 1257 1257 + state protoimpl.MessageState `protogen:"open.v1"` 1258 1258 + Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"` 1259 1259 + UpToSeqno uint64 `protobuf:"varint,2,opt,name=up_to_seqno,json=upToSeqno,proto3" json:"up_to_seqno,omitempty"` 1260 1260 + unknownFields protoimpl.UnknownFields 1261 1261 + sizeCache protoimpl.SizeCache 1262 1262 + } 1263 1263 + 1264 1264 + func (x *Ack) Reset() { 1265 1265 + *x = Ack{} 1266 1266 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] 1267 1267 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1268 1268 + ms.StoreMessageInfo(mi) 1269 1269 + } 1270 1270 + 1271 1271 + func (x *Ack) String() string { 1272 1272 + return protoimpl.X.MessageStringOf(x) 1273 1273 + } 1274 1274 + 1275 1275 + func (*Ack) ProtoMessage() {} 1276 1276 + 1277 1277 + func (x *Ack) ProtoReflect() protoreflect.Message { 1278 1278 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] 1279 1279 + if x != nil { 1280 1280 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1281 1281 + if ms.LoadMessageInfo() == nil { 1282 1282 + ms.StoreMessageInfo(mi) 1283 1283 + } 1284 1284 + return ms 1285 1285 + } 1286 1286 + return mi.MessageOf(x) 1287 1287 + } 1288 1288 + 1289 1289 + // Deprecated: Use Ack.ProtoReflect.Descriptor instead. 1290 1290 + func (*Ack) Descriptor() ([]byte, []int) { 1291 1291 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18} 1292 1292 + } 1293 1293 + 1294 1294 + func (x *Ack) GetEpoch() string { 1295 1295 + if x != nil { 1296 1296 + return x.Epoch 1297 1297 + } 1298 1298 + return "" 1299 1299 + } 1300 1300 + 1301 1301 + func (x *Ack) GetUpToSeqno() uint64 { 1302 1302 + if x != nil { 1303 1303 + return x.UpToSeqno 1304 1304 + } 1305 1305 + return 0 1306 1306 + } 1307 1307 + 1308 1308 + type Message struct { 1309 1309 + state protoimpl.MessageState `protogen:"open.v1"` 1310 1310 + Hello *Hello `protobuf:"bytes,1,opt,name=hello,proto3" json:"hello,omitempty"` 1311 1311 + Resume *Resume `protobuf:"bytes,2,opt,name=resume,proto3" json:"resume,omitempty"` 1312 1312 + NodeSnapshot *NodeSnapshot `protobuf:"bytes,3,opt,name=node_snapshot,json=nodeSnapshot,proto3" json:"node_snapshot,omitempty"` 1313 1313 + ReserveSeat *ReserveSeat `protobuf:"bytes,4,opt,name=reserve_seat,json=reserveSeat,proto3" json:"reserve_seat,omitempty"` 1314 1314 + ReserveResult *ReserveResult `protobuf:"bytes,5,opt,name=reserve_result,json=reserveResult,proto3" json:"reserve_result,omitempty"` 1315 1315 + CommitLease *CommitLease `protobuf:"bytes,6,opt,name=commit_lease,json=commitLease,proto3" json:"commit_lease,omitempty"` 1316 1316 + Committed *Committed `protobuf:"bytes,7,opt,name=committed,proto3" json:"committed,omitempty"` 1317 1317 + ReleaseLease *ReleaseLease `protobuf:"bytes,8,opt,name=release_lease,json=releaseLease,proto3" json:"release_lease,omitempty"` 1318 1318 + CancelAttempt *CancelAttempt `protobuf:"bytes,9,opt,name=cancel_attempt,json=cancelAttempt,proto3" json:"cancel_attempt,omitempty"` 1319 1319 + CancelAck *CancelAck `protobuf:"bytes,10,opt,name=cancel_ack,json=cancelAck,proto3" json:"cancel_ack,omitempty"` 1320 1320 + EventBatch *EventBatch `protobuf:"bytes,11,opt,name=event_batch,json=eventBatch,proto3" json:"event_batch,omitempty"` 1321 1321 + Ack *Ack `protobuf:"bytes,12,opt,name=ack,proto3" json:"ack,omitempty"` 1322 1322 + LiveLog *LiveLog `protobuf:"bytes,13,opt,name=live_log,json=liveLog,proto3" json:"live_log,omitempty"` 1323 1323 + unknownFields protoimpl.UnknownFields 1324 1324 + sizeCache protoimpl.SizeCache 1325 1325 + } 1326 1326 + 1327 1327 + func (x *Message) Reset() { 1328 1328 + *x = Message{} 1329 1329 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] 1330 1330 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1331 1331 + ms.StoreMessageInfo(mi) 1332 1332 + } 1333 1333 + 1334 1334 + func (x *Message) String() string { 1335 1335 + return protoimpl.X.MessageStringOf(x) 1336 1336 + } 1337 1337 + 1338 1338 + func (*Message) ProtoMessage() {} 1339 1339 + 1340 1340 + func (x *Message) ProtoReflect() protoreflect.Message { 1341 1341 + mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] 1342 1342 + if x != nil { 1343 1343 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1344 1344 + if ms.LoadMessageInfo() == nil { 1345 1345 + ms.StoreMessageInfo(mi) 1346 1346 + } 1347 1347 + return ms 1348 1348 + } 1349 1349 + return mi.MessageOf(x) 1350 1350 + } 1351 1351 + 1352 1352 + // Deprecated: Use Message.ProtoReflect.Descriptor instead. 1353 1353 + func (*Message) Descriptor() ([]byte, []int) { 1354 1354 + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{19} 1355 1355 + } 1356 1356 + 1357 1357 + func (x *Message) GetHello() *Hello { 1358 1358 + if x != nil { 1359 1359 + return x.Hello 1360 1360 + } 1361 1361 + return nil 1362 1362 + } 1363 1363 + 1364 1364 + func (x *Message) GetResume() *Resume { 1365 1365 + if x != nil { 1366 1366 + return x.Resume 1367 1367 + } 1368 1368 + return nil 1369 1369 + } 1370 1370 + 1371 1371 + func (x *Message) GetNodeSnapshot() *NodeSnapshot { 1372 1372 + if x != nil { 1373 1373 + return x.NodeSnapshot 1374 1374 + } 1375 1375 + return nil 1376 1376 + } 1377 1377 + 1378 1378 + func (x *Message) GetReserveSeat() *ReserveSeat { 1379 1379 + if x != nil { 1380 1380 + return x.ReserveSeat 1381 1381 + } 1382 1382 + return nil 1383 1383 + } 1384 1384 + 1385 1385 + func (x *Message) GetReserveResult() *ReserveResult { 1386 1386 + if x != nil { 1387 1387 + return x.ReserveResult 1388 1388 + } 1389 1389 + return nil 1390 1390 + } 1391 1391 + 1392 1392 + func (x *Message) GetCommitLease() *CommitLease { 1393 1393 + if x != nil { 1394 1394 + return x.CommitLease 1395 1395 + } 1396 1396 + return nil 1397 1397 + } 1398 1398 + 1399 1399 + func (x *Message) GetCommitted() *Committed { 1400 1400 + if x != nil { 1401 1401 + return x.Committed 1402 1402 + } 1403 1403 + return nil 1404 1404 + } 1405 1405 + 1406 1406 + func (x *Message) GetReleaseLease() *ReleaseLease { 1407 1407 + if x != nil { 1408 1408 + return x.ReleaseLease 1409 1409 + } 1410 1410 + return nil 1411 1411 + } 1412 1412 + 1413 1413 + func (x *Message) GetCancelAttempt() *CancelAttempt { 1414 1414 + if x != nil { 1415 1415 + return x.CancelAttempt 1416 1416 + } 1417 1417 + return nil 1418 1418 + } 1419 1419 + 1420 1420 + func (x *Message) GetCancelAck() *CancelAck { 1421 1421 + if x != nil { 1422 1422 + return x.CancelAck 1423 1423 + } 1424 1424 + return nil 1425 1425 + } 1426 1426 + 1427 1427 + func (x *Message) GetEventBatch() *EventBatch { 1428 1428 + if x != nil { 1429 1429 + return x.EventBatch 1430 1430 + } 1431 1431 + return nil 1432 1432 + } 1433 1433 + 1434 1434 + func (x *Message) GetAck() *Ack { 1435 1435 + if x != nil { 1436 1436 + return x.Ack 1437 1437 + } 1438 1438 + return nil 1439 1439 + } 1440 1440 + 1441 1441 + func (x *Message) GetLiveLog() *LiveLog { 1442 1442 + if x != nil { 1443 1443 + return x.LiveLog 1444 1444 + } 1445 1445 + return nil 1446 1446 + } 1447 1447 + 1448 1448 + var File_spindle_mill_v1_mill_proto protoreflect.FileDescriptor 1449 1449 + 1450 1450 + const file_spindle_mill_v1_mill_proto_rawDesc = "" + 1451 1451 + "\n" + 1452 1452 + "\x1aspindle/mill/v1/mill.proto\x12\x0fspindle.mill.v1\x1a\x1bbuf/validate/validate.proto\"}\n" + 1453 1453 + "\x05Hello\x12)\n" + 1454 1454 + "\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x12\x12\n" + 1455 1455 + "\x04arch\x18\x02 \x01(\tR\x04arch\x12\x16\n" + 1456 1456 + "\x06labels\x18\x03 \x03(\tR\x06labels\x12\x1d\n" + 1457 1457 + "\x05epoch\x18\x04 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\"D\n" + 1458 1458 + "\x06Resume\x12\x1d\n" + 1459 1459 + "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1b\n" + 1460 1460 + "\tack_seqno\x18\x02 \x01(\x04R\backSeqno\"\xae\x01\n" + 1461 1461 + "\x12EngineAvailability\x12\x1c\n" + 1462 1462 + "\tavailable\x18\x01 \x01(\bR\tavailable\x12A\n" + 1463 1463 + "\x04load\x18\x02 \x03(\v2-.spindle.mill.v1.EngineAvailability.LoadEntryR\x04load\x1a7\n" + 1464 1464 + "\tLoadEntry\x12\x10\n" + 1465 1465 + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + 1466 1466 + "\x05value\x18\x02 \x01(\x01R\x05value:\x028\x01\"\xfe\x01\n" + 1467 1467 + "\fNodeSnapshot\x12\x1d\n" + 1468 1468 + "\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12D\n" + 1469 1469 + "\aengines\x18\x02 \x03(\v2*.spindle.mill.v1.NodeSnapshot.EnginesEntryR\aengines\x12(\n" + 1470 1470 + "\x10active_lease_ids\x18\x03 \x03(\tR\x0eactiveLeaseIds\x1a_\n" + 1471 1471 + "\fEnginesEntry\x12\x10\n" + 1472 1472 + "\x03key\x18\x01 \x01(\tR\x03key\x129\n" + 1473 1473 + "\x05value\x18\x02 \x01(\v2#.spindle.mill.v1.EngineAvailabilityR\x05value:\x028\x01\"\x80\x02\n" + 1474 1474 + "\vReserveSeat\x12\"\n" + 1475 1475 + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12,\n" + 1476 1476 + "\rtarget_engine\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\ftargetEngine\x12*\n" + 1477 1477 + "\x11raw_pipeline_json\x18\x03 \x01(\tR\x0frawPipelineJson\x12*\n" + 1478 1478 + "\x11raw_workflow_json\x18\x04 \x01(\tR\x0frawWorkflowJson\x12\x12\n" + 1479 1479 + "\x04knot\x18\x05 \x01(\tR\x04knot\x12\x12\n" + 1480 1480 + "\x04rkey\x18\x06 \x01(\tR\x04rkey\x12\x1f\n" + 1481 1481 + "\vttl_seconds\x18\a \x01(\rR\n" + 1482 1482 + "ttlSeconds\"\xbf\x01\n" + 1483 1483 + "\rReserveResult\x12\"\n" + 1484 1484 + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x1a\n" + 1485 1485 + "\baccepted\x18\x02 \x01(\bR\baccepted\x12#\n" + 1486 1486 + "\rreject_reason\x18\x03 \x01(\tR\frejectReason\x12I\n" + 1487 1487 + "\freject_class\x18\x04 \x01(\x0e2\x1c.spindle.mill.v1.RejectClassB\b\xbaH\x05\x82\x01\x02\x10\x01R\vrejectClass\"0\n" + 1488 1488 + "\x06Secret\x12\x10\n" + 1489 1489 + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + 1490 1490 + "\x05value\x18\x02 \x01(\tR\x05value\"d\n" + 1491 1491 + "\vCommitLease\x12\"\n" + 1492 1492 + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x121\n" + 1493 1493 + "\asecrets\x18\x02 \x03(\v2\x17.spindle.mill.v1.SecretR\asecrets\"/\n" + 1494 1494 + "\tCommitted\x12\"\n" + 1495 1495 + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"2\n" + 1496 1496 + "\fReleaseLease\x12\"\n" + 1497 1497 + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"K\n" + 1498 1498 + "\rCancelAttempt\x12\"\n" + 1499 1499 + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x16\n" + 1500 1500 + "\x06reason\x18\x02 \x01(\tR\x06reason\"/\n" + 1501 1501 + "\tCancelAck\x12\"\n" + 1502 1502 + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"\x93\x01\n" + 1503 1503 + "\vStatusEvent\x12F\n" + 1504 1504 + "\x06status\x18\x01 \x01(\x0e2\".spindle.mill.v1.NonterminalStatusB\n" + 1505 1505 + "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" + 1506 1506 + "\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" + 1507 1507 + "\texit_code\x18\x03 \x01(\x03R\bexitCode\"E\n" + 1508 1508 + "\vLogArtifact\x12\x19\n" + 1509 1509 + "\x03ref\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x03ref\x12\x1b\n" + 1510 1510 + "\x04hash\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x04hash\"\xd3\x01\n" + 1511 1511 + "\rAttemptResult\x12C\n" + 1512 1512 + "\x06status\x18\x01 \x01(\x0e2\x1f.spindle.mill.v1.TerminalStatusB\n" + 1513 1513 + "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" + 1514 1514 + "\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" + 1515 1515 + "\texit_code\x18\x03 \x01(\x03R\bexitCode\x12?\n" + 1516 1516 + "\flog_artifact\x18\x04 \x01(\v2\x1c.spindle.mill.v1.LogArtifactR\vlogArtifact\"Q\n" + 1517 1517 + "\aLiveLog\x12\"\n" + 1518 1518 + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\"\n" + 1519 1519 + "\braw_json\x18\x02 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\xe8\x01\n" + 1520 1520 + "\x05Event\x12\x1d\n" + 1521 1521 + "\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12\"\n" + 1522 1522 + "\blease_id\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12A\n" + 1523 1523 + "\fstatus_event\x18\x03 \x01(\v2\x1c.spindle.mill.v1.StatusEventH\x00R\vstatusEvent\x12G\n" + 1524 1524 + "\x0eattempt_result\x18\x04 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResultB\x10\n" + 1525 1525 + "\apayload\x12\x05\xbaH\x02\b\x01\"e\n" + 1526 1526 + "\n" + 1527 1527 + "EventBatch\x12\x1d\n" + 1528 1528 + "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x128\n" + 1529 1529 + "\x06events\x18\x02 \x03(\v2\x16.spindle.mill.v1.EventB\b\xbaH\x05\x92\x01\x02\b\x01R\x06events\"D\n" + 1530 1530 + "\x03Ack\x12\x1d\n" + 1531 1531 + "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1e\n" + 1532 1532 + "\vup_to_seqno\x18\x02 \x01(\x04R\tupToSeqno\"\xb8\a\n" + 1533 1533 + "\aMessage\x12,\n" + 1534 1534 + "\x05hello\x18\x01 \x01(\v2\x16.spindle.mill.v1.HelloR\x05hello\x12/\n" + 1535 1535 + "\x06resume\x18\x02 \x01(\v2\x17.spindle.mill.v1.ResumeR\x06resume\x12B\n" + 1536 1536 + "\rnode_snapshot\x18\x03 \x01(\v2\x1d.spindle.mill.v1.NodeSnapshotR\fnodeSnapshot\x12?\n" + 1537 1537 + "\freserve_seat\x18\x04 \x01(\v2\x1c.spindle.mill.v1.ReserveSeatR\vreserveSeat\x12E\n" + 1538 1538 + "\x0ereserve_result\x18\x05 \x01(\v2\x1e.spindle.mill.v1.ReserveResultR\rreserveResult\x12?\n" + 1539 1539 + "\fcommit_lease\x18\x06 \x01(\v2\x1c.spindle.mill.v1.CommitLeaseR\vcommitLease\x128\n" + 1540 1540 + "\tcommitted\x18\a \x01(\v2\x1a.spindle.mill.v1.CommittedR\tcommitted\x12B\n" + 1541 1541 + "\rrelease_lease\x18\b \x01(\v2\x1d.spindle.mill.v1.ReleaseLeaseR\freleaseLease\x12E\n" + 1542 1542 + "\x0ecancel_attempt\x18\t \x01(\v2\x1e.spindle.mill.v1.CancelAttemptR\rcancelAttempt\x129\n" + 1543 1543 + "\n" + 1544 1544 + "cancel_ack\x18\n" + 1545 1545 + " \x01(\v2\x1a.spindle.mill.v1.CancelAckR\tcancelAck\x12<\n" + 1546 1546 + "\vevent_batch\x18\v \x01(\v2\x1b.spindle.mill.v1.EventBatchR\n" + 1547 1547 + "eventBatch\x12&\n" + 1548 1548 + "\x03ack\x18\f \x01(\v2\x14.spindle.mill.v1.AckR\x03ack\x123\n" + 1549 1549 + "\blive_log\x18\r \x01(\v2\x18.spindle.mill.v1.LiveLogR\aliveLog:\xa5\x01\xbaH\xa1\x01\"\x9e\x01\n" + 1550 1550 + "\x05hello\n" + 1551 1551 + "\x06resume\n" + 1552 1552 + "\rnode_snapshot\n" + 1553 1553 + "\freserve_seat\n" + 1554 1554 + "\x0ereserve_result\n" + 1555 1555 + "\fcommit_lease\n" + 1556 1556 + "\tcommitted\n" + 1557 1557 + "\rrelease_lease\n" + 1558 1558 + "\x0ecancel_attempt\n" + 1559 1559 + "\n" + 1560 1560 + "cancel_ack\n" + 1561 1561 + "\vevent_batch\n" + 1562 1562 + "\x03ack\n" + 1563 1563 + "\blive_log\x10\x01*f\n" + 1564 1564 + "\vRejectClass\x12\x1c\n" + 1565 1565 + "\x18REJECT_CLASS_UNSPECIFIED\x10\x00\x12\x1a\n" + 1566 1566 + "\x16REJECT_CLASS_TRANSIENT\x10\x01\x12\x1d\n" + 1567 1567 + "\x19REJECT_CLASS_INCOMPATIBLE\x10\x02*D\n" + 1568 1568 + "\x11NonterminalStatus\x12\"\n" + 1569 1569 + "\x1eNONTERMINAL_STATUS_UNSPECIFIED\x10\x00\x12\v\n" + 1570 1570 + "\aRUNNING\x10\x01*f\n" + 1571 1571 + "\x0eTerminalStatus\x12\x1f\n" + 1572 1572 + "\x1bTERMINAL_STATUS_UNSPECIFIED\x10\x00\x12\v\n" + 1573 1573 + "\aSUCCESS\x10\x01\x12\n" + 1574 1574 + "\n" + 1575 1575 + "\x06FAILED\x10\x02\x12\v\n" + 1576 1576 + "\aTIMEOUT\x10\x03\x12\r\n" + 1577 1577 + "\tCANCELLED\x10\x04B0Z.tangled.org/core/spindle/mill/proto/gen;millv1b\x06proto3" 1578 1578 + 1579 1579 + var ( 1580 1580 + file_spindle_mill_v1_mill_proto_rawDescOnce sync.Once 1581 1581 + file_spindle_mill_v1_mill_proto_rawDescData []byte 1582 1582 + ) 1583 1583 + 1584 1584 + func file_spindle_mill_v1_mill_proto_rawDescGZIP() []byte { 1585 1585 + file_spindle_mill_v1_mill_proto_rawDescOnce.Do(func() { 1586 1586 + file_spindle_mill_v1_mill_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc))) 1587 1587 + }) 1588 1588 + return file_spindle_mill_v1_mill_proto_rawDescData 1589 1589 + } 1590 1590 + 1591 1591 + var file_spindle_mill_v1_mill_proto_enumTypes = make([]protoimpl.EnumInfo, 3) 1592 1592 + var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 22) 1593 1593 + var file_spindle_mill_v1_mill_proto_goTypes = []any{ 1594 1594 + (RejectClass)(0), // 0: spindle.mill.v1.RejectClass 1595 1595 + (NonterminalStatus)(0), // 1: spindle.mill.v1.NonterminalStatus 1596 1596 + (TerminalStatus)(0), // 2: spindle.mill.v1.TerminalStatus 1597 1597 + (*Hello)(nil), // 3: spindle.mill.v1.Hello 1598 1598 + (*Resume)(nil), // 4: spindle.mill.v1.Resume 1599 1599 + (*EngineAvailability)(nil), // 5: spindle.mill.v1.EngineAvailability 1600 1600 + (*NodeSnapshot)(nil), // 6: spindle.mill.v1.NodeSnapshot 1601 1601 + (*ReserveSeat)(nil), // 7: spindle.mill.v1.ReserveSeat 1602 1602 + (*ReserveResult)(nil), // 8: spindle.mill.v1.ReserveResult 1603 1603 + (*Secret)(nil), // 9: spindle.mill.v1.Secret 1604 1604 + (*CommitLease)(nil), // 10: spindle.mill.v1.CommitLease 1605 1605 + (*Committed)(nil), // 11: spindle.mill.v1.Committed 1606 1606 + (*ReleaseLease)(nil), // 12: spindle.mill.v1.ReleaseLease 1607 1607 + (*CancelAttempt)(nil), // 13: spindle.mill.v1.CancelAttempt 1608 1608 + (*CancelAck)(nil), // 14: spindle.mill.v1.CancelAck 1609 1609 + (*StatusEvent)(nil), // 15: spindle.mill.v1.StatusEvent 1610 1610 + (*LogArtifact)(nil), // 16: spindle.mill.v1.LogArtifact 1611 1611 + (*AttemptResult)(nil), // 17: spindle.mill.v1.AttemptResult 1612 1612 + (*LiveLog)(nil), // 18: spindle.mill.v1.LiveLog 1613 1613 + (*Event)(nil), // 19: spindle.mill.v1.Event 1614 1614 + (*EventBatch)(nil), // 20: spindle.mill.v1.EventBatch 1615 1615 + (*Ack)(nil), // 21: spindle.mill.v1.Ack 1616 1616 + (*Message)(nil), // 22: spindle.mill.v1.Message 1617 1617 + nil, // 23: spindle.mill.v1.EngineAvailability.LoadEntry 1618 1618 + nil, // 24: spindle.mill.v1.NodeSnapshot.EnginesEntry 1619 1619 + } 1620 1620 + var file_spindle_mill_v1_mill_proto_depIdxs = []int32{ 1621 1621 + 23, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry 1622 1622 + 24, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry 1623 1623 + 0, // 2: spindle.mill.v1.ReserveResult.reject_class:type_name -> spindle.mill.v1.RejectClass 1624 1624 + 9, // 3: spindle.mill.v1.CommitLease.secrets:type_name -> spindle.mill.v1.Secret 1625 1625 + 1, // 4: spindle.mill.v1.StatusEvent.status:type_name -> spindle.mill.v1.NonterminalStatus 1626 1626 + 2, // 5: spindle.mill.v1.AttemptResult.status:type_name -> spindle.mill.v1.TerminalStatus 1627 1627 + 16, // 6: spindle.mill.v1.AttemptResult.log_artifact:type_name -> spindle.mill.v1.LogArtifact 1628 1628 + 15, // 7: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent 1629 1629 + 17, // 8: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult 1630 1630 + 19, // 9: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event 1631 1631 + 3, // 10: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello 1632 1632 + 4, // 11: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume 1633 1633 + 6, // 12: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot 1634 1634 + 7, // 13: spindle.mill.v1.Message.reserve_seat:type_name -> spindle.mill.v1.ReserveSeat 1635 1635 + 8, // 14: spindle.mill.v1.Message.reserve_result:type_name -> spindle.mill.v1.ReserveResult 1636 1636 + 10, // 15: spindle.mill.v1.Message.commit_lease:type_name -> spindle.mill.v1.CommitLease 1637 1637 + 11, // 16: spindle.mill.v1.Message.committed:type_name -> spindle.mill.v1.Committed 1638 1638 + 12, // 17: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease 1639 1639 + 13, // 18: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt 1640 1640 + 14, // 19: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck 1641 1641 + 20, // 20: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch 1642 1642 + 21, // 21: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack 1643 1643 + 18, // 22: spindle.mill.v1.Message.live_log:type_name -> spindle.mill.v1.LiveLog 1644 1644 + 5, // 23: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability 1645 1645 + 24, // [24:24] is the sub-list for method output_type 1646 1646 + 24, // [24:24] is the sub-list for method input_type 1647 1647 + 24, // [24:24] is the sub-list for extension type_name 1648 1648 + 24, // [24:24] is the sub-list for extension extendee 1649 1649 + 0, // [0:24] is the sub-list for field type_name 1650 1650 + } 1651 1651 + 1652 1652 + func init() { file_spindle_mill_v1_mill_proto_init() } 1653 1653 + func file_spindle_mill_v1_mill_proto_init() { 1654 1654 + if File_spindle_mill_v1_mill_proto != nil { 1655 1655 + return 1656 1656 + } 1657 1657 + file_spindle_mill_v1_mill_proto_msgTypes[16].OneofWrappers = []any{ 1658 1658 + (*Event_StatusEvent)(nil), 1659 1659 + (*Event_AttemptResult)(nil), 1660 1660 + } 1661 1661 + type x struct{} 1662 1662 + out := protoimpl.TypeBuilder{ 1663 1663 + File: protoimpl.DescBuilder{ 1664 1664 + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 1665 1665 + RawDescriptor: unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc)), 1666 1666 + NumEnums: 3, 1667 1667 + NumMessages: 22, 1668 1668 + NumExtensions: 0, 1669 1669 + NumServices: 0, 1670 1670 + }, 1671 1671 + GoTypes: file_spindle_mill_v1_mill_proto_goTypes, 1672 1672 + DependencyIndexes: file_spindle_mill_v1_mill_proto_depIdxs, 1673 1673 + EnumInfos: file_spindle_mill_v1_mill_proto_enumTypes, 1674 1674 + MessageInfos: file_spindle_mill_v1_mill_proto_msgTypes, 1675 1675 + }.Build() 1676 1676 + File_spindle_mill_v1_mill_proto = out.File 1677 1677 + file_spindle_mill_v1_mill_proto_goTypes = nil 1678 1678 + file_spindle_mill_v1_mill_proto_depIdxs = nil 1679 1679 + }

+103

spindle/mill/proto/protocol.go

Reviewed

··· 1 1 + // package millproto carries the mill<->executor session protocol. a new message 2 2 + // vocabulary over the same length-prefixed protobuf framing the spindle already 3 3 + // uses to talk to the microVM guest (see spindle/agentproto). only the framing 4 4 + // pattern is shared. the messages are entirely separate 5 5 + package millproto 6 6 + 7 7 + import ( 8 8 + "encoding/binary" 9 9 + "fmt" 10 10 + "io" 11 11 + "sync" 12 12 + 13 13 + "buf.build/go/protovalidate" 14 14 + "google.golang.org/protobuf/proto" 15 15 + 16 16 + millv1 "tangled.org/core/spindle/mill/proto/gen" 17 17 + ) 18 18 + 19 19 + const ( 20 20 + ProtocolVersion = 1 21 21 + // generous vs agentproto's 1 MiB. a ReserveSeat carries the raw pipeline and 22 22 + // workflow JSON, and streamed log lines can be chunky 23 23 + MaxMessageBytes = 8 * 1024 * 1024 24 24 + ) 25 25 + 26 26 + type Message = millv1.Message 27 27 + 28 28 + var validator protovalidate.Validator 29 29 + 30 30 + func init() { 31 31 + var err error 32 32 + validator, err = protovalidate.New() 33 33 + if err != nil { 34 34 + panic(fmt.Errorf("failed to initialize protovalidate validator: %w", err)) 35 35 + } 36 36 + } 37 37 + 38 38 + type Encoder struct { 39 39 + mu sync.Mutex 40 40 + w io.Writer 41 41 + } 42 42 + 43 43 + func NewEncoder(w io.Writer) *Encoder { 44 44 + return &Encoder{w: w} 45 45 + } 46 46 + 47 47 + func (e *Encoder) Encode(msg *Message) error { 48 48 + if err := validator.Validate(msg); err != nil { 49 49 + return fmt.Errorf("validate fleet message: %w", err) 50 50 + } 51 51 + 52 52 + data, err := proto.Marshal(msg) 53 53 + if err != nil { 54 54 + return fmt.Errorf("marshal fleet message: %w", err) 55 55 + } 56 56 + if len(data) > MaxMessageBytes { 57 57 + return fmt.Errorf("fleet message exceeded %d bytes", MaxMessageBytes) 58 58 + } 59 59 + 60 60 + // single write of header and payload maps to exactly one websocket binary 61 61 + // frame when the writer is a ws stream 62 62 + frame := make([]byte, 4+len(data)) 63 63 + binary.BigEndian.PutUint32(frame[:4], uint32(len(data))) 64 64 + copy(frame[4:], data) 65 65 + 66 66 + e.mu.Lock() 67 67 + defer e.mu.Unlock() 68 68 + _, err = e.w.Write(frame) 69 69 + return err 70 70 + } 71 71 + 72 72 + type Decoder struct { 73 73 + r io.Reader 74 74 + } 75 75 + 76 76 + func NewDecoder(r io.Reader) *Decoder { 77 77 + return &Decoder{r: r} 78 78 + } 79 79 + 80 80 + func (d *Decoder) Decode() (*Message, error) { 81 81 + msg := &Message{} 82 82 + var header [4]byte 83 83 + if _, err := io.ReadFull(d.r, header[:]); err != nil { 84 84 + return msg, err 85 85 + } 86 86 + 87 87 + size := binary.BigEndian.Uint32(header[:]) 88 88 + if size > MaxMessageBytes { 89 89 + return msg, fmt.Errorf("fleet message exceeded %d bytes", MaxMessageBytes) 90 90 + } 91 91 + 92 92 + data := make([]byte, size) 93 93 + if _, err := io.ReadFull(d.r, data); err != nil { 94 94 + return msg, err 95 95 + } 96 96 + if err := proto.Unmarshal(data, msg); err != nil { 97 97 + return msg, fmt.Errorf("parse fleet message: %w", err) 98 98 + } 99 99 + if err := validator.Validate(msg); err != nil { 100 100 + return msg, fmt.Errorf("validate fleet message: %w", err) 101 101 + } 102 102 + return msg, nil 103 103 + }

+227

spindle/mill/proto/protocol_test.go

Reviewed

··· 1 1 + package millproto 2 2 + 3 3 + import ( 4 4 + "bytes" 5 5 + "encoding/binary" 6 6 + "testing" 7 7 + 8 8 + millv1 "tangled.org/core/spindle/mill/proto/gen" 9 9 + ) 10 10 + 11 11 + func TestEncodeDecodeRoundTrip(t *testing.T) { 12 12 + var buf bytes.Buffer 13 13 + enc := NewEncoder(&buf) 14 14 + 15 15 + want := &Message{ 16 16 + ReserveSeat: &millv1.ReserveSeat{ 17 17 + LeaseId: "lease-1", 18 18 + TargetEngine: "microvm", 19 19 + RawWorkflowJson: `{"name":"build"}`, 20 20 + Knot: "knot.example", 21 21 + Rkey: "abc123", 22 22 + TtlSeconds: 30, 23 23 + }, 24 24 + } 25 25 + if err := enc.Encode(want); err != nil { 26 26 + t.Fatalf("Encode() error = %v", err) 27 27 + } 28 28 + 29 29 + got, err := NewDecoder(&buf).Decode() 30 30 + if err != nil { 31 31 + t.Fatalf("Decode() error = %v", err) 32 32 + } 33 33 + rs := got.GetReserveSeat() 34 34 + if rs == nil { 35 35 + t.Fatal("decoded message missing reserve_seat") 36 36 + } 37 37 + if rs.LeaseId != "lease-1" || rs.TargetEngine != "microvm" || rs.TtlSeconds != 30 { 38 38 + t.Fatalf("round-trip mismatch: %+v", rs) 39 39 + } 40 40 + } 41 41 + 42 42 + func TestDecoderRejectsOversizedMessage(t *testing.T) { 43 43 + var tooLarge bytes.Buffer 44 44 + var header [4]byte 45 45 + binary.BigEndian.PutUint32(header[:], MaxMessageBytes+1) 46 46 + tooLarge.Write(header[:]) 47 47 + 48 48 + if _, err := NewDecoder(&tooLarge).Decode(); err == nil { 49 49 + t.Fatal("expected oversized message error") 50 50 + } 51 51 + } 52 52 + 53 53 + func TestValidationRules(t *testing.T) { 54 54 + tests := []struct { 55 55 + name string 56 56 + msg *Message 57 57 + wantErr bool 58 58 + }{ 59 59 + { 60 60 + name: "valid ack message", 61 61 + msg: &Message{ 62 62 + Ack: &millv1.Ack{ 63 63 + Epoch: "inc-1", 64 64 + UpToSeqno: 5, 65 65 + }, 66 66 + }, 67 67 + wantErr: false, 68 68 + }, 69 69 + { 70 70 + name: "valid hello message", 71 71 + msg: &Message{ 72 72 + Hello: &millv1.Hello{ 73 73 + ProtocolVersion: 1, 74 74 + Arch: "amd64", 75 75 + Labels: []string{"linux"}, 76 76 + Epoch: "inc-1", 77 77 + }, 78 78 + }, 79 79 + wantErr: false, 80 80 + }, 81 81 + { 82 82 + name: "invalid message with zero payloads", 83 83 + msg: &Message{}, 84 84 + wantErr: true, 85 85 + }, 86 86 + { 87 87 + name: "invalid message with multiple payloads", 88 88 + msg: &Message{ 89 89 + Ack: &millv1.Ack{Epoch: "inc-1", UpToSeqno: 5}, 90 90 + Committed: &millv1.Committed{LeaseId: "x"}, 91 91 + }, 92 92 + wantErr: true, 93 93 + }, 94 94 + { 95 95 + name: "invalid ack message missing epoch", 96 96 + msg: &Message{ 97 97 + Ack: &millv1.Ack{ 98 98 + UpToSeqno: 5, 99 99 + }, 100 100 + }, 101 101 + wantErr: true, 102 102 + }, 103 103 + { 104 104 + name: "invalid node snapshot with zero seqno", 105 105 + msg: &Message{ 106 106 + NodeSnapshot: &millv1.NodeSnapshot{ 107 107 + Seqno: 0, 108 108 + }, 109 109 + }, 110 110 + wantErr: true, 111 111 + }, 112 112 + { 113 113 + name: "valid node snapshot with positive seqno", 114 114 + msg: &Message{ 115 115 + NodeSnapshot: &millv1.NodeSnapshot{ 116 116 + Seqno: 1, 117 117 + }, 118 118 + }, 119 119 + wantErr: false, 120 120 + }, 121 121 + { 122 122 + name: "invalid stream batch with zero seqno entry", 123 123 + msg: &Message{ 124 124 + EventBatch: &millv1.EventBatch{ 125 125 + Epoch: "inc-1", 126 126 + Events: []*millv1.Event{ 127 127 + { 128 128 + Seqno: 0, 129 129 + LeaseId: "lease-1", 130 130 + Payload: &millv1.Event_StatusEvent{ 131 131 + StatusEvent: &millv1.StatusEvent{ 132 132 + Status: millv1.NonterminalStatus_RUNNING, 133 133 + }, 134 134 + }, 135 135 + }, 136 136 + }, 137 137 + }, 138 138 + }, 139 139 + wantErr: true, 140 140 + }, 141 141 + { 142 142 + name: "invalid stream batch with empty entries", 143 143 + msg: &Message{ 144 144 + EventBatch: &millv1.EventBatch{ 145 145 + Epoch: "inc-1", 146 146 + Events: []*millv1.Event{}, 147 147 + }, 148 148 + }, 149 149 + wantErr: true, 150 150 + }, 151 151 + { 152 152 + name: "invalid reserve result with unknown enum", 153 153 + msg: &Message{ 154 154 + ReserveResult: &millv1.ReserveResult{ 155 155 + LeaseId: "lease-1", 156 156 + RejectClass: millv1.RejectClass(99), 157 157 + }, 158 158 + }, 159 159 + wantErr: true, 160 160 + }, 161 161 + { 162 162 + name: "invalid stream entry with malformed oneof (empty payload)", 163 163 + msg: &Message{ 164 164 + EventBatch: &millv1.EventBatch{ 165 165 + Epoch: "inc-1", 166 166 + Events: []*millv1.Event{ 167 167 + { 168 168 + Seqno: 1, 169 169 + LeaseId: "lease-1", 170 170 + Payload: nil, 171 171 + }, 172 172 + }, 173 173 + }, 174 174 + }, 175 175 + wantErr: true, 176 176 + }, 177 177 + { 178 178 + name: "valid stream batch status event", 179 179 + msg: &Message{ 180 180 + EventBatch: &millv1.EventBatch{ 181 181 + Epoch: "inc-1", 182 182 + Events: []*millv1.Event{ 183 183 + { 184 184 + Seqno: 1, 185 185 + LeaseId: "lease-1", 186 186 + Payload: &millv1.Event_StatusEvent{ 187 187 + StatusEvent: &millv1.StatusEvent{ 188 188 + Status: millv1.NonterminalStatus_RUNNING, 189 189 + }, 190 190 + }, 191 191 + }, 192 192 + }, 193 193 + }, 194 194 + }, 195 195 + wantErr: false, 196 196 + }, 197 197 + { 198 198 + name: "valid stream batch attempt result", 199 199 + msg: &Message{ 200 200 + EventBatch: &millv1.EventBatch{ 201 201 + Epoch: "inc-1", 202 202 + Events: []*millv1.Event{ 203 203 + { 204 204 + Seqno: 1, 205 205 + LeaseId: "lease-1", 206 206 + Payload: &millv1.Event_AttemptResult{ 207 207 + AttemptResult: &millv1.AttemptResult{ 208 208 + Status: millv1.TerminalStatus_SUCCESS, 209 209 + }, 210 210 + }, 211 211 + }, 212 212 + }, 213 213 + }, 214 214 + }, 215 215 + wantErr: false, 216 216 + }, 217 217 + } 218 218 + 219 219 + for _, tc := range tests { 220 220 + t.Run(tc.name, func(t *testing.T) { 221 221 + err := validator.Validate(tc.msg) 222 222 + if (err != nil) != tc.wantErr { 223 223 + t.Fatalf("Validate() error = %v, wantErr = %v", err, tc.wantErr) 224 224 + } 225 225 + }) 226 226 + } 227 227 + }

+187

spindle/mill/proto/spindle/mill/v1/mill.proto

Reviewed

··· 1 1 + syntax = "proto3"; 2 2 + 3 3 + package spindle.mill.v1; 4 4 + 5 5 + import "buf/validate/validate.proto"; 6 6 + 7 7 + option go_package = "tangled.org/core/spindle/mill/proto/gen;millv1"; 8 8 + 9 9 + // executor identity, sent on connect 10 10 + message Hello { 11 11 + uint32 protocol_version = 1; 12 12 + // GOARCH of the node, informational only 13 13 + string arch = 2; 14 14 + // operator-defined labels, matched against runs_on 15 15 + repeated string labels = 3; 16 16 + string epoch = 4 [(buf.validate.field).string.min_len = 1]; 17 17 + } 18 18 + 19 19 + // reconnect state for an existing epoch 20 20 + message Resume { 21 21 + string epoch = 1 [(buf.validate.field).string.min_len = 1]; 22 22 + uint64 ack_seqno = 2; 23 23 + } 24 24 + 25 25 + // per-engine state of a node 26 26 + message EngineAvailability { 27 27 + bool available = 1; 28 28 + // opaque engine-defined load metrics, higher means more loaded 29 29 + map<string, double> load = 2; 30 30 + } 31 31 + 32 32 + // full node state 33 33 + message NodeSnapshot { 34 34 + uint64 seqno = 1 [(buf.validate.field).uint64.gt = 0]; 35 35 + map<string, EngineAvailability> engines = 2; 36 36 + // every lease the executor currently holds, reserved or running 37 37 + repeated string active_lease_ids = 3; 38 38 + } 39 39 + 40 40 + // a seat reservation, carries no secrets 41 41 + message ReserveSeat { 42 42 + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 43 43 + string target_engine = 2 [(buf.validate.field).string.min_len = 1]; 44 44 + string raw_pipeline_json = 3; 45 45 + string raw_workflow_json = 4; 46 46 + // pipeline id, the executor reconstructs the exact WorkflowId from it 47 47 + string knot = 5; 48 48 + string rkey = 6; 49 49 + uint32 ttl_seconds = 7; 50 50 + } 51 51 + 52 52 + enum RejectClass { 53 53 + REJECT_CLASS_UNSPECIFIED = 0; 54 54 + REJECT_CLASS_TRANSIENT = 1; 55 55 + REJECT_CLASS_INCOMPATIBLE = 2; 56 56 + } 57 57 + 58 58 + message ReserveResult { 59 59 + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 60 60 + bool accepted = 2; 61 61 + string reject_reason = 3; 62 62 + RejectClass reject_class = 4 [(buf.validate.field).enum.defined_only = true]; 63 63 + } 64 64 + 65 65 + // a single unlocked secret 66 66 + message Secret { 67 67 + string key = 1; 68 68 + string value = 2; 69 69 + } 70 70 + 71 71 + // promotes a reservation to a running job and hands over the secrets 72 72 + message CommitLease { 73 73 + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 74 74 + repeated Secret secrets = 2; 75 75 + } 76 76 + 77 77 + message Committed { 78 78 + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 79 79 + } 80 80 + 81 81 + // drops a reservation that was never committed 82 82 + message ReleaseLease { 83 83 + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 84 84 + } 85 85 + 86 86 + // cancels a running attempt 87 87 + message CancelAttempt { 88 88 + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 89 89 + string reason = 2; 90 90 + } 91 91 + 92 92 + message CancelAck { 93 93 + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 94 94 + } 95 95 + 96 96 + enum NonterminalStatus { 97 97 + NONTERMINAL_STATUS_UNSPECIFIED = 0; 98 98 + RUNNING = 1; 99 99 + } 100 100 + 101 101 + enum TerminalStatus { 102 102 + TERMINAL_STATUS_UNSPECIFIED = 0; 103 103 + SUCCESS = 1; 104 104 + FAILED = 2; 105 105 + TIMEOUT = 3; 106 106 + CANCELLED = 4; 107 107 + } 108 108 + 109 109 + message StatusEvent { 110 110 + NonterminalStatus status = 1 [(buf.validate.field).enum = { 111 111 + defined_only: true 112 112 + not_in: 0 113 113 + }]; 114 114 + string error = 2 [(buf.validate.field).string.max_bytes = 65536]; 115 115 + int64 exit_code = 3; 116 116 + } 117 117 + 118 118 + message LogArtifact { 119 119 + string ref = 1 [(buf.validate.field).string.min_len = 1]; 120 120 + string hash = 2 [(buf.validate.field).string.min_len = 1]; 121 121 + } 122 122 + 123 123 + // terminal outcome of an attempt 124 124 + message AttemptResult { 125 125 + TerminalStatus status = 1 [(buf.validate.field).enum = { 126 126 + defined_only: true 127 127 + not_in: 0 128 128 + }]; 129 129 + string error = 2 [(buf.validate.field).string.max_bytes = 65536]; 130 130 + int64 exit_code = 3; 131 131 + LogArtifact log_artifact = 4; 132 132 + } 133 133 + 134 134 + // live non-replay log frame 135 135 + message LiveLog { 136 136 + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 137 137 + bytes raw_json = 2 [(buf.validate.field).bytes.min_len = 1]; 138 138 + } 139 139 + 140 140 + // one event in the executor's stream back to the mill 141 141 + message Event { 142 142 + uint64 seqno = 1 [(buf.validate.field).uint64.gt = 0]; 143 143 + string lease_id = 2 [(buf.validate.field).string.min_len = 1]; 144 144 + 145 145 + oneof payload { 146 146 + option (buf.validate.oneof).required = true; 147 147 + StatusEvent status_event = 3; 148 148 + AttemptResult attempt_result = 4; 149 149 + } 150 150 + } 151 151 + 152 152 + // a flushed bundle of events 153 153 + message EventBatch { 154 154 + string epoch = 1 [(buf.validate.field).string.min_len = 1]; 155 155 + repeated Event events = 2 [(buf.validate.field).repeated.min_items = 1]; 156 156 + } 157 157 + 158 158 + // all events up to and including up_to_seqno are durably processed 159 159 + message Ack { 160 160 + string epoch = 1 [(buf.validate.field).string.min_len = 1]; 161 161 + uint64 up_to_seqno = 2; 162 162 + } 163 163 + 164 164 + message Message { 165 165 + option (buf.validate.message).oneof = { 166 166 + fields: [ 167 167 + "hello", "resume", "node_snapshot", "reserve_seat", "reserve_result", 168 168 + "commit_lease", "committed", "release_lease", "cancel_attempt", 169 169 + "cancel_ack", "event_batch", "ack", "live_log" 170 170 + ], 171 171 + required: true 172 172 + }; 173 173 + 174 174 + Hello hello = 1; 175 175 + Resume resume = 2; 176 176 + NodeSnapshot node_snapshot = 3; 177 177 + ReserveSeat reserve_seat = 4; 178 178 + ReserveResult reserve_result = 5; 179 179 + CommitLease commit_lease = 6; 180 180 + Committed committed = 7; 181 181 + ReleaseLease release_lease = 8; 182 182 + CancelAttempt cancel_attempt = 9; 183 183 + CancelAck cancel_ack = 10; 184 184 + EventBatch event_batch = 11; 185 185 + Ack ack = 12; 186 186 + LiveLog live_log = 13; 187 187 + }

+58

spindle/mill/proto/ws.go

Reviewed

··· 1 1 + package millproto 2 2 + 3 3 + import ( 4 4 + "io" 5 5 + "sync" 6 6 + 7 7 + "github.com/gorilla/websocket" 8 8 + ) 9 9 + 10 10 + // adapts a gorilla websocket connection to an io.ReadWriteCloser so the 11 11 + // length-prefixed fleet framing rides over it. each Encode produces exactly one 12 12 + // binary frame. the reader reassembles the byte stream across frames 13 13 + type WSStream struct { 14 14 + conn *websocket.Conn 15 15 + 16 16 + rmu sync.Mutex 17 17 + r io.Reader // current message reader, advanced as frames are consumed 18 18 + 19 19 + wmu sync.Mutex 20 20 + } 21 21 + 22 22 + func NewWSStream(conn *websocket.Conn) *WSStream { 23 23 + return &WSStream{conn: conn} 24 24 + } 25 25 + 26 26 + func (s *WSStream) Read(p []byte) (int, error) { 27 27 + s.rmu.Lock() 28 28 + defer s.rmu.Unlock() 29 29 + for { 30 30 + if s.r == nil { 31 31 + _, r, err := s.conn.NextReader() 32 32 + if err != nil { 33 33 + return 0, err 34 34 + } 35 35 + s.r = r 36 36 + } 37 37 + n, err := s.r.Read(p) 38 38 + if err == io.EOF { 39 39 + s.r = nil 40 40 + if n > 0 { 41 41 + return n, nil 42 42 + } 43 43 + continue 44 44 + } 45 45 + return n, err 46 46 + } 47 47 + } 48 48 + 49 49 + func (s *WSStream) Write(p []byte) (int, error) { 50 50 + s.wmu.Lock() 51 51 + defer s.wmu.Unlock() 52 52 + if err := s.conn.WriteMessage(websocket.BinaryMessage, p); err != nil { 53 53 + return 0, err 54 54 + } 55 55 + return len(p), nil 56 56 + } 57 57 + 58 58 + func (s *WSStream) Close() error { return s.conn.Close() }

+221

spindle/mill/restore.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "fmt" 5 5 + "tangled.org/core/spindle/db" 6 6 + millproto "tangled.org/core/spindle/mill/proto" 7 7 + millv1 "tangled.org/core/spindle/mill/proto/gen" 8 8 + "tangled.org/core/spindle/models" 9 9 + "time" 10 10 + ) 11 11 + 12 12 + const ( 13 13 + leaseRowReserved = "reserved" 14 14 + leaseRowRunning = "running" 15 15 + ) 16 16 + 17 17 + func (m *Mill) persistLease(lease *RemoteLease, state string) error { 18 18 + if m.db == nil { 19 19 + return nil 20 20 + } 21 21 + return m.db.SaveMillLease(db.MillLease{ 22 22 + LeaseID: lease.id, 23 23 + NodeID: lease.nodeID, 24 24 + Epoch: lease.epoch, 25 25 + Engine: lease.engine, 26 26 + Knot: lease.wid.Knot, 27 27 + Rkey: lease.wid.Rkey, 28 28 + Workflow: lease.wid.Name, 29 29 + State: state, 30 30 + }) 31 31 + } 32 32 + 33 33 + func (m *Mill) RestoreState() error { 34 34 + if m.db == nil { 35 35 + return nil 36 36 + } 37 37 + cursors, err := m.db.ListExecutorCursors() 38 38 + if err != nil { 39 39 + return err 40 40 + } 41 41 + rows, err := m.db.ListMillLeases() 42 42 + if err != nil { 43 43 + return err 44 44 + } 45 45 + 46 46 + m.mu.Lock() 47 47 + for _, c := range cursors { 48 48 + m.nodeSeqno[c.NodeID+"/"+c.Epoch] = c.AckedSeqno 49 49 + } 50 50 + for _, r := range rows { 51 51 + lease := newLease(r.LeaseID, r.NodeID, r.Epoch, r.Engine) 52 52 + lease.wid = models.WorkflowId{ 53 53 + PipelineId: models.PipelineId{Knot: r.Knot, Rkey: r.Rkey}, 54 54 + Name: r.Workflow, 55 55 + } 56 56 + // restored leases start as orphans, an executor must reclaim it via 57 57 + // its first snapshot, or the sweep will fail it 58 58 + lease.orphaned = true 59 59 + lease.claimed = false 60 60 + if r.State == leaseRowRunning { 61 61 + lease.state = leaseRunning 62 62 + } 63 63 + m.leases[r.LeaseID] = lease 64 64 + } 65 65 + restored := len(rows) 66 66 + m.mu.Unlock() 67 67 + 68 68 + if restored > 0 { 69 69 + m.l.Info("restored mill leases from previous run", "leases", restored, "cursors", len(cursors)) 70 70 + // executors get a grace window to reconnect and claim their leases 71 71 + time.AfterFunc(m.cfg.ReconnectGrace, m.sweepUnclaimedOrphans) 72 72 + } 73 73 + return nil 74 74 + } 75 75 + func (m *Mill) sweepUnclaimedOrphans() { 76 76 + m.mu.Lock() 77 77 + var unclaimed []*RemoteLease 78 78 + for _, lease := range m.leases { 79 79 + if !lease.orphaned { 80 80 + continue 81 81 + } 82 82 + if lease.claimed { 83 83 + continue 84 84 + } 85 85 + if lease.getState() == leaseDone { 86 86 + continue 87 87 + } 88 88 + if sess := m.sessions[lease.nodeID]; sess == nil || sess.disconnected { 89 89 + unclaimed = append(unclaimed, lease) 90 90 + } 91 91 + } 92 92 + m.mu.Unlock() 93 93 + 94 94 + retry := false 95 95 + reason := "executor did not reconnect after mill restart" 96 96 + for _, lease := range unclaimed { 97 97 + m.l.Warn("failing unclaimed restored lease", "lease", lease.id, "node", lease.nodeID) 98 98 + if err := m.finishOrphan(lease, string(models.StatusKindFailed), &reason, nil); err != nil { 99 99 + m.l.Error("finish unclaimed restored lease", "lease", lease.id, "err", err) 100 100 + retry = true 101 101 + } 102 102 + } 103 103 + if retry { 104 104 + time.AfterFunc(5*time.Second, m.sweepUnclaimedOrphans) 105 105 + } 106 106 + } 107 107 + 108 108 + func (m *Mill) reconcileLeases(sess *millSession, activeLeaseIDs []string) error { 109 109 + active := make(map[string]struct{}, len(activeLeaseIDs)) 110 110 + for _, id := range activeLeaseIDs { 111 111 + active[id] = struct{}{} 112 112 + } 113 113 + 114 114 + known := make(map[string]struct{}) 115 115 + m.mu.Lock() 116 116 + var gone []*RemoteLease 117 117 + for _, lease := range m.leases { 118 118 + if lease.nodeID == sess.nodeID { 119 119 + known[lease.id] = struct{}{} 120 120 + if lease.epoch != sess.epoch { 121 121 + gone = append(gone, lease) 122 122 + } else if _, ok := active[lease.id]; !ok { 123 123 + gone = append(gone, lease) 124 124 + } 125 125 + } 126 126 + } 127 127 + for _, lease := range m.reservations { 128 128 + if lease.nodeID == sess.nodeID && lease.epoch == sess.epoch { 129 129 + known[lease.id] = struct{}{} 130 130 + } 131 131 + } 132 132 + m.mu.Unlock() 133 133 + var unknown []string 134 134 + for id := range active { 135 135 + if _, ok := known[id]; !ok { 136 136 + unknown = append(unknown, id) 137 137 + } 138 138 + } 139 139 + 140 140 + for _, lease := range gone { 141 141 + status := string(models.StatusKindFailed) 142 142 + reason := "executor no longer holds lease" 143 143 + if cancelled, cancelReason := lease.cancelRequested(); cancelled { 144 144 + status = string(models.StatusKindCancelled) 145 145 + reason = cancelReason 146 146 + } 147 147 + m.l.Warn("finishing reconciled lease", "lease", lease.id, "node", sess.nodeID, "leaseInc", lease.epoch, "sessInc", sess.epoch) 148 148 + if lease.orphaned { 149 149 + if err := m.finishOrphan(lease, status, &reason, nil); err != nil { 150 150 + return err 151 151 + } 152 152 + } else if err := m.finishLiveLease(lease, status, reason); err != nil { 153 153 + return err 154 154 + } 155 155 + } 156 156 + for _, id := range unknown { 157 157 + if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{LeaseId: id, Reason: "lease is not owned by this mill"}}); err != nil { 158 158 + return fmt.Errorf("cancel unknown executor lease %q: %w", id, err) 159 159 + } 160 160 + } 161 161 + return nil 162 162 + } 163 163 + 164 164 + func (m *Mill) completeLeaseRow(lease *RemoteLease, status string, errMsg *string, exitCode *int64) error { 165 165 + if m.db == nil { 166 166 + return nil 167 167 + } 168 168 + return m.db.CompleteMillLease( 169 169 + lease.id, 170 170 + string(lease.wid.PipelineId.AtUri()), 171 171 + lease.wid.Name, 172 172 + status, 173 173 + errMsg, 174 174 + exitCode, 175 175 + m.n, 176 176 + ) 177 177 + } 178 178 + 179 179 + func (m *Mill) finishLiveLease(lease *RemoteLease, status, reason string) error { 180 180 + lease.finishMu.Lock() 181 181 + defer lease.finishMu.Unlock() 182 182 + if lease.getState() == leaseDone { 183 183 + return nil 184 184 + } 185 185 + if err := m.completeLeaseRow(lease, status, &reason, nil); err != nil { 186 186 + return err 187 187 + } 188 188 + lease.deliverTerminal(&millv1.AttemptResult{ 189 189 + Status: mapTerminalStatusString(status), 190 190 + Error: reason, 191 191 + }) 192 192 + return m.cleanupLeaseLocked(lease) 193 193 + } 194 194 + 195 195 + func (m *Mill) finishOrphan(lease *RemoteLease, status string, errMsg *string, exitCode *int64) error { 196 196 + lease.finishMu.Lock() 197 197 + defer lease.finishMu.Unlock() 198 198 + if lease.getState() == leaseDone { 199 199 + return nil 200 200 + } 201 201 + if err := m.completeLeaseRow(lease, status, errMsg, exitCode); err != nil { 202 202 + return err 203 203 + } 204 204 + lease.markDone() 205 205 + return m.cleanupLeaseLocked(lease) 206 206 + } 207 207 + 208 208 + func mapTerminalStatusString(s string) millv1.TerminalStatus { 209 209 + switch s { 210 210 + case "success": 211 211 + return millv1.TerminalStatus_SUCCESS 212 212 + case "failed": 213 213 + return millv1.TerminalStatus_FAILED 214 214 + case "timeout": 215 215 + return millv1.TerminalStatus_TIMEOUT 216 216 + case "cancelled": 217 217 + return millv1.TerminalStatus_CANCELLED 218 218 + default: 219 219 + return millv1.TerminalStatus_TERMINAL_STATUS_UNSPECIFIED 220 220 + } 221 221 + }

+386

spindle/mill/restore_test.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "path/filepath" 6 6 + "testing" 7 7 + "time" 8 8 + 9 9 + "tangled.org/core/notifier" 10 10 + "tangled.org/core/spindle/db" 11 11 + millproto "tangled.org/core/spindle/mill/proto" 12 12 + millv1 "tangled.org/core/spindle/mill/proto/gen" 13 13 + "tangled.org/core/spindle/models" 14 14 + ) 15 15 + 16 16 + func restoreTestMill(t *testing.T, cfg Config) (*Mill, *db.DB) { 17 17 + t.Helper() 18 18 + bdb, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "mill.db")) 19 19 + if err != nil { 20 20 + t.Fatalf("db.Make: %v", err) 21 21 + } 22 22 + t.Cleanup(func() { bdb.Close() }) 23 23 + n := notifier.New() 24 24 + m := New(discardLogger(), cfg) 25 25 + m.Attach(bdb, &n) 26 26 + return m, bdb 27 27 + } 28 28 + 29 29 + func restoredMill(t *testing.T, bdb *db.DB, cfg Config) *Mill { 30 30 + t.Helper() 31 31 + n := notifier.New() 32 32 + m := New(discardLogger(), cfg) 33 33 + m.Attach(bdb, &n) 34 34 + if err := m.RestoreState(); err != nil { 35 35 + t.Fatalf("RestoreState: %v", err) 36 36 + } 37 37 + return m 38 38 + } 39 39 + 40 40 + func TestRestoreStateRebuildsLeasesAndCursors(t *testing.T) { 41 41 + _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 42 42 + 43 43 + if err := bdb.SaveMillLease(db.MillLease{ 44 44 + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 45 45 + Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, 46 46 + }); err != nil { 47 47 + t.Fatalf("SaveMillLease: %v", err) 48 48 + } 49 49 + if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 7) }); err != nil { 50 50 + t.Fatalf("AdvanceCursor: %v", err) 51 51 + } 52 52 + 53 53 + m2 := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 54 54 + 55 55 + m2.mu.Lock() 56 56 + lease := m2.leases["lease-1"] 57 57 + seqno := m2.nodeSeqno["node-1/inc-1"] 58 58 + m2.mu.Unlock() 59 59 + 60 60 + if lease == nil { 61 61 + t.Fatal("restored mill has no lease-1") 62 62 + } 63 63 + if !lease.orphaned { 64 64 + t.Fatal("restored lease is not orphaned; a terminal would be delivered to a waiter that does not exist") 65 65 + } 66 66 + if lease.getState() != leaseRunning { 67 67 + t.Fatalf("restored lease state = %v, want leaseRunning", lease.getState()) 68 68 + } 69 69 + wantWid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"} 70 70 + if lease.wid != wantWid { 71 71 + t.Fatalf("restored lease wid = %+v, want %+v", lease.wid, wantWid) 72 72 + } 73 73 + if seqno != 7 { 74 74 + t.Fatalf("restored cursor = %d, want 7", seqno) 75 75 + } 76 76 + } 77 77 + 78 78 + func TestOrphanTerminalAuthorsStatusRow(t *testing.T) { 79 79 + _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 80 80 + if err := bdb.SaveMillLease(db.MillLease{ 81 81 + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 82 82 + Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, 83 83 + }); err != nil { 84 84 + t.Fatalf("SaveMillLease: %v", err) 85 85 + } 86 86 + if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 3) }); err != nil { 87 87 + t.Fatalf("AdvanceCursor: %v", err) 88 88 + } 89 89 + 90 90 + m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 91 91 + 92 92 + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 93 93 + resume, ok := m.attachSession(sess) 94 94 + if !ok { 95 95 + t.Fatal("attachSession rejected the reconnecting executor") 96 96 + } 97 97 + if resume != 3 { 98 98 + t.Fatalf("attachSession resume seqno = %d, want restored cursor 3", resume) 99 99 + } 100 100 + 101 101 + _ = m.onEventBatch(sess, &millv1.EventBatch{ 102 102 + Epoch: sess.epoch, 103 103 + Events: []*millv1.Event{ 104 104 + { 105 105 + Seqno: 4, 106 106 + LeaseId: "lease-1", 107 107 + Payload: &millv1.Event_AttemptResult{ 108 108 + AttemptResult: &millv1.AttemptResult{ 109 109 + Status: millv1.TerminalStatus_SUCCESS, 110 110 + }, 111 111 + }, 112 112 + }, 113 113 + }, 114 114 + }) 115 115 + 116 116 + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"} 117 117 + st, err := bdb.GetStatus(wid) 118 118 + if err != nil { 119 119 + t.Fatalf("GetStatus after orphan terminal: %v", err) 120 120 + } 121 121 + if st.Status != string(models.StatusKindSuccess) { 122 122 + t.Fatalf("orphan terminal authored status %q, want success", st.Status) 123 123 + } 124 124 + 125 125 + m.mu.Lock() 126 126 + _, still := m.leases["lease-1"] 127 127 + m.mu.Unlock() 128 128 + if still { 129 129 + t.Fatal("finished orphan still in the lease map") 130 130 + } 131 131 + if rows, _ := bdb.ListMillLeases(); len(rows) != 0 { 132 132 + t.Fatalf("finished orphan still persisted: %+v", rows) 133 133 + } 134 134 + } 135 135 + 136 136 + func TestSnapshotReconciliationFailsDroppedOrphans(t *testing.T) { 137 137 + _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 138 138 + for _, l := range []db.MillLease{ 139 139 + {LeaseID: "lease-kept", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", Knot: "k", Rkey: "r1", Workflow: "w", State: leaseRowRunning}, 140 140 + {LeaseID: "lease-gone", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", Knot: "k", Rkey: "r2", Workflow: "w", State: leaseRowRunning}, 141 141 + } { 142 142 + if err := bdb.SaveMillLease(l); err != nil { 143 143 + t.Fatalf("SaveMillLease(%s): %v", l.LeaseID, err) 144 144 + } 145 145 + } 146 146 + 147 147 + m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 148 148 + 149 149 + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 150 150 + m.attachSession(sess) 151 151 + 152 152 + m.onSnapshot(sess, &millv1.NodeSnapshot{ 153 153 + Seqno: 1, 154 154 + ActiveLeaseIds: []string{"lease-kept"}, 155 155 + }) 156 156 + 157 157 + m.mu.Lock() 158 158 + _, kept := m.leases["lease-kept"] 159 159 + _, gone := m.leases["lease-gone"] 160 160 + m.mu.Unlock() 161 161 + if !kept { 162 162 + t.Fatal("reconciliation dropped a lease the executor still holds") 163 163 + } 164 164 + if gone { 165 165 + t.Fatal("reconciliation kept a lease the executor no longer holds") 166 166 + } 167 167 + 168 168 + st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r2"}, Name: "w"}) 169 169 + if err != nil { 170 170 + t.Fatalf("GetStatus for dropped orphan: %v", err) 171 171 + } 172 172 + if st.Status != string(models.StatusKindFailed) { 173 173 + t.Fatalf("dropped orphan authored status %q, want failed", st.Status) 174 174 + } 175 175 + } 176 176 + func TestSnapshotReconciliationPreservesRequestedCancellation(t *testing.T) { 177 177 + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 178 178 + lease := newLease("lease-1", "node-1", "inc-old", "dummy") 179 179 + lease.wid = models.WorkflowId{ 180 180 + PipelineId: models.PipelineId{Knot: "k", Rkey: "r1"}, 181 181 + Name: "w", 182 182 + } 183 183 + lease.setState(leaseRunning) 184 184 + lease.requestCancel("workflow destroyed") 185 185 + if err := m.persistLease(lease, leaseRowRunning); err != nil { 186 186 + t.Fatalf("persistLease: %v", err) 187 187 + } 188 188 + m.mu.Lock() 189 189 + m.leases[lease.id] = lease 190 190 + m.mu.Unlock() 191 191 + 192 192 + sess := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) 193 193 + m.attachSession(sess) 194 194 + if err := m.onSnapshot(sess, &millv1.NodeSnapshot{Seqno: 1}); err != nil { 195 195 + t.Fatalf("onSnapshot: %v", err) 196 196 + } 197 197 + 198 198 + st, err := bdb.GetStatus(lease.wid) 199 199 + if err != nil { 200 200 + t.Fatalf("GetStatus: %v", err) 201 201 + } 202 202 + if st.Status != string(models.StatusKindCancelled) { 203 203 + t.Fatalf("reconciled status = %q, want cancelled", st.Status) 204 204 + } 205 205 + } 206 206 + 207 207 + func TestSnapshotReconciliationCancelsUnknownExecutorLease(t *testing.T) { 208 208 + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 209 209 + cancelled := make(chan string, 1) 210 210 + sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 211 211 + if cancel := msg.GetCancelAttempt(); cancel != nil { 212 212 + cancelled <- cancel.GetLeaseId() 213 213 + } 214 214 + return nil 215 215 + }), discardLogger()) 216 216 + m.attachSession(sess) 217 217 + 218 218 + if err := m.onSnapshot(sess, &millv1.NodeSnapshot{ 219 219 + Seqno: 1, 220 220 + ActiveLeaseIds: []string{"executor-only"}, 221 221 + }); err != nil { 222 222 + t.Fatalf("onSnapshot: %v", err) 223 223 + } 224 224 + select { 225 225 + case id := <-cancelled: 226 226 + if id != "executor-only" { 227 227 + t.Fatalf("cancelled lease = %q, want executor-only", id) 228 228 + } 229 229 + default: 230 230 + t.Fatal("snapshot reconciliation left an executor-only lease running") 231 231 + } 232 232 + } 233 233 + 234 234 + func TestSweepFailsOrphansOfAbsentExecutors(t *testing.T) { 235 235 + _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 236 236 + if err := bdb.SaveMillLease(db.MillLease{ 237 237 + LeaseID: "lease-1", NodeID: "node-absent", Epoch: "inc-absent", Engine: "dummy", 238 238 + Knot: "k", Rkey: "r1", Workflow: "w", State: leaseRowReserved, 239 239 + }); err != nil { 240 240 + t.Fatalf("SaveMillLease: %v", err) 241 241 + } 242 242 + 243 243 + m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 244 244 + m.sweepUnclaimedOrphans() 245 245 + 246 246 + m.mu.Lock() 247 247 + _, still := m.leases["lease-1"] 248 248 + m.mu.Unlock() 249 249 + if still { 250 250 + t.Fatal("sweep kept an orphan whose executor never reconnected") 251 251 + } 252 252 + st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r1"}, Name: "w"}) 253 253 + if err != nil { 254 254 + t.Fatalf("GetStatus after sweep: %v", err) 255 255 + } 256 256 + if st.Status != string(models.StatusKindFailed) { 257 257 + t.Fatalf("sweep authored status %q, want failed", st.Status) 258 258 + } 259 259 + if rows, _ := bdb.ListMillLeases(); len(rows) != 0 { 260 260 + t.Fatalf("swept orphan still persisted: %+v", rows) 261 261 + } 262 262 + } 263 263 + 264 264 + func TestAckSeqnoPersistsCursor(t *testing.T) { 265 265 + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 266 266 + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 267 267 + m.attachSession(sess) 268 268 + 269 269 + owned := newLease("lease-1", "node-1", "inc-1", "dummy") 270 270 + owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 271 271 + m.mu.Lock() 272 272 + m.leases[owned.id] = owned 273 273 + m.mu.Unlock() 274 274 + 275 275 + err := m.onEventBatch(sess, &millv1.EventBatch{ 276 276 + Epoch: sess.epoch, 277 277 + Events: []*millv1.Event{ 278 278 + { 279 279 + Seqno: 1, 280 280 + LeaseId: owned.id, 281 281 + Payload: &millv1.Event_StatusEvent{ 282 282 + StatusEvent: &millv1.StatusEvent{ 283 283 + Status: millv1.NonterminalStatus_RUNNING, 284 284 + }, 285 285 + }, 286 286 + }, 287 287 + }, 288 288 + }) 289 289 + if err != nil { 290 290 + t.Fatalf("onEventBatch: %v", err) 291 291 + } 292 292 + 293 293 + cursors, err := bdb.ListExecutorCursors() 294 294 + if err != nil { 295 295 + t.Fatalf("ListExecutorCursors: %v", err) 296 296 + } 297 297 + if len(cursors) != 1 || cursors[0].AckedSeqno != 1 { 298 298 + t.Fatalf("persisted cursor = %+v, want seqno 1", cursors) 299 299 + } 300 300 + } 301 301 + 302 302 + func TestOrphanTerminalFailureKeepsLeaseAndSeqnoRetryable(t *testing.T) { 303 303 + _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 304 304 + if err := bdb.SaveMillLease(db.MillLease{ 305 305 + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 306 306 + Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, 307 307 + }); err != nil { 308 308 + t.Fatalf("SaveMillLease: %v", err) 309 309 + } 310 310 + if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 3) }); err != nil { 311 311 + t.Fatalf("AdvanceCursor: %v", err) 312 312 + } 313 313 + if _, err := bdb.Exec(` 314 314 + create trigger reject_orphan_lease_delete 315 315 + before delete on mill_leases 316 316 + begin 317 317 + select raise(abort, 'forced delete failure'); 318 318 + end 319 319 + `); err != nil { 320 320 + t.Fatalf("create failure trigger: %v", err) 321 321 + } 322 322 + 323 323 + m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 324 324 + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 325 325 + if _, ok := m.attachSession(sess); !ok { 326 326 + t.Fatal("attachSession rejected reconnect") 327 327 + } 328 328 + batch := &millv1.EventBatch{ 329 329 + Epoch: sess.epoch, 330 330 + Events: []*millv1.Event{ 331 331 + { 332 332 + Seqno: 4, 333 333 + LeaseId: "lease-1", 334 334 + Payload: &millv1.Event_AttemptResult{ 335 335 + AttemptResult: &millv1.AttemptResult{ 336 336 + Status: millv1.TerminalStatus_SUCCESS, 337 337 + }, 338 338 + }, 339 339 + }, 340 340 + }, 341 341 + } 342 342 + if err := m.onEventBatch(sess, batch); err == nil { 343 343 + t.Fatal("orphan terminal stream succeeded despite forced transaction failure") 344 344 + } 345 345 + 346 346 + m.mu.Lock() 347 347 + lease := m.leases["lease-1"] 348 348 + seqno := m.nodeSeqno["node-1/inc-1"] 349 349 + m.mu.Unlock() 350 350 + if lease == nil || lease.getState() == leaseDone { 351 351 + t.Fatal("failed orphan completion made the in-memory lease unretryable") 352 352 + } 353 353 + if seqno != 3 { 354 354 + t.Fatalf("in-memory stream seqno = %d, want 3", seqno) 355 355 + } 356 356 + if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 1 { 357 357 + t.Fatalf("durable leases after transaction rollback = %+v, err = %v; want retained lease", rows, err) 358 358 + } 359 359 + var events int 360 360 + if err := bdb.QueryRow(`select count(*) from events`).Scan(&events); err != nil { 361 361 + t.Fatalf("count events: %v", err) 362 362 + } 363 363 + if events != 0 { 364 364 + t.Fatalf("terminal events after transaction rollback = %d, want 0", events) 365 365 + } 366 366 + 367 367 + if _, err := bdb.Exec(`drop trigger reject_orphan_lease_delete`); err != nil { 368 368 + t.Fatalf("drop failure trigger: %v", err) 369 369 + } 370 370 + if err := m.onEventBatch(sess, batch); err != nil { 371 371 + t.Fatalf("retry orphan terminal: %v", err) 372 372 + } 373 373 + m.mu.Lock() 374 374 + _, still := m.leases["lease-1"] 375 375 + seqno = m.nodeSeqno["node-1/inc-1"] 376 376 + m.mu.Unlock() 377 377 + if still { 378 378 + t.Fatal("successful orphan completion retained in-memory lease") 379 379 + } 380 380 + if seqno != 4 { 381 381 + t.Fatalf("in-memory stream seqno after retry = %d, want 4", seqno) 382 382 + } 383 383 + if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 0 { 384 384 + t.Fatalf("durable leases after successful retry = %+v, err = %v; want none", rows, err) 385 385 + } 386 386 + }

+160

spindle/mill/session.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "errors" 6 6 + "log/slog" 7 7 + "sync" 8 8 + "time" 9 9 + 10 10 + millproto "tangled.org/core/spindle/mill/proto" 11 11 + millv1 "tangled.org/core/spindle/mill/proto/gen" 12 12 + ) 13 13 + 14 14 + var errSessionClosed = errors.New("mill: executor session closed") 15 15 + 16 16 + // one live websocket to an executor. many leases and async streams share 17 17 + // it, so one reader goroutine demuxes by message type and correlates by 18 18 + // lease ID. never hold locks across decodes 19 19 + type millSession struct { 20 20 + nodeID string 21 21 + epoch string 22 22 + labels []string 23 23 + enc messageEncoder 24 24 + l *slog.Logger 25 25 + closeTransport func() error 26 26 + 27 27 + // snapshot, disconnected and graceTimer are guarded by Mill.mu, the 28 28 + // fleet ranks across sessions under its own lock 29 29 + snapshot *millv1.NodeSnapshot 30 30 + disconnected bool 31 31 + graceTimer *time.Timer 32 32 + lastSeen time.Time 33 33 + 34 34 + mu sync.Mutex 35 35 + pending map[string]chan *millproto.Message // maps lease ID to response waiter 36 36 + 37 37 + closeOnce sync.Once 38 38 + closed chan struct{} 39 39 + } 40 40 + 41 41 + type messageEncoder interface { 42 42 + Encode(*millproto.Message) error 43 43 + } 44 44 + 45 45 + func newSession(nodeID string, epoch string, labels []string, enc messageEncoder, l *slog.Logger) *millSession { 46 46 + return &millSession{ 47 47 + nodeID: nodeID, 48 48 + epoch: epoch, 49 49 + labels: labels, 50 50 + enc: enc, 51 51 + l: l, 52 52 + pending: make(map[string]chan *millproto.Message), 53 53 + closed: make(chan struct{}), 54 54 + lastSeen: time.Now(), 55 55 + } 56 56 + } 57 57 + 58 58 + // caller holds Mill.mu 59 59 + func (s *millSession) live(grace time.Duration) bool { 60 60 + return !s.disconnected && time.Since(s.lastSeen) <= grace 61 61 + } 62 62 + 63 63 + func (s *millSession) send(msg *millproto.Message) error { 64 64 + return s.enc.Encode(msg) 65 65 + } 66 66 + 67 67 + func (s *millSession) close() { 68 68 + s.closeOnce.Do(func() { 69 69 + close(s.closed) 70 70 + if s.closeTransport != nil { 71 71 + _ = s.closeTransport() 72 72 + } 73 73 + }) 74 74 + } 75 75 + 76 76 + // one-shot waiter for the next response on the lease, cancel unregisters it 77 77 + func (s *millSession) await(leaseID string) (<-chan *millproto.Message, func()) { 78 78 + ch := make(chan *millproto.Message, 1) 79 79 + s.mu.Lock() 80 80 + s.pending[leaseID] = ch 81 81 + s.mu.Unlock() 82 82 + return ch, func() { 83 83 + s.mu.Lock() 84 84 + if s.pending[leaseID] == ch { 85 85 + delete(s.pending, leaseID) 86 86 + } 87 87 + s.mu.Unlock() 88 88 + } 89 89 + } 90 90 + 91 91 + func (s *millSession) deliver(leaseID string, msg *millproto.Message) { 92 92 + s.mu.Lock() 93 93 + ch := s.pending[leaseID] 94 94 + delete(s.pending, leaseID) 95 95 + s.mu.Unlock() 96 96 + if ch != nil { 97 97 + select { 98 98 + case ch <- msg: 99 99 + default: 100 100 + } 101 101 + } 102 102 + } 103 103 + 104 104 + // sends a message and waits for its response, respecting ctx and session closure 105 105 + func (s *millSession) request(ctx context.Context, leaseID string, msg *millproto.Message) (*millproto.Message, error) { 106 106 + if err := ctx.Err(); err != nil { 107 107 + return nil, err 108 108 + } 109 109 + ch, cancel := s.await(leaseID) 110 110 + defer cancel() 111 111 + 112 112 + if err := s.send(msg); err != nil { 113 113 + return nil, err 114 114 + } 115 115 + 116 116 + select { 117 117 + case resp := <-ch: 118 118 + return resp, nil 119 119 + case <-ctx.Done(): 120 120 + return nil, ctx.Err() 121 121 + case <-s.closed: 122 122 + return nil, errSessionClosed 123 123 + } 124 124 + } 125 125 + 126 126 + // demuxes frames until the decoder errors (connection gone) 127 127 + func (s *millSession) readLoop(m *Mill, dec *millproto.Decoder) error { 128 128 + for { 129 129 + msg, err := dec.Decode() 130 130 + if err != nil { 131 131 + return err 132 132 + } 133 133 + if err := s.dispatch(m, msg); err != nil { 134 134 + return err 135 135 + } 136 136 + } 137 137 + } 138 138 + 139 139 + func (s *millSession) dispatch(m *Mill, msg *millproto.Message) error { 140 140 + if !m.touchSession(s) { 141 141 + return errSessionClosed 142 142 + } 143 143 + switch { 144 144 + case msg.GetNodeSnapshot() != nil: 145 145 + return m.onSnapshot(s, msg.GetNodeSnapshot()) 146 146 + case msg.GetReserveResult() != nil: 147 147 + s.deliver(msg.GetReserveResult().GetLeaseId(), msg) 148 148 + case msg.GetCommitted() != nil: 149 149 + s.deliver(msg.GetCommitted().GetLeaseId(), msg) 150 150 + case msg.GetEventBatch() != nil: 151 151 + return m.onEventBatch(s, msg.GetEventBatch()) 152 152 + case msg.GetCancelAck() != nil: 153 153 + m.onCancelAck(s, msg.GetCancelAck()) 154 154 + case msg.GetLiveLog() != nil: 155 155 + return m.onLiveLog(s, msg.GetLiveLog()) 156 156 + default: 157 157 + s.l.Warn("session received unexpected message", "node", s.nodeID) 158 158 + } 159 159 + return nil 160 160 + }

+23

spindle/mill/token.go

Reviewed

··· 1 1 + package mill 2 2 + 3 3 + import ( 4 4 + "crypto/rand" 5 5 + "crypto/sha256" 6 6 + "encoding/base64" 7 7 + "encoding/hex" 8 8 + ) 9 9 + 10 10 + func GenerateToken() (string, error) { 11 11 + var b [32]byte 12 12 + if _, err := rand.Read(b[:]); err != nil { 13 13 + return "", err 14 14 + } 15 15 + return base64.RawURLEncoding.EncodeToString(b[:]), nil 16 16 + } 17 17 + 18 18 + // hashes the token for persistence and comparison. a database leak never 19 19 + // exposes a usable token, and lookup avoids a secret-dependent comparison 20 20 + func HashToken(token string) string { 21 21 + sum := sha256.Sum256([]byte(token)) 22 22 + return hex.EncodeToString(sum[:]) 23 23 + }

+236 -151

spindle/server.go

Reviewed

··· 32 32 "tangled.org/core/rbac" 33 33 "tangled.org/core/repoident" 34 34 "tangled.org/core/repoverify" 35 35 + "tangled.org/core/spindle/artifactstore" 35 36 "tangled.org/core/spindle/config" 36 37 "tangled.org/core/spindle/db" 37 38 "tangled.org/core/spindle/engine" 38 39 "tangled.org/core/spindle/engines/dummy" 39 40 "tangled.org/core/spindle/engines/nixery" 40 41 "tangled.org/core/spindle/git" 42 42 + "tangled.org/core/spindle/mill" 43 43 + "tangled.org/core/spindle/mill/executor" 41 44 "tangled.org/core/spindle/models" 42 45 "tangled.org/core/spindle/secrets" 43 46 "tangled.org/core/spindle/xrpc" ··· 62 65 l *slog.Logger 63 66 n *notifier.Notifier 64 67 engs map[string]models.Engine 68 68 + jobWake chan struct{} 65 69 cfg *config.Config 66 70 ks *eventconsumer.Consumer 67 71 res *idresolver.Resolver ··· 70 74 motd []byte 71 75 motdMu sync.RWMutex 72 76 rootCtx context.Context 73 73 - jobWake chan struct{} 77 77 + store artifactstore.Store 78 78 + stores *artifactstore.Stores 79 79 + reader artifactstore.Reader 80 80 + // set only when this spindle hosts the mill or joins one as an executor 81 81 + mill *mill.Mill 82 82 + exec *executor.Executor 74 83 } 75 84 76 85 // New creates a new Spindle server with the provided configuration and engines. 77 86 func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { 78 87 logger := log.FromContext(ctx) 88 88 + n := notifier.New() 89 89 + 90 90 + if cfg.Role == config.RoleExecutor { 91 91 + if err := cleanupOrphanRepos(ctx, d, logger); err != nil { 92 92 + return nil, fmt.Errorf("failed to run startup cleanup: %w", err) 93 93 + } 94 94 + } else if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { 95 95 + return nil, fmt.Errorf("failed to run startup migrations: %w", err) 96 96 + } 97 97 + 98 98 + spindle := &Spindle{ 99 99 + db: d, 100 100 + l: logger, 101 101 + n: &n, 102 102 + engs: engines, 103 103 + cfg: cfg, 104 104 + motd: defaultMotd, 105 105 + rootCtx: ctx, 106 106 + jobWake: make(chan struct{}, 1), 107 107 + } 108 108 + diskFallback := "" 109 109 + if cfg.Role == config.RoleStandalone { 110 110 + diskFallback = cfg.Server.LogDir 111 111 + if cfg.ArtifactStores.Disk.Dir == "" { 112 112 + logger.Warn("using SPINDLE_SERVER_LOG_DIR as the implicit disk artifact store; configure SPINDLE_ARTIFACT_STORES_DISK_DIR explicitly") 113 113 + } 114 114 + } 115 115 + stores, err := artifactstore.NewStores(cfg.ArtifactStores, diskFallback, cfg.LegacyS3.LogBucket) 116 116 + if err != nil { 117 117 + return nil, fmt.Errorf("failed to setup artifact stores: %w", err) 118 118 + } 119 119 + spindle.stores = stores 120 120 + if cfg.LegacyS3.LogBucket != "" { 121 121 + logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") 122 122 + } 123 123 + if cfg.Role == config.RoleStandalone { 124 124 + spindle.reader = stores 125 125 + } else { 126 126 + name := cfg.Mill.ArtifactStore 127 127 + if name == "" { 128 128 + names := stores.Names() 129 129 + if len(names) != 1 { 130 130 + return nil, fmt.Errorf("%s requires SPINDLE_MILL_ARTIFACT_STORE when %d artifact stores are configured", cfg.Role, len(names)) 131 131 + } 132 132 + name = names[0] 133 133 + logger.Warn("SPINDLE_MILL_ARTIFACT_STORE is not set; inferred the only configured store", "store", name) 134 134 + } 135 135 + store, ok := stores.Store(name) 136 136 + if !ok { 137 137 + return nil, fmt.Errorf("SPINDLE_MILL_ARTIFACT_STORE=%q is not configured", name) 138 138 + } 139 139 + spindle.store = store 140 140 + spindle.reader = store 141 141 + } 142 142 + if cfg.Role == config.RoleExecutor { 143 143 + return spindle, nil 144 144 + } 79 145 80 146 e, err := rbac.NewEnforcer(cfg.Server.DBPath) 81 147 if err != nil { 82 148 return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) 83 149 } 84 150 e.E.EnableAutoSave(true) 151 151 + spindle.e = e 85 152 86 86 - n := notifier.New() 87 87 - 88 88 - var vault secrets.Manager 89 153 switch cfg.Server.Secrets.Provider { 90 154 case "openbao": 91 155 if cfg.Server.Secrets.OpenBao.ProxyAddr == "" { 92 156 return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider") 93 157 } 94 94 - vault, err = secrets.NewOpenBaoManager( 158 158 + spindle.vault, err = secrets.NewOpenBaoManager( 95 159 cfg.Server.Secrets.OpenBao.ProxyAddr, 96 160 logger, 97 161 secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount), ··· 101 165 } 102 166 logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount) 103 167 case "sqlite", "": 104 104 - vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) 168 168 + spindle.vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) 105 169 if err != nil { 106 170 return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err) 107 171 } ··· 110 174 return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) 111 175 } 112 176 113 113 - if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { 114 114 - return nil, fmt.Errorf("failed to run startup migrations: %w", err) 115 115 - } 116 116 - 117 177 collections := []string{ 118 178 tangled.SpindleMemberNSID, 119 179 tangled.RepoNSID, ··· 124 184 if err != nil { 125 185 return nil, fmt.Errorf("failed to setup jetstream client: %w", err) 126 186 } 187 187 + spindle.jc = jc 127 188 jc.AddDid(cfg.Server.Owner) 128 189 // pull records are created by arbitrary users too, same hack as in tap 129 190 jc.ExemptCollection(tangled.RepoPullNSID) ··· 147 208 } 148 209 } 149 210 150 150 - resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl) 151 151 - 152 152 - spindle := &Spindle{ 153 153 - jc: jc, 154 154 - e: e, 155 155 - db: d, 156 156 - l: logger, 157 157 - n: &n, 158 158 - engs: engines, 159 159 - cfg: cfg, 160 160 - res: resolver, 161 161 - verify: repoverify.New(resolver, cfg.Server.Dev), 162 162 - vault: vault, 163 163 - motd: defaultMotd, 164 164 - rootCtx: ctx, 165 165 - jobWake: make(chan struct{}, 1), 166 166 - } 211 211 + spindle.res = idresolver.DefaultResolver(cfg.Server.PlcUrl) 212 212 + spindle.verify = repoverify.New(spindle.res, cfg.Server.Dev) 167 213 168 214 err = e.AddSpindle(rbacDomain) 169 215 if err != nil { ··· 192 238 ccfg.Logger = log.SubLogger(logger, "eventconsumer") 193 239 ccfg.ProcessFunc = spindle.processKnotStream 194 240 ccfg.CursorStore = cursorStore 195 195 - ccfg.WorkerCount = 16 196 196 - ccfg.QueueSize = 200 197 241 if cfg.Server.Dev { 198 242 ccfg.RetryInterval = 5 * time.Second 199 243 ccfg.MaxRetryInterval = 10 * time.Second ··· 212 256 ccfg.Sources[src] = struct{}{} 213 257 } 214 258 spindle.ks = eventconsumer.NewConsumer(*ccfg) 215 215 - 216 259 if cfg.Server.Tap.Embed { 217 260 pw, err := randomAdminPassword() 218 261 if err != nil { ··· 225 268 226 269 return spindle, nil 227 270 } 228 228 - 229 229 - // DB returns the database instance. 230 271 func (s *Spindle) DB() *db.DB { 231 272 return s.db 232 273 } ··· 265 306 return s.motd 266 307 } 267 308 268 268 - // Start starts the Spindle server (blocking). 309 309 + // runs the server. blocks 269 310 func (s *Spindle) Start(ctx context.Context) error { 270 270 - // starts a job queue runner in the background 311 311 + // only standalone runs the local queue. mill hosts place directly onto 312 312 + // executors, and executors only run jobs explicitly assigned by a mill 271 313 s.StartJobWorkers(ctx) 272 314 273 273 - // Stop vault token renewal if it implements Stopper 315 315 + // an executor dials out to its mill and takes work from it 316 316 + if s.exec != nil { 317 317 + go s.exec.Connect(ctx) 318 318 + } 319 319 + 274 320 if stopper, ok := s.vault.(secrets.Stopper); ok { 275 321 defer stopper.Stop() 276 322 } 277 323 278 278 - tapCtx, tapCancel := context.WithCancel(ctx) 324 324 + if s.cfg.Role != config.RoleExecutor { 325 325 + tapCtx, tapCancel := context.WithCancel(ctx) 279 326 280 280 - if s.cfg.Server.Tap.Embed { 281 281 - emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) 282 282 - if err != nil { 283 283 - tapCancel() 284 284 - return fmt.Errorf("starting embedded tap: %w", err) 327 327 + if s.cfg.Server.Tap.Embed { 328 328 + emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) 329 329 + if err != nil { 330 330 + tapCancel() 331 331 + return fmt.Errorf("starting embedded tap: %w", err) 332 332 + } 333 333 + s.embedTap = emb 334 334 + defer func() { 335 335 + tapCancel() 336 336 + s.embedTap.Shutdown() 337 337 + }() 338 338 + 339 339 + go s.watchTapDrain(tapCtx, tapCancel) 340 340 + } else { 341 341 + defer tapCancel() 285 342 } 286 286 - s.embedTap = emb 287 287 - defer func() { 288 288 - tapCancel() 289 289 - s.embedTap.Shutdown() 343 343 + 344 344 + go func() { 345 345 + s.l.Info("starting knot event consumer") 346 346 + s.ks.Start(ctx) 290 347 }() 291 348 292 292 - go s.watchTapDrain(tapCtx, tapCancel) 293 293 - } else { 294 294 - defer tapCancel() 349 349 + s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) 350 350 + s.tap.Start(tapCtx) 295 351 } 296 352 297 297 - go func() { 298 298 - s.l.Info("starting knot event consumer") 299 299 - s.ks.Start(ctx) 300 300 - }() 301 301 - 302 302 - s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) 303 303 - s.tap.Start(tapCtx) 304 304 - 305 353 s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr) 306 354 return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router()) 307 355 } ··· 346 394 return fmt.Errorf("failed to setup db: %w", err) 347 395 } 348 396 349 349 - nixeryEng, err := nixery.New(ctx, cfg) 350 350 - if err != nil { 351 351 - return err 397 397 + logger := log.FromContext(ctx) 398 398 + 399 399 + var engines map[string]models.Engine 400 400 + var m *mill.Mill 401 401 + 402 402 + if cfg.Role == config.RoleMill { 403 403 + // mill host: register engines that place jobs on executors instead of 404 404 + // running them. all names share one Mill 405 405 + m = mill.New(log.SubLogger(logger, "mill"), mill.Config{ 406 406 + LogDir: cfg.Server.LogDir, 407 407 + MaxPending: cfg.Mill.MaxPending, 408 408 + ReconnectGrace: cfg.Mill.ReconnectGrace, 409 409 + }) 410 410 + engines = map[string]models.Engine{ 411 411 + "nixery": mill.NewEngine("nixery", m), 412 412 + "microvm": mill.NewEngine("microvm", m), 413 413 + "dummy": mill.NewEngine("dummy", m), 414 414 + } 415 415 + } else { 416 416 + // standalone and executor both run real engines locally. 417 417 + nixeryEng, err := nixery.New(ctx, cfg) 418 418 + if err != nil { 419 419 + return err 420 420 + } 421 421 + microvmEng, err := newMicrovmEngine(ctx, cfg, d) 422 422 + if err != nil { 423 423 + return err 424 424 + } 425 425 + engines = map[string]models.Engine{ 426 426 + "nixery": nixeryEng, 427 427 + "microvm": microvmEng, 428 428 + "dummy": dummy.New(logger), 429 429 + } 352 430 } 353 431 354 354 - microvmEng, err := newMicrovmEngine(ctx, cfg, d) 432 432 + s, err := New(ctx, cfg, d, engines) 355 433 if err != nil { 356 434 return err 357 435 } 358 436 359 359 - s, err := New(ctx, cfg, d, map[string]models.Engine{ 360 360 - "nixery": nixeryEng, 361 361 - "microvm": microvmEng, 362 362 - "dummy": dummy.New(log.FromContext(ctx)), 363 363 - }) 364 364 - if err != nil { 365 365 - return err 437 437 + if m != nil { 438 438 + // resolve the chicken-and-egg: the engines (built above) hold the mill, 439 439 + // but the mill's db/notifier are created inside New 440 440 + m.Attach(s.DB(), s.Notifier()) 441 441 + s.mill = m 442 442 + if err := m.RestoreState(); err != nil { 443 443 + return fmt.Errorf("restoring mill state: %w", err) 444 444 + } 445 445 + } 446 446 + if cfg.Role == config.RoleExecutor { 447 447 + s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store) 448 448 + if err != nil { 449 449 + return err 450 450 + } 366 451 } 367 452 368 453 return s.Start(ctx) ··· 374 459 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 375 460 w.Write(s.GetMotdContent()) 376 461 }) 462 462 + if s.cfg.Role == config.RoleExecutor { 463 463 + return mux 464 464 + } 465 465 + 377 466 mux.HandleFunc("/events", s.Events) 378 467 mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) 379 468 469 469 + // mill host: executors dial in here (plain ws, shared-secret auth) 470 470 + if s.mill != nil { 471 471 + mux.HandleFunc("/mill", s.mill.HandleExecutorConn) 472 472 + } 473 473 + 380 474 mux.Mount("/xrpc", s.XrpcRouter()) 381 475 return mux 382 476 } ··· 387 481 l := log.SubLogger(s.l, "xrpc") 388 482 389 483 x := xrpc.Xrpc{ 390 390 - Logger: l, 391 391 - Db: s.db, 392 392 - Enforcer: s.e, 393 393 - Engines: s.engs, 394 394 - Config: s.cfg, 395 395 - Resolver: s.res, 396 396 - Vault: s.vault, 397 397 - Notifier: s.Notifier(), 398 398 - ServiceAuth: serviceAuth, 399 399 - Trigger: s, 484 484 + Logger: l, 485 485 + Db: s.db, 486 486 + Enforcer: s.e, 487 487 + Engines: s.engs, 488 488 + Config: s.cfg, 489 489 + ArtifactReader: s.reader, 490 490 + Resolver: s.res, 491 491 + Vault: s.vault, 492 492 + Notifier: s.Notifier(), 493 493 + ServiceAuth: serviceAuth, 494 494 + Trigger: s, 400 495 } 401 496 402 497 return x.Router() ··· 435 530 // NOTE: we are blindly trusting the knot that it will return only repos it own 436 531 repoCloneUri := s.newRepoCloneUrl(src.Host, repoDid) 437 532 repoPath := s.newRepoPath(repoDid) 533 533 + if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, event.NewSha); err != nil { 534 534 + return fmt.Errorf("sync git repo: %w", err) 535 535 + } 536 536 + l.Info("synced git repo") 438 537 439 538 triggerRepo, err := s.buildTriggerRepo(ctx, repo) 440 539 if err != nil { ··· 747 846 return rawPipeline, nil 748 847 } 749 848 849 849 + // newRepoPath creates a path to store repository by its did and rkey. 850 850 + // The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey 851 851 + func (s *Spindle) newRepoPath(repo syntax.DID) string { 852 852 + return filepath.Join(s.cfg.Server.RepoDir, repo.String()) 853 853 + } 854 854 + 855 855 + func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { 856 856 + scheme := "https://" 857 857 + if s.cfg.Server.Dev { 858 858 + scheme = "http://" 859 859 + } 860 860 + return fmt.Sprintf("%s%s/%s", scheme, knot, did) 861 861 + } 862 862 + 863 863 + const RequiredVersion = "2.49.0" 864 864 + 865 865 + func ensureGitVersion() error { 866 866 + v, err := git.Version() 867 867 + if err != nil { 868 868 + return fmt.Errorf("fetching git version: %w", err) 869 869 + } 870 870 + if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { 871 871 + return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) 872 872 + } 873 873 + return nil 874 874 + } 875 875 + 876 876 + func (s *Spindle) configureOwner() error { 877 877 + cfgOwner := s.cfg.Server.Owner 878 878 + 879 879 + existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) 880 880 + if err != nil { 881 881 + return err 882 882 + } 883 883 + 884 884 + switch len(existing) { 885 885 + case 0: 886 886 + // no owner configured, continue 887 887 + case 1: 888 888 + // find existing owner 889 889 + existingOwner := existing[0] 890 890 + 891 891 + // no ownership change, this is okay 892 892 + if existingOwner == s.cfg.Server.Owner { 893 893 + break 894 894 + } 895 895 + 896 896 + // remove existing owner 897 897 + err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) 898 898 + if err != nil { 899 899 + return nil 900 900 + } 901 901 + default: 902 902 + return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) 903 903 + } 904 904 + 905 905 + return s.e.AddSpindleOwner(rbacDomain, cfgOwner) 906 906 + } 907 907 + 750 908 func (s *Spindle) StartJobWorkers(ctx context.Context) { 751 909 for range s.cfg.Server.MaxJobCount { 752 910 go func() { ··· 820 978 workflows[eng] = append(workflows[eng], *ewf) 821 979 } 822 980 823 823 - engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, &models.Pipeline{ 981 981 + engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.rootCtx, &models.Pipeline{ 824 982 RepoDid: syntax.DID(job.RepoDid), 825 983 Workflows: workflows, 826 984 TrustedSource: trustedSource, ··· 855 1013 } 856 1014 return nil 857 1015 } 858 858 - 859 859 - // newRepoPath creates a path to store repository by its did and rkey. 860 860 - // The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey 861 861 - func (s *Spindle) newRepoPath(repo syntax.DID) string { 862 862 - return filepath.Join(s.cfg.Server.RepoDir, repo.String()) 863 863 - } 864 864 - 865 865 - func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { 866 866 - scheme := "https://" 867 867 - if s.cfg.Server.Dev { 868 868 - scheme = "http://" 869 869 - } 870 870 - return fmt.Sprintf("%s%s/%s", scheme, knot, did) 871 871 - } 872 872 - 873 873 - const RequiredVersion = "2.49.0" 874 874 - 875 875 - func ensureGitVersion() error { 876 876 - v, err := git.Version() 877 877 - if err != nil { 878 878 - return fmt.Errorf("fetching git version: %w", err) 879 879 - } 880 880 - if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { 881 881 - return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) 882 882 - } 883 883 - return nil 884 884 - } 885 885 - 886 886 - func (s *Spindle) resolvePipelineRepoDid(repo *tangled.Pipeline_TriggerRepo) (syntax.DID, error) { 887 887 - if repo.RepoDid == nil || *repo.RepoDid == "" { 888 888 - return "", fmt.Errorf("pipeline trigger missing repoDid") 889 889 - } 890 890 - repoDid, err := syntax.ParseDID(*repo.RepoDid) 891 891 - if err != nil { 892 892 - return "", fmt.Errorf("parse repoDid %s: %w", *repo.RepoDid, err) 893 893 - } 894 894 - if _, err := s.db.GetRepoByDid(repoDid); err != nil { 895 895 - return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 896 896 - } 897 897 - return repoDid, nil 898 898 - } 899 899 - 900 900 - func (s *Spindle) configureOwner() error { 901 901 - cfgOwner := s.cfg.Server.Owner 902 902 - 903 903 - existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) 904 904 - if err != nil { 905 905 - return err 906 906 - } 907 907 - 908 908 - switch len(existing) { 909 909 - case 0: 910 910 - // no owner configured, continue 911 911 - case 1: 912 912 - // find existing owner 913 913 - existingOwner := existing[0] 914 914 - 915 915 - // no ownership change, this is okay 916 916 - if existingOwner == s.cfg.Server.Owner { 917 917 - break 918 918 - } 919 919 - 920 920 - // remove existing owner 921 921 - err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) 922 922 - if err != nil { 923 923 - return nil 924 924 - } 925 925 - default: 926 926 - return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) 927 927 - } 928 928 - 929 929 - return s.e.AddSpindleOwner(rbacDomain, cfgOwner) 930 930 - }

+930

spindle/server.go.master

Reviewed

··· 1 1 + package spindle 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "database/sql" 6 6 + _ "embed" 7 7 + "encoding/json" 8 8 + "errors" 9 9 + "fmt" 10 10 + "log/slog" 11 11 + "maps" 12 12 + "net/http" 13 13 + "path/filepath" 14 14 + "sync" 15 15 + "time" 16 16 + 17 17 + "github.com/bluesky-social/indigo/atproto/syntax" 18 18 + indigoxrpc "github.com/bluesky-social/indigo/xrpc" 19 19 + "github.com/go-chi/chi/v5" 20 20 + "github.com/go-git/go-git/v5/plumbing/object" 21 21 + "github.com/hashicorp/go-version" 22 22 + "tangled.org/core/api/tangled" 23 23 + "tangled.org/core/eventconsumer" 24 24 + "tangled.org/core/eventconsumer/cursor" 25 25 + "tangled.org/core/eventstream" 26 26 + "tangled.org/core/idresolver" 27 27 + "tangled.org/core/jetstream" 28 28 + knotdb "tangled.org/core/knotserver/db" 29 29 + kgit "tangled.org/core/knotserver/git" 30 30 + "tangled.org/core/log" 31 31 + "tangled.org/core/notifier" 32 32 + "tangled.org/core/rbac" 33 33 + "tangled.org/core/repoident" 34 34 + "tangled.org/core/repoverify" 35 35 + "tangled.org/core/spindle/config" 36 36 + "tangled.org/core/spindle/db" 37 37 + "tangled.org/core/spindle/engine" 38 38 + "tangled.org/core/spindle/engines/dummy" 39 39 + "tangled.org/core/spindle/engines/nixery" 40 40 + "tangled.org/core/spindle/git" 41 41 + "tangled.org/core/spindle/models" 42 42 + "tangled.org/core/spindle/secrets" 43 43 + "tangled.org/core/spindle/xrpc" 44 44 + "tangled.org/core/tid" 45 45 + "tangled.org/core/workflow" 46 46 + "tangled.org/core/xrpc/serviceauth" 47 47 + ) 48 48 + 49 49 + //go:embed motd 50 50 + var defaultMotd []byte 51 51 + 52 52 + const ( 53 53 + rbacDomain = "thisserver" 54 54 + ) 55 55 + 56 56 + type Spindle struct { 57 57 + jc *jetstream.JetstreamClient 58 58 + tap *Tap 59 59 + embedTap *embeddedTap 60 60 + db *db.DB 61 61 + e *rbac.Enforcer 62 62 + l *slog.Logger 63 63 + n *notifier.Notifier 64 64 + engs map[string]models.Engine 65 65 + cfg *config.Config 66 66 + ks *eventconsumer.Consumer 67 67 + res *idresolver.Resolver 68 68 + verify repoverify.Verifier 69 69 + vault secrets.Manager 70 70 + motd []byte 71 71 + motdMu sync.RWMutex 72 72 + rootCtx context.Context 73 73 + jobWake chan struct{} 74 74 + } 75 75 + 76 76 + // New creates a new Spindle server with the provided configuration and engines. 77 77 + func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { 78 78 + logger := log.FromContext(ctx) 79 79 + 80 80 + e, err := rbac.NewEnforcer(cfg.Server.DBPath) 81 81 + if err != nil { 82 82 + return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) 83 83 + } 84 84 + e.E.EnableAutoSave(true) 85 85 + 86 86 + n := notifier.New() 87 87 + 88 88 + var vault secrets.Manager 89 89 + switch cfg.Server.Secrets.Provider { 90 90 + case "openbao": 91 91 + if cfg.Server.Secrets.OpenBao.ProxyAddr == "" { 92 92 + return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider") 93 93 + } 94 94 + vault, err = secrets.NewOpenBaoManager( 95 95 + cfg.Server.Secrets.OpenBao.ProxyAddr, 96 96 + logger, 97 97 + secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount), 98 98 + ) 99 99 + if err != nil { 100 100 + return nil, fmt.Errorf("failed to setup openbao secrets provider: %w", err) 101 101 + } 102 102 + logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount) 103 103 + case "sqlite", "": 104 104 + vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) 105 105 + if err != nil { 106 106 + return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err) 107 107 + } 108 108 + logger.Info("using sqlite secrets provider", "path", cfg.Server.DBPath) 109 109 + default: 110 110 + return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) 111 111 + } 112 112 + 113 113 + if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { 114 114 + return nil, fmt.Errorf("failed to run startup migrations: %w", err) 115 115 + } 116 116 + 117 117 + collections := []string{ 118 118 + tangled.SpindleMemberNSID, 119 119 + tangled.RepoNSID, 120 120 + tangled.RepoCollaboratorNSID, 121 121 + tangled.RepoPullNSID, 122 122 + } 123 123 + jc, err := jetstream.NewJetstreamClient(cfg.Server.JetstreamEndpoint, "spindle", collections, nil, log.SubLogger(logger, "jetstream"), d, true, true) 124 124 + if err != nil { 125 125 + return nil, fmt.Errorf("failed to setup jetstream client: %w", err) 126 126 + } 127 127 + jc.AddDid(cfg.Server.Owner) 128 128 + // pull records are created by arbitrary users too, same hack as in tap 129 129 + jc.ExemptCollection(tangled.RepoPullNSID) 130 130 + 131 131 + // Check if the spindle knows about any Dids; 132 132 + dids, err := d.GetAllDids() 133 133 + if err != nil { 134 134 + return nil, fmt.Errorf("failed to get all dids: %w", err) 135 135 + } 136 136 + for _, d := range dids { 137 137 + jc.AddDid(d) 138 138 + } 139 139 + 140 140 + knownRepos, err := d.AllRepos() 141 141 + if err != nil { 142 142 + return nil, fmt.Errorf("failed to get known repos: %w", err) 143 143 + } 144 144 + for _, r := range knownRepos { 145 145 + if r.Owner != "" { 146 146 + jc.AddDid(r.Owner.String()) 147 147 + } 148 148 + } 149 149 + 150 150 + resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl) 151 151 + 152 152 + spindle := &Spindle{ 153 153 + jc: jc, 154 154 + e: e, 155 155 + db: d, 156 156 + l: logger, 157 157 + n: &n, 158 158 + engs: engines, 159 159 + cfg: cfg, 160 160 + res: resolver, 161 161 + verify: repoverify.New(resolver, cfg.Server.Dev), 162 162 + vault: vault, 163 163 + motd: defaultMotd, 164 164 + rootCtx: ctx, 165 165 + jobWake: make(chan struct{}, 1), 166 166 + } 167 167 + 168 168 + err = e.AddSpindle(rbacDomain) 169 169 + if err != nil { 170 170 + return nil, fmt.Errorf("failed to set rbac domain: %w", err) 171 171 + } 172 172 + err = spindle.configureOwner() 173 173 + if err != nil { 174 174 + return nil, err 175 175 + } 176 176 + logger.Info("owner set", "did", cfg.Server.Owner) 177 177 + 178 178 + cursorStore, err := cursor.NewSQLiteStore(cfg.Server.DBPath) 179 179 + if err != nil { 180 180 + return nil, fmt.Errorf("failed to setup sqlite3 cursor store: %w", err) 181 181 + } 182 182 + 183 183 + err = jc.StartJetstream(ctx, spindle.ingest()) 184 184 + if err != nil { 185 185 + return nil, fmt.Errorf("failed to start jetstream consumer: %w", err) 186 186 + } 187 187 + 188 188 + // spindle listen to knot stream for sh.tangled.git.refUpdate 189 189 + // which will sync the local workflow files in spindle and enqueues the 190 190 + // pipeline job for on-push workflows 191 191 + ccfg := eventconsumer.NewConsumerConfig() 192 192 + ccfg.Logger = log.SubLogger(logger, "eventconsumer") 193 193 + ccfg.ProcessFunc = spindle.processKnotStream 194 194 + ccfg.CursorStore = cursorStore 195 195 + ccfg.WorkerCount = 16 196 196 + ccfg.QueueSize = 200 197 197 + if cfg.Server.Dev { 198 198 + ccfg.RetryInterval = 5 * time.Second 199 199 + ccfg.MaxRetryInterval = 10 * time.Second 200 200 + } else { 201 201 + ccfg.RetryInterval = 1 * time.Minute 202 202 + ccfg.MaxRetryInterval = 10 * time.Minute 203 203 + } 204 204 + knownKnots, err := d.Knots() 205 205 + if err != nil { 206 206 + return nil, err 207 207 + } 208 208 + for _, knot := range knownKnots { 209 209 + logger.Info("adding source start", "knot", knot) 210 210 + src := eventconsumer.NewKnotSource(knot) 211 211 + eventconsumer.MigrateLegacyCursor(cursorStore, src) 212 212 + ccfg.Sources[src] = struct{}{} 213 213 + } 214 214 + spindle.ks = eventconsumer.NewConsumer(*ccfg) 215 215 + 216 216 + if cfg.Server.Tap.Embed { 217 217 + pw, err := randomAdminPassword() 218 218 + if err != nil { 219 219 + return nil, err 220 220 + } 221 221 + cfg.Server.Tap.AdminPassword = pw 222 222 + logger.Info("embedded tap: using random admin password") 223 223 + } 224 224 + spindle.tap = NewTapClient(spindle) 225 225 + 226 226 + return spindle, nil 227 227 + } 228 228 + 229 229 + // DB returns the database instance. 230 230 + func (s *Spindle) DB() *db.DB { 231 231 + return s.db 232 232 + } 233 233 + 234 234 + // Engines returns the map of available engines. 235 235 + func (s *Spindle) Engines() map[string]models.Engine { 236 236 + return s.engs 237 237 + } 238 238 + 239 239 + // Vault returns the secrets manager instance. 240 240 + func (s *Spindle) Vault() secrets.Manager { 241 241 + return s.vault 242 242 + } 243 243 + 244 244 + // Notifier returns the notifier instance. 245 245 + func (s *Spindle) Notifier() *notifier.Notifier { 246 246 + return s.n 247 247 + } 248 248 + 249 249 + // Enforcer returns the RBAC enforcer instance. 250 250 + func (s *Spindle) Enforcer() *rbac.Enforcer { 251 251 + return s.e 252 252 + } 253 253 + 254 254 + // SetMotdContent sets custom MOTD content, replacing the embedded default. 255 255 + func (s *Spindle) SetMotdContent(content []byte) { 256 256 + s.motdMu.Lock() 257 257 + defer s.motdMu.Unlock() 258 258 + s.motd = content 259 259 + } 260 260 + 261 261 + // GetMotdContent returns the current MOTD content. 262 262 + func (s *Spindle) GetMotdContent() []byte { 263 263 + s.motdMu.RLock() 264 264 + defer s.motdMu.RUnlock() 265 265 + return s.motd 266 266 + } 267 267 + 268 268 + // Start starts the Spindle server (blocking). 269 269 + func (s *Spindle) Start(ctx context.Context) error { 270 270 + // starts a job queue runner in the background 271 271 + s.StartJobWorkers(ctx) 272 272 + 273 273 + // Stop vault token renewal if it implements Stopper 274 274 + if stopper, ok := s.vault.(secrets.Stopper); ok { 275 275 + defer stopper.Stop() 276 276 + } 277 277 + 278 278 + tapCtx, tapCancel := context.WithCancel(ctx) 279 279 + 280 280 + if s.cfg.Server.Tap.Embed { 281 281 + emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) 282 282 + if err != nil { 283 283 + tapCancel() 284 284 + return fmt.Errorf("starting embedded tap: %w", err) 285 285 + } 286 286 + s.embedTap = emb 287 287 + defer func() { 288 288 + tapCancel() 289 289 + s.embedTap.Shutdown() 290 290 + }() 291 291 + 292 292 + go s.watchTapDrain(tapCtx, tapCancel) 293 293 + } else { 294 294 + defer tapCancel() 295 295 + } 296 296 + 297 297 + go func() { 298 298 + s.l.Info("starting knot event consumer") 299 299 + s.ks.Start(ctx) 300 300 + }() 301 301 + 302 302 + s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) 303 303 + s.tap.Start(tapCtx) 304 304 + 305 305 + s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr) 306 306 + return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router()) 307 307 + } 308 308 + 309 309 + func (s *Spindle) declareTapInterest(ctx context.Context) { 310 310 + repos, err := s.db.AllRepos() 311 311 + if err != nil { 312 312 + s.l.Warn("tap declare: failed to load known repos", "err", err) 313 313 + return 314 314 + } 315 315 + seen := make(map[syntax.DID]struct{}, len(repos)) 316 316 + dids := make([]syntax.DID, 0, len(repos)) 317 317 + for _, r := range repos { 318 318 + if r.Owner == "" { 319 319 + continue 320 320 + } 321 321 + if _, ok := seen[r.Owner]; ok { 322 322 + continue 323 323 + } 324 324 + seen[r.Owner] = struct{}{} 325 325 + dids = append(dids, r.Owner) 326 326 + } 327 327 + if err := s.tap.AddOwnerDIDs(ctx, dids); err != nil { 328 328 + s.l.Warn("tap declare: AddRepos rejected", "count", len(dids), "err", err) 329 329 + return 330 330 + } 331 331 + s.l.Info("tap declare: known owner DIDs registered", "count", len(dids)) 332 332 + } 333 333 + 334 334 + func Run(ctx context.Context) error { 335 335 + cfg, err := config.Load(ctx) 336 336 + if err != nil { 337 337 + return fmt.Errorf("failed to load config: %w", err) 338 338 + } 339 339 + 340 340 + if err := ensureGitVersion(); err != nil { 341 341 + return fmt.Errorf("ensuring git version: %w", err) 342 342 + } 343 343 + 344 344 + d, err := db.Make(ctx, cfg.Server.DBPath) 345 345 + if err != nil { 346 346 + return fmt.Errorf("failed to setup db: %w", err) 347 347 + } 348 348 + 349 349 + nixeryEng, err := nixery.New(ctx, cfg) 350 350 + if err != nil { 351 351 + return err 352 352 + } 353 353 + 354 354 + microvmEng, err := newMicrovmEngine(ctx, cfg, d) 355 355 + if err != nil { 356 356 + return err 357 357 + } 358 358 + 359 359 + s, err := New(ctx, cfg, d, map[string]models.Engine{ 360 360 + "nixery": nixeryEng, 361 361 + "microvm": microvmEng, 362 362 + "dummy": dummy.New(log.FromContext(ctx)), 363 363 + }) 364 364 + if err != nil { 365 365 + return err 366 366 + } 367 367 + 368 368 + return s.Start(ctx) 369 369 + } 370 370 + 371 371 + func (s *Spindle) Router() http.Handler { 372 372 + mux := chi.NewRouter() 373 373 + 374 374 + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 375 375 + w.Write(s.GetMotdContent()) 376 376 + }) 377 377 + mux.HandleFunc("/events", s.Events) 378 378 + mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) 379 379 + 380 380 + mux.Mount("/xrpc", s.XrpcRouter()) 381 381 + return mux 382 382 + } 383 383 + 384 384 + func (s *Spindle) XrpcRouter() http.Handler { 385 385 + serviceAuth := serviceauth.NewServiceAuth(s.l, s.res.Directory(), s.cfg.Server.Did().String()) 386 386 + 387 387 + l := log.SubLogger(s.l, "xrpc") 388 388 + 389 389 + x := xrpc.Xrpc{ 390 390 + Logger: l, 391 391 + Db: s.db, 392 392 + Enforcer: s.e, 393 393 + Engines: s.engs, 394 394 + Config: s.cfg, 395 395 + Resolver: s.res, 396 396 + Vault: s.vault, 397 397 + Notifier: s.Notifier(), 398 398 + ServiceAuth: serviceAuth, 399 399 + Trigger: s, 400 400 + } 401 401 + 402 402 + return x.Router() 403 403 + } 404 404 + 405 405 + func (s *Spindle) processKnotStream(ctx context.Context, src eventconsumer.Source, msg eventstream.Event) error { 406 406 + l := log.FromContext(ctx).With("handler", "processKnotStream") 407 407 + l = l.With("src", src.Key(), "msg.Nsid", msg.Nsid, "msg.Rkey", msg.Rkey) 408 408 + if msg.Nsid == knotdb.RepoCollaboratorUpdateNSID { 409 409 + return s.ingestKnotCollaborator(ctx, l, src, msg) 410 410 + } 411 411 + if msg.Nsid == tangled.GitRefUpdateNSID { 412 412 + event := tangled.GitRefUpdate{} 413 413 + if err := json.Unmarshal(msg.EventJson, &event); err != nil { 414 414 + l.Error("error unmarshalling", "err", err) 415 415 + return err 416 416 + } 417 417 + l = l.With("repo", event.Repo, "ref", event.Ref, "newSha", event.NewSha) 418 418 + l.Debug("debug") 419 419 + 420 420 + repoDid := syntax.DID(event.Repo) 421 421 + repo, err := s.db.GetRepoByDid(repoDid) 422 422 + if err != nil { 423 423 + return fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 424 424 + } 425 425 + 426 426 + if src.Host != repo.Knot { 427 427 + return fmt.Errorf("repo knot does not match event source: %s != %s", src.Host, repo.Knot) 428 428 + } 429 429 + 430 430 + if kgit.HasSkipCIPushOption(event.PushOptions) { 431 431 + l.Info("push event requested ci skip, skipping the event") 432 432 + return nil 433 433 + } 434 434 + 435 435 + // NOTE: we are blindly trusting the knot that it will return only repos it own 436 436 + repoCloneUri := s.newRepoCloneUrl(src.Host, repoDid) 437 437 + repoPath := s.newRepoPath(repoDid) 438 438 + 439 439 + triggerRepo, err := s.buildTriggerRepo(ctx, repo) 440 440 + if err != nil { 441 441 + return fmt.Errorf("building trigger repo: %w", err) 442 442 + } 443 443 + 444 444 + trigger := tangled.Pipeline_TriggerMetadata{ 445 445 + Kind: string(workflow.TriggerKindPush), 446 446 + Push: &tangled.Pipeline_PushTriggerData{ 447 447 + Ref: event.Ref, 448 448 + OldSha: event.OldSha, 449 449 + NewSha: event.NewSha, 450 450 + }, 451 451 + Repo: triggerRepo, 452 452 + } 453 453 + 454 454 + pipelineId, err := s.runPipeline(ctx, repoDid, trigger, event.ChangedFiles, repoCloneUri, repoPath, event.NewSha, nil, triggerRepo) 455 455 + if err != nil { 456 456 + return err 457 457 + } 458 458 + if pipelineId.Rkey == "" { 459 459 + l.Info("no workflow matched 'push' trigger, skipping the event") 460 460 + return nil 461 461 + } 462 462 + l.Info("pipeline triggered", "pipeline", pipelineId.AtUri()) 463 463 + } 464 464 + 465 465 + return nil 466 466 + } 467 467 + 468 468 + func (s *Spindle) ingestKnotCollaborator(ctx context.Context, l *slog.Logger, src eventconsumer.Source, msg eventstream.Event) error { 469 469 + var rec knotdb.RepoCollaboratorUpdate 470 470 + if err := json.Unmarshal(msg.EventJson, &rec); err != nil { 471 471 + l.Error("error unmarshalling collaboratorUpdate", "err", err) 472 472 + return err 473 473 + } 474 474 + 475 475 + subject, err := syntax.ParseDID(rec.Subject) 476 476 + if err != nil { 477 477 + l.Info("skipping collaboratorUpdate with malformed subject", "subject", rec.Subject, "err", err) 478 478 + return nil 479 479 + } 480 480 + repoDid, err := syntax.ParseDID(rec.Repo) 481 481 + if err != nil { 482 482 + l.Info("skipping collaboratorUpdate with malformed repo", "repo", rec.Repo, "err", err) 483 483 + return nil 484 484 + } 485 485 + 486 486 + repo, err := s.db.GetRepoByDid(repoDid) 487 487 + if errors.Is(err, sql.ErrNoRows) { 488 488 + l.Info("skipping collaboratorUpdate for unknown repo", "repo", repoDid) 489 489 + return nil 490 490 + } 491 491 + if err != nil { 492 492 + return fmt.Errorf("lookup repo %s: %w", repoDid, err) 493 493 + } 494 494 + if src.Host != repo.Knot { 495 495 + l.Warn("dropping collaboratorUpdate from non-owning knot", "src", src.Host, "repoKnot", repo.Knot) 496 496 + return nil 497 497 + } 498 498 + 499 499 + switch rec.Op { 500 500 + case knotdb.AclOpAdd: 501 501 + if err := s.e.AddCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { 502 502 + return fmt.Errorf("add collaborator policy: %w", err) 503 503 + } 504 504 + if err := s.db.AddKnotCollaborator(repoDid, subject); err != nil { 505 505 + return fmt.Errorf("track collaborator: %w", err) 506 506 + } 507 507 + l.Info("added knot-managed collaborator", "subject", subject, "repo", repoDid) 508 508 + case knotdb.AclOpRemove: 509 509 + if err := s.e.RemoveCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { 510 510 + return fmt.Errorf("remove collaborator policy: %w", err) 511 511 + } 512 512 + if err := s.db.DeleteRepoCollaboratorBySubjectRepo(subject, repoDid); err != nil { 513 513 + return fmt.Errorf("delete collaborator row: %w", err) 514 514 + } 515 515 + l.Info("removed knot-managed collaborator", "subject", subject, "repo", repoDid) 516 516 + default: 517 517 + return fmt.Errorf("collaboratorUpdate unknown op %q", rec.Op) 518 518 + } 519 519 + return nil 520 520 + } 521 521 + 522 522 + // buildTriggerRepo gathers trigger metadata, resolving default branch from the knot 523 523 + func (s *Spindle) buildTriggerRepo(ctx context.Context, repo *db.Repo) (*tangled.Pipeline_TriggerRepo, error) { 524 524 + rkey := string(repo.Rkey) 525 525 + repoDid := repo.RepoDid.String() 526 526 + return s.buildTriggerRepoFrom(ctx, repo.Knot, repo.Owner.String(), rkey, repoDid), nil 527 527 + } 528 528 + 529 529 + func (s *Spindle) buildTriggerRepoFrom(ctx context.Context, knot, did, rkey, repoDid string) *tangled.Pipeline_TriggerRepo { 530 530 + scheme := "https" 531 531 + if s.cfg.Server.Dev { 532 532 + scheme = "http" 533 533 + } 534 534 + client := &indigoxrpc.Client{Host: fmt.Sprintf("%s://%s", scheme, knot)} 535 535 + 536 536 + // this should maybe (?) be in the refUpdate event itself to save a roundtrip 537 537 + defaultBranch := "" 538 538 + if out, err := tangled.RepoGetDefaultBranch(ctx, client, repoDid); err == nil { 539 539 + defaultBranch = out.Name 540 540 + } 541 541 + 542 542 + var rkeyPtr *string 543 543 + if rkey != "" { 544 544 + rkeyPtr = &rkey 545 545 + } 546 546 + return &tangled.Pipeline_TriggerRepo{ 547 547 + Did: did, 548 548 + Knot: knot, 549 549 + Repo: rkeyPtr, 550 550 + RepoDid: &repoDid, 551 551 + DefaultBranch: defaultBranch, 552 552 + } 553 553 + } 554 554 + 555 555 + func (s *Spindle) resolvePipelineSourceRepo(ctx context.Context, trigger *tangled.Pipeline_TriggerMetadata) (*tangled.Pipeline_TriggerRepo, error) { 556 556 + if trigger == nil { 557 557 + return nil, nil 558 558 + } 559 559 + if trigger.SourceRepo == nil || *trigger.SourceRepo == "" { 560 560 + return trigger.Repo, nil 561 561 + } 562 562 + repoDid, err := syntax.ParseDID(*trigger.SourceRepo) 563 563 + if err != nil { 564 564 + return nil, fmt.Errorf("parse sourceRepo %s: %w", *trigger.SourceRepo, err) 565 565 + } 566 566 + return s.resolveSourceRepoInfo(ctx, repoDid) 567 567 + } 568 568 + 569 569 + // resolveSourceRepoInfo resolves trigger-repo metadata for a source repo DID. 570 570 + func (s *Spindle) resolveSourceRepoInfo(ctx context.Context, repoDid syntax.DID) (*tangled.Pipeline_TriggerRepo, error) { 571 571 + repo, err := s.db.GetRepoByDid(repoDid) 572 572 + if err == nil { 573 573 + return s.buildTriggerRepo(ctx, repo) 574 574 + } 575 575 + 576 576 + // verify repo, we don't want git sync to point to arbitrary endpoints 577 577 + res, err := s.verify(ctx, repoident.RepoDid(repoDid)) 578 578 + if err != nil { 579 579 + return nil, fmt.Errorf("verify sourceRepo %s: %w", repoDid, err) 580 580 + } 581 581 + return s.buildTriggerRepoFrom(ctx, res.KnotURL.Host, res.OwnerDid.String(), res.Rkey, repoDid.String()), nil 582 582 + } 583 583 + 584 584 + // runPipeline compiles and enqueues the pipeline for the given revision. 585 585 + // sourceRepo is the resolved repo the code was checked out from, forwarded to 586 586 + // processPipeline for env vars. 587 587 + func (s *Spindle) runPipeline(ctx context.Context, repoDid syntax.DID, trigger tangled.Pipeline_TriggerMetadata, changedFiles []string, repoCloneUri, repoPath, rev string, only []string, sourceRepo *tangled.Pipeline_TriggerRepo) (models.PipelineId, error) { 588 588 + l := log.FromContext(ctx) 589 589 + 590 590 + compiler := workflow.Compiler{ 591 591 + ChangedFiles: changedFiles, 592 592 + Trigger: trigger, 593 593 + } 594 594 + 595 595 + rawPipeline, err := s.loadPipeline(ctx, repoCloneUri, repoPath, rev) 596 596 + if err != nil { 597 597 + return models.PipelineId{}, fmt.Errorf("loading pipeline: %w", err) 598 598 + } 599 599 + if len(rawPipeline) == 0 { 600 600 + return models.PipelineId{}, nil 601 601 + } 602 602 + 603 603 + tpl := compiler.Compile(compiler.Parse(rawPipeline)) 604 604 + // todo(dawn): pass compile error to workflow log 605 605 + for _, w := range compiler.Diagnostics.Errors { 606 606 + l.Error(w.String()) 607 607 + } 608 608 + for _, w := range compiler.Diagnostics.Warnings { 609 609 + l.Warn(w.String()) 610 610 + } 611 611 + 612 612 + if len(only) > 0 { 613 613 + tpl.Workflows = filterWorkflows(tpl.Workflows, only) 614 614 + } 615 615 + if len(tpl.Workflows) == 0 { 616 616 + return models.PipelineId{}, nil 617 617 + } 618 618 + 619 619 + pipelineId := models.PipelineId{ 620 620 + Knot: trigger.Repo.Knot, 621 621 + Rkey: tid.TID(), 622 622 + } 623 623 + if err := s.db.CreatePipelineEvent(pipelineId.Rkey, tpl, s.n); err != nil { 624 624 + return models.PipelineId{}, fmt.Errorf("creating pipeline event: %w", err) 625 625 + } 626 626 + err = s.processPipeline(repoDid, tpl, pipelineId, sourceRepo) 627 627 + return pipelineId, err 628 628 + } 629 629 + 630 630 + // filterWorkflows filters workflows to the requested names 631 631 + func filterWorkflows(workflows []*tangled.Pipeline_Workflow, only []string) []*tangled.Pipeline_Workflow { 632 632 + allowed := make(map[string]struct{}, len(only)) 633 633 + for _, n := range only { 634 634 + allowed[n] = struct{}{} 635 635 + } 636 636 + var filtered []*tangled.Pipeline_Workflow 637 637 + for _, w := range workflows { 638 638 + if w == nil { 639 639 + continue 640 640 + } 641 641 + if _, ok := allowed[w.Name]; ok { 642 642 + filtered = append(filtered, w) 643 643 + } 644 644 + } 645 645 + return filtered 646 646 + } 647 647 + 648 648 + // TriggerManual dispatches a pipeline at sha, authorized against and recorded 649 649 + // under repoDid. sourceRepo, pull, and inputs are optional trigger payload. 650 650 + func (s *Spindle) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull xrpc.PullContext, inputs []*tangled.Pipeline_Pair) (syntax.ATURI, error) { 651 651 + repo, err := s.db.GetRepoByDid(repoDid) 652 652 + if err != nil { 653 653 + return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 654 654 + } 655 655 + 656 656 + triggerRepo, err := s.buildTriggerRepo(ctx, repo) 657 657 + if err != nil { 658 658 + return "", fmt.Errorf("building trigger repo: %w", err) 659 659 + } 660 660 + 661 661 + trigger := tangled.Pipeline_TriggerMetadata{Repo: triggerRepo} 662 662 + if pull.IsPullRequest { 663 663 + var pullAt *string 664 664 + if pull.Pull != "" { 665 665 + pullAtStr := pull.Pull.String() 666 666 + pullAt = &pullAtStr 667 667 + } 668 668 + trigger.Kind = string(workflow.TriggerKindPullRequest) 669 669 + trigger.PullRequest = &tangled.Pipeline_PullRequestTriggerData{ 670 670 + SourceBranch: pull.SourceBranch, 671 671 + TargetBranch: pull.TargetBranch, 672 672 + SourceSha: sha, 673 673 + Pull: pullAt, 674 674 + } 675 675 + } else { 676 676 + var refPtr *string 677 677 + if ref != "" { 678 678 + refPtr = &ref 679 679 + } 680 680 + trigger.Kind = string(workflow.TriggerKindManual) 681 681 + trigger.Manual = &tangled.Pipeline_ManualTriggerData{ 682 682 + Sha: sha, 683 683 + Ref: refPtr, 684 684 + Inputs: inputs, 685 685 + } 686 686 + } 687 687 + 688 688 + repoCloneUri := s.newRepoCloneUrl(repo.Knot, repoDid) 689 689 + repoPath := s.newRepoPath(repoDid) 690 690 + sourceInfo := triggerRepo // default: code comes from the repo itself 691 691 + if sourceRepo != "" && sourceRepo != repoDid { 692 692 + sourceInfo, err = s.resolveSourceRepoInfo(ctx, sourceRepo) 693 693 + if err != nil { 694 694 + return "", err 695 695 + } 696 696 + sourceRepoStr := sourceRepo.String() 697 697 + trigger.SourceRepo = &sourceRepoStr 698 698 + repoCloneUri = models.BuildRepoURL(sourceInfo) 699 699 + repoPath = s.newRepoPath(sourceRepo) 700 700 + } 701 701 + 702 702 + pipelineId, err := s.runPipeline(ctx, repoDid, trigger, nil, repoCloneUri, repoPath, sha, workflows, sourceInfo) 703 703 + if err != nil { 704 704 + return "", err 705 705 + } 706 706 + if pipelineId.Rkey == "" { 707 707 + return "", xrpc.ErrNoMatchingWorkflows 708 708 + } 709 709 + return pipelineId.AtUri(), nil 710 710 + } 711 711 + 712 712 + func (s *Spindle) loadPipeline(ctx context.Context, repoUri, repoPath, rev string) (workflow.RawPipeline, error) { 713 713 + if err := git.SparseSyncGitRepo(ctx, repoUri, repoPath, rev); err != nil { 714 714 + return nil, fmt.Errorf("syncing git repo: %w", err) 715 715 + } 716 716 + gr, err := kgit.Open(repoPath, rev) 717 717 + if err != nil { 718 718 + return nil, fmt.Errorf("opening git repo: %w", err) 719 719 + } 720 720 + 721 721 + workflowDir, err := gr.FileTree(ctx, workflow.WorkflowDir) 722 722 + if errors.Is(err, object.ErrDirectoryNotFound) { 723 723 + // return empty RawPipeline when directory doesn't exist 724 724 + return nil, nil 725 725 + } else if err != nil { 726 726 + return nil, fmt.Errorf("loading file tree: %w", err) 727 727 + } 728 728 + 729 729 + var rawPipeline workflow.RawPipeline 730 730 + for _, e := range workflowDir { 731 731 + if !e.IsFile() { 732 732 + continue 733 733 + } 734 734 + 735 735 + fpath := filepath.Join(workflow.WorkflowDir, e.Name) 736 736 + contents, err := gr.RawContent(fpath) 737 737 + if err != nil { 738 738 + return nil, fmt.Errorf("reading raw content of '%s': %w", fpath, err) 739 739 + } 740 740 + 741 741 + rawPipeline = append(rawPipeline, workflow.RawWorkflow{ 742 742 + Name: e.Name, 743 743 + Contents: contents, 744 744 + }) 745 745 + } 746 746 + 747 747 + return rawPipeline, nil 748 748 + } 749 749 + 750 750 + func (s *Spindle) StartJobWorkers(ctx context.Context) { 751 751 + for range s.cfg.Server.MaxJobCount { 752 752 + go func() { 753 753 + for { 754 754 + job, err := s.db.DequeueJob(ctx) 755 755 + if err != nil { 756 756 + s.l.Error("failed to dequeue job", "error", err) 757 757 + } 758 758 + if job == nil { 759 759 + // sleep until a new job wakes us 760 760 + select { 761 761 + case <-ctx.Done(): 762 762 + return 763 763 + case <-s.jobWake: 764 764 + } 765 765 + continue 766 766 + } 767 767 + s.runJob(ctx, job) 768 768 + } 769 769 + }() 770 770 + } 771 771 + } 772 772 + 773 773 + func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { 774 774 + pipelineId := models.PipelineId{ 775 775 + Knot: job.PipelineIdKnot, 776 776 + Rkey: job.PipelineIdRkey, 777 777 + } 778 778 + 779 779 + pipelineEnv := models.PipelineEnvVarsForSource(job.Tpl.TriggerMetadata, pipelineId, job.SourceRepo) 780 780 + trustedSource := true 781 781 + if tm := job.Tpl.TriggerMetadata; tm != nil && tm.SourceRepo != nil && 782 782 + *tm.SourceRepo != "" && *tm.SourceRepo != job.RepoDid { 783 783 + trustedSource = false 784 784 + } 785 785 + 786 786 + initTpl := job.Tpl 787 787 + if job.SourceRepo != nil && job.Tpl.TriggerMetadata != nil { 788 788 + tm := *job.Tpl.TriggerMetadata 789 789 + tm.Repo = job.SourceRepo 790 790 + initTpl.TriggerMetadata = &tm 791 791 + } 792 792 + 793 793 + workflows := make(map[models.Engine][]models.Workflow) 794 794 + for _, w := range job.Tpl.Workflows { 795 795 + if w == nil { 796 796 + continue 797 797 + } 798 798 + eng, ok := s.engs[w.Engine] 799 799 + if !ok { 800 800 + _ = s.db.StatusFailed(models.WorkflowId{ 801 801 + PipelineId: pipelineId, 802 802 + Name: w.Name, 803 803 + }, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n) 804 804 + continue 805 805 + } 806 806 + 807 807 + ewf, err := eng.InitWorkflow(*w, initTpl) 808 808 + if err != nil { 809 809 + _ = s.db.StatusFailed(models.WorkflowId{ 810 810 + PipelineId: pipelineId, 811 811 + Name: w.Name, 812 812 + }, fmt.Sprintf("init workflow: %s", err), -1, s.n) 813 813 + continue 814 814 + } 815 815 + 816 816 + if ewf.Environment == nil { 817 817 + ewf.Environment = make(map[string]string) 818 818 + } 819 819 + maps.Copy(ewf.Environment, pipelineEnv) 820 820 + workflows[eng] = append(workflows[eng], *ewf) 821 821 + } 822 822 + 823 823 + engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, &models.Pipeline{ 824 824 + RepoDid: syntax.DID(job.RepoDid), 825 825 + Workflows: workflows, 826 826 + TrustedSource: trustedSource, 827 827 + }, pipelineId) 828 828 + } 829 829 + 830 830 + // enqueues the workflows in tpl. 831 831 + func (s *Spindle) processPipeline(repoDid syntax.DID, tpl tangled.Pipeline, pipelineId models.PipelineId, sourceRepo *tangled.Pipeline_TriggerRepo) error { 832 832 + err := s.db.EnqueueJob(s.rootCtx, repoDid.String(), pipelineId, sourceRepo, tpl) 833 833 + if err != nil { 834 834 + return fmt.Errorf("failed to enqueue durable job: %w", err) 835 835 + } 836 836 + s.l.Info("pipeline enqueued successfully to db", "id", pipelineId) 837 837 + 838 838 + // wake up an idle worker to pick up more jobs if any 839 839 + select { 840 840 + case s.jobWake <- struct{}{}: 841 841 + default: 842 842 + } 843 843 + 844 844 + // pipelines visible from now on, they are sitting in queue 845 845 + for _, w := range tpl.Workflows { 846 846 + if w == nil { 847 847 + continue 848 848 + } 849 849 + if err := s.db.StatusPending(models.WorkflowId{ 850 850 + PipelineId: pipelineId, 851 851 + Name: w.Name, 852 852 + }, s.n); err != nil { 853 853 + return fmt.Errorf("db.StatusPending: %w", err) 854 854 + } 855 855 + } 856 856 + return nil 857 857 + } 858 858 + 859 859 + // newRepoPath creates a path to store repository by its did and rkey. 860 860 + // The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey 861 861 + func (s *Spindle) newRepoPath(repo syntax.DID) string { 862 862 + return filepath.Join(s.cfg.Server.RepoDir, repo.String()) 863 863 + } 864 864 + 865 865 + func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { 866 866 + scheme := "https://" 867 867 + if s.cfg.Server.Dev { 868 868 + scheme = "http://" 869 869 + } 870 870 + return fmt.Sprintf("%s%s/%s", scheme, knot, did) 871 871 + } 872 872 + 873 873 + const RequiredVersion = "2.49.0" 874 874 + 875 875 + func ensureGitVersion() error { 876 876 + v, err := git.Version() 877 877 + if err != nil { 878 878 + return fmt.Errorf("fetching git version: %w", err) 879 879 + } 880 880 + if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { 881 881 + return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) 882 882 + } 883 883 + return nil 884 884 + } 885 885 + 886 886 + func (s *Spindle) resolvePipelineRepoDid(repo *tangled.Pipeline_TriggerRepo) (syntax.DID, error) { 887 887 + if repo.RepoDid == nil || *repo.RepoDid == "" { 888 888 + return "", fmt.Errorf("pipeline trigger missing repoDid") 889 889 + } 890 890 + repoDid, err := syntax.ParseDID(*repo.RepoDid) 891 891 + if err != nil { 892 892 + return "", fmt.Errorf("parse repoDid %s: %w", *repo.RepoDid, err) 893 893 + } 894 894 + if _, err := s.db.GetRepoByDid(repoDid); err != nil { 895 895 + return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 896 896 + } 897 897 + return repoDid, nil 898 898 + } 899 899 + 900 900 + func (s *Spindle) configureOwner() error { 901 901 + cfgOwner := s.cfg.Server.Owner 902 902 + 903 903 + existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) 904 904 + if err != nil { 905 905 + return err 906 906 + } 907 907 + 908 908 + switch len(existing) { 909 909 + case 0: 910 910 + // no owner configured, continue 911 911 + case 1: 912 912 + // find existing owner 913 913 + existingOwner := existing[0] 914 914 + 915 915 + // no ownership change, this is okay 916 916 + if existingOwner == s.cfg.Server.Owner { 917 917 + break 918 918 + } 919 919 + 920 920 + // remove existing owner 921 921 + err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) 922 922 + if err != nil { 923 923 + return nil 924 924 + } 925 925 + default: 926 926 + return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) 927 927 + } 928 928 + 929 929 + return s.e.AddSpindleOwner(rbacDomain, cfgOwner) 930 930 + }

+1028

spindle/server.go.mill

Reviewed

··· 1 1 + package spindle 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "database/sql" 6 6 + _ "embed" 7 7 + "encoding/json" 8 8 + "errors" 9 9 + "fmt" 10 10 + "log/slog" 11 11 + "maps" 12 12 + "net/http" 13 13 + "path/filepath" 14 14 + "sync" 15 15 + "time" 16 16 + 17 17 + "github.com/bluesky-social/indigo/atproto/syntax" 18 18 + indigoxrpc "github.com/bluesky-social/indigo/xrpc" 19 19 + "github.com/go-chi/chi/v5" 20 20 + "github.com/go-git/go-git/v5/plumbing/object" 21 21 + "github.com/hashicorp/go-version" 22 22 + "tangled.org/core/api/tangled" 23 23 + "tangled.org/core/eventconsumer" 24 24 + "tangled.org/core/eventconsumer/cursor" 25 25 + "tangled.org/core/eventstream" 26 26 + "tangled.org/core/idresolver" 27 27 + "tangled.org/core/jetstream" 28 28 + knotdb "tangled.org/core/knotserver/db" 29 29 + kgit "tangled.org/core/knotserver/git" 30 30 + "tangled.org/core/log" 31 31 + "tangled.org/core/notifier" 32 32 + "tangled.org/core/rbac" 33 33 + "tangled.org/core/repoident" 34 34 + "tangled.org/core/repoverify" 35 35 + "tangled.org/core/spindle/artifactstore" 36 36 + "tangled.org/core/spindle/config" 37 37 + "tangled.org/core/spindle/db" 38 38 + "tangled.org/core/spindle/engine" 39 39 + "tangled.org/core/spindle/engines/dummy" 40 40 + "tangled.org/core/spindle/engines/nixery" 41 41 + "tangled.org/core/spindle/git" 42 42 + "tangled.org/core/spindle/mill" 43 43 + "tangled.org/core/spindle/mill/executor" 44 44 + "tangled.org/core/spindle/models" 45 45 + "tangled.org/core/spindle/queue" 46 46 + "tangled.org/core/spindle/secrets" 47 47 + "tangled.org/core/spindle/xrpc" 48 48 + "tangled.org/core/tid" 49 49 + "tangled.org/core/workflow" 50 50 + "tangled.org/core/xrpc/serviceauth" 51 51 + ) 52 52 + 53 53 + //go:embed motd 54 54 + var defaultMotd []byte 55 55 + 56 56 + const ( 57 57 + rbacDomain = "thisserver" 58 58 + ) 59 59 + 60 60 + type Spindle struct { 61 61 + jc *jetstream.JetstreamClient 62 62 + tap *Tap 63 63 + embedTap *embeddedTap 64 64 + db *db.DB 65 65 + e *rbac.Enforcer 66 66 + l *slog.Logger 67 67 + n *notifier.Notifier 68 68 + engs map[string]models.Engine 69 69 + jq *queue.Queue 70 70 + cfg *config.Config 71 71 + ks *eventconsumer.Consumer 72 72 + res *idresolver.Resolver 73 73 + verify repoverify.Verifier 74 74 + vault secrets.Manager 75 75 + motd []byte 76 76 + motdMu sync.RWMutex 77 77 + rootCtx context.Context 78 78 + store artifactstore.Store 79 79 + stores *artifactstore.Stores 80 80 + reader artifactstore.Reader 81 81 + // set only when this spindle hosts the mill or joins one as an executor 82 82 + mill *mill.Mill 83 83 + exec *executor.Executor 84 84 + } 85 85 + 86 86 + // New creates a new Spindle server with the provided configuration and engines. 87 87 + func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { 88 88 + logger := log.FromContext(ctx) 89 89 + n := notifier.New() 90 90 + 91 91 + if cfg.Role == config.RoleExecutor { 92 92 + if err := cleanupOrphanRepos(ctx, d, logger); err != nil { 93 93 + return nil, fmt.Errorf("failed to run startup cleanup: %w", err) 94 94 + } 95 95 + } else if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { 96 96 + return nil, fmt.Errorf("failed to run startup migrations: %w", err) 97 97 + } 98 98 + 99 99 + spindle := &Spindle{ 100 100 + db: d, 101 101 + l: logger, 102 102 + n: &n, 103 103 + engs: engines, 104 104 + cfg: cfg, 105 105 + motd: defaultMotd, 106 106 + rootCtx: ctx, 107 107 + } 108 108 + diskFallback := "" 109 109 + if cfg.Role == config.RoleStandalone { 110 110 + diskFallback = cfg.Server.LogDir 111 111 + if cfg.ArtifactStores.Disk.Dir == "" { 112 112 + logger.Warn("using SPINDLE_SERVER_LOG_DIR as the implicit disk artifact store; configure SPINDLE_ARTIFACT_STORES_DISK_DIR explicitly") 113 113 + } 114 114 + } 115 115 + stores, err := artifactstore.NewStores(cfg.ArtifactStores, diskFallback, cfg.LegacyS3.LogBucket) 116 116 + if err != nil { 117 117 + return nil, fmt.Errorf("failed to setup artifact stores: %w", err) 118 118 + } 119 119 + spindle.stores = stores 120 120 + if cfg.LegacyS3.LogBucket != "" { 121 121 + logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") 122 122 + } 123 123 + if cfg.Role == config.RoleStandalone { 124 124 + spindle.reader = stores 125 125 + } else { 126 126 + name := cfg.Mill.ArtifactStore 127 127 + if name == "" { 128 128 + names := stores.Names() 129 129 + if len(names) != 1 { 130 130 + return nil, fmt.Errorf("%s requires SPINDLE_MILL_ARTIFACT_STORE when %d artifact stores are configured", cfg.Role, len(names)) 131 131 + } 132 132 + name = names[0] 133 133 + logger.Warn("SPINDLE_MILL_ARTIFACT_STORE is not set; inferred the only configured store", "store", name) 134 134 + } 135 135 + store, ok := stores.Store(name) 136 136 + if !ok { 137 137 + return nil, fmt.Errorf("SPINDLE_MILL_ARTIFACT_STORE=%q is not configured", name) 138 138 + } 139 139 + spindle.store = store 140 140 + spindle.reader = store 141 141 + } 142 142 + if cfg.Role == config.RoleExecutor { 143 143 + return spindle, nil 144 144 + } 145 145 + 146 146 + e, err := rbac.NewEnforcer(cfg.Server.DBPath) 147 147 + if err != nil { 148 148 + return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) 149 149 + } 150 150 + e.E.EnableAutoSave(true) 151 151 + spindle.e = e 152 152 + 153 153 + switch cfg.Server.Secrets.Provider { 154 154 + case "openbao": 155 155 + if cfg.Server.Secrets.OpenBao.ProxyAddr == "" { 156 156 + return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider") 157 157 + } 158 158 + spindle.vault, err = secrets.NewOpenBaoManager( 159 159 + cfg.Server.Secrets.OpenBao.ProxyAddr, 160 160 + logger, 161 161 + secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount), 162 162 + ) 163 163 + if err != nil { 164 164 + return nil, fmt.Errorf("failed to setup openbao secrets provider: %w", err) 165 165 + } 166 166 + logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount) 167 167 + case "sqlite", "": 168 168 + spindle.vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) 169 169 + if err != nil { 170 170 + return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err) 171 171 + } 172 172 + logger.Info("using sqlite secrets provider", "path", cfg.Server.DBPath) 173 173 + default: 174 174 + return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) 175 175 + } 176 176 + 177 177 + if cfg.Role == config.RoleStandalone { 178 178 + spindle.jq = queue.NewQueue(cfg.Server.QueueSize, cfg.Server.MaxJobCount) 179 179 + logger.Info("initialized queue", "queueSize", cfg.Server.QueueSize, "numWorkers", cfg.Server.MaxJobCount) 180 180 + } 181 181 + 182 182 + collections := []string{ 183 183 + tangled.SpindleMemberNSID, 184 184 + tangled.RepoNSID, 185 185 + tangled.RepoCollaboratorNSID, 186 186 + tangled.RepoPullNSID, 187 187 + } 188 188 + jc, err := jetstream.NewJetstreamClient(cfg.Server.JetstreamEndpoint, "spindle", collections, nil, log.SubLogger(logger, "jetstream"), d, true, true) 189 189 + if err != nil { 190 190 + return nil, fmt.Errorf("failed to setup jetstream client: %w", err) 191 191 + } 192 192 + spindle.jc = jc 193 193 + jc.AddDid(cfg.Server.Owner) 194 194 + // pull records are created by arbitrary users too, same hack as in tap 195 195 + jc.ExemptCollection(tangled.RepoPullNSID) 196 196 + 197 197 + // Check if the spindle knows about any Dids; 198 198 + dids, err := d.GetAllDids() 199 199 + if err != nil { 200 200 + return nil, fmt.Errorf("failed to get all dids: %w", err) 201 201 + } 202 202 + for _, d := range dids { 203 203 + jc.AddDid(d) 204 204 + } 205 205 + 206 206 + knownRepos, err := d.AllRepos() 207 207 + if err != nil { 208 208 + return nil, fmt.Errorf("failed to get known repos: %w", err) 209 209 + } 210 210 + for _, r := range knownRepos { 211 211 + if r.Owner != "" { 212 212 + jc.AddDid(r.Owner.String()) 213 213 + } 214 214 + } 215 215 + 216 216 + spindle.res = idresolver.DefaultResolver(cfg.Server.PlcUrl) 217 217 + spindle.verify = repoverify.New(spindle.res, cfg.Server.Dev) 218 218 + 219 219 + err = e.AddSpindle(rbacDomain) 220 220 + if err != nil { 221 221 + return nil, fmt.Errorf("failed to set rbac domain: %w", err) 222 222 + } 223 223 + err = spindle.configureOwner() 224 224 + if err != nil { 225 225 + return nil, err 226 226 + } 227 227 + logger.Info("owner set", "did", cfg.Server.Owner) 228 228 + 229 229 + cursorStore, err := cursor.NewSQLiteStore(cfg.Server.DBPath) 230 230 + if err != nil { 231 231 + return nil, fmt.Errorf("failed to setup sqlite3 cursor store: %w", err) 232 232 + } 233 233 + 234 234 + err = jc.StartJetstream(ctx, spindle.ingest()) 235 235 + if err != nil { 236 236 + return nil, fmt.Errorf("failed to start jetstream consumer: %w", err) 237 237 + } 238 238 + 239 239 + // spindle listen to knot stream for sh.tangled.git.refUpdate 240 240 + // which will sync the local workflow files in spindle and enqueues the 241 241 + // pipeline job for on-push workflows 242 242 + ccfg := eventconsumer.NewConsumerConfig() 243 243 + ccfg.Logger = log.SubLogger(logger, "eventconsumer") 244 244 + ccfg.ProcessFunc = spindle.processKnotStream 245 245 + ccfg.CursorStore = cursorStore 246 246 + if cfg.Server.Dev { 247 247 + ccfg.RetryInterval = 5 * time.Second 248 248 + ccfg.MaxRetryInterval = 10 * time.Second 249 249 + } else { 250 250 + ccfg.RetryInterval = 1 * time.Minute 251 251 + ccfg.MaxRetryInterval = 10 * time.Minute 252 252 + } 253 253 + knownKnots, err := d.Knots() 254 254 + if err != nil { 255 255 + return nil, err 256 256 + } 257 257 + for _, knot := range knownKnots { 258 258 + logger.Info("adding source start", "knot", knot) 259 259 + src := eventconsumer.NewKnotSource(knot) 260 260 + eventconsumer.MigrateLegacyCursor(cursorStore, src) 261 261 + ccfg.Sources[src] = struct{}{} 262 262 + } 263 263 + spindle.ks = eventconsumer.NewConsumer(*ccfg) 264 264 + if cfg.Server.Tap.Embed { 265 265 + pw, err := randomAdminPassword() 266 266 + if err != nil { 267 267 + return nil, err 268 268 + } 269 269 + cfg.Server.Tap.AdminPassword = pw 270 270 + logger.Info("embedded tap: using random admin password") 271 271 + } 272 272 + spindle.tap = NewTapClient(spindle) 273 273 + 274 274 + return spindle, nil 275 275 + } 276 276 + func (s *Spindle) DB() *db.DB { 277 277 + return s.db 278 278 + } 279 279 + func (s *Spindle) Queue() *queue.Queue { 280 280 + return s.jq 281 281 + } 282 282 + 283 283 + // Engines returns the map of available engines. 284 284 + func (s *Spindle) Engines() map[string]models.Engine { 285 285 + return s.engs 286 286 + } 287 287 + 288 288 + // Vault returns the secrets manager instance. 289 289 + func (s *Spindle) Vault() secrets.Manager { 290 290 + return s.vault 291 291 + } 292 292 + 293 293 + // Notifier returns the notifier instance. 294 294 + func (s *Spindle) Notifier() *notifier.Notifier { 295 295 + return s.n 296 296 + } 297 297 + 298 298 + // Enforcer returns the RBAC enforcer instance. 299 299 + func (s *Spindle) Enforcer() *rbac.Enforcer { 300 300 + return s.e 301 301 + } 302 302 + 303 303 + // SetMotdContent sets custom MOTD content, replacing the embedded default. 304 304 + func (s *Spindle) SetMotdContent(content []byte) { 305 305 + s.motdMu.Lock() 306 306 + defer s.motdMu.Unlock() 307 307 + s.motd = content 308 308 + } 309 309 + 310 310 + // GetMotdContent returns the current MOTD content. 311 311 + func (s *Spindle) GetMotdContent() []byte { 312 312 + s.motdMu.RLock() 313 313 + defer s.motdMu.RUnlock() 314 314 + return s.motd 315 315 + } 316 316 + 317 317 + // runs the server. blocks 318 318 + func (s *Spindle) Start(ctx context.Context) error { 319 319 + // only standalone runs the local queue. mill hosts place directly onto 320 320 + // executors, and executors only run jobs explicitly assigned by a mill 321 321 + if s.cfg.Role == config.RoleStandalone { 322 322 + if s.jq != nil { 323 323 + s.jq.Start() 324 324 + defer s.jq.Stop() 325 325 + } 326 326 + } 327 327 + 328 328 + // an executor dials out to its mill and takes work from it 329 329 + if s.exec != nil { 330 330 + go s.exec.Connect(ctx) 331 331 + } 332 332 + 333 333 + if stopper, ok := s.vault.(secrets.Stopper); ok { 334 334 + defer stopper.Stop() 335 335 + } 336 336 + 337 337 + if s.cfg.Role != config.RoleExecutor { 338 338 + tapCtx, tapCancel := context.WithCancel(ctx) 339 339 + 340 340 + if s.cfg.Server.Tap.Embed { 341 341 + emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) 342 342 + if err != nil { 343 343 + tapCancel() 344 344 + return fmt.Errorf("starting embedded tap: %w", err) 345 345 + } 346 346 + s.embedTap = emb 347 347 + defer func() { 348 348 + tapCancel() 349 349 + s.embedTap.Shutdown() 350 350 + }() 351 351 + 352 352 + go s.watchTapDrain(tapCtx, tapCancel) 353 353 + } else { 354 354 + defer tapCancel() 355 355 + } 356 356 + 357 357 + go func() { 358 358 + s.l.Info("starting knot event consumer") 359 359 + s.ks.Start(ctx) 360 360 + }() 361 361 + 362 362 + s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) 363 363 + s.tap.Start(tapCtx) 364 364 + } 365 365 + 366 366 + s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr) 367 367 + return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router()) 368 368 + } 369 369 + 370 370 + func (s *Spindle) declareTapInterest(ctx context.Context) { 371 371 + repos, err := s.db.AllRepos() 372 372 + if err != nil { 373 373 + s.l.Warn("tap declare: failed to load known repos", "err", err) 374 374 + return 375 375 + } 376 376 + seen := make(map[syntax.DID]struct{}, len(repos)) 377 377 + dids := make([]syntax.DID, 0, len(repos)) 378 378 + for _, r := range repos { 379 379 + if r.Owner == "" { 380 380 + continue 381 381 + } 382 382 + if _, ok := seen[r.Owner]; ok { 383 383 + continue 384 384 + } 385 385 + seen[r.Owner] = struct{}{} 386 386 + dids = append(dids, r.Owner) 387 387 + } 388 388 + if err := s.tap.AddOwnerDIDs(ctx, dids); err != nil { 389 389 + s.l.Warn("tap declare: AddRepos rejected", "count", len(dids), "err", err) 390 390 + return 391 391 + } 392 392 + s.l.Info("tap declare: known owner DIDs registered", "count", len(dids)) 393 393 + } 394 394 + 395 395 + func Run(ctx context.Context) error { 396 396 + cfg, err := config.Load(ctx) 397 397 + if err != nil { 398 398 + return fmt.Errorf("failed to load config: %w", err) 399 399 + } 400 400 + 401 401 + if err := ensureGitVersion(); err != nil { 402 402 + return fmt.Errorf("ensuring git version: %w", err) 403 403 + } 404 404 + 405 405 + d, err := db.Make(ctx, cfg.Server.DBPath) 406 406 + if err != nil { 407 407 + return fmt.Errorf("failed to setup db: %w", err) 408 408 + } 409 409 + 410 410 + logger := log.FromContext(ctx) 411 411 + 412 412 + var engines map[string]models.Engine 413 413 + var m *mill.Mill 414 414 + 415 415 + if cfg.Role == config.RoleMill { 416 416 + // mill host: register engines that place jobs on executors instead of 417 417 + // running them. all names share one Mill 418 418 + m = mill.New(log.SubLogger(logger, "mill"), mill.Config{ 419 419 + LogDir: cfg.Server.LogDir, 420 420 + MaxPending: cfg.Mill.MaxPending, 421 421 + ReconnectGrace: cfg.Mill.ReconnectGrace, 422 422 + }) 423 423 + engines = map[string]models.Engine{ 424 424 + "nixery": mill.NewEngine("nixery", m), 425 425 + "microvm": mill.NewEngine("microvm", m), 426 426 + "dummy": mill.NewEngine("dummy", m), 427 427 + } 428 428 + } else { 429 429 + // standalone and executor both run real engines locally. 430 430 + nixeryEng, err := nixery.New(ctx, cfg) 431 431 + if err != nil { 432 432 + return err 433 433 + } 434 434 + microvmEng, err := newMicrovmEngine(ctx, cfg, d) 435 435 + if err != nil { 436 436 + return err 437 437 + } 438 438 + engines = map[string]models.Engine{ 439 439 + "nixery": nixeryEng, 440 440 + "microvm": microvmEng, 441 441 + "dummy": dummy.New(logger), 442 442 + } 443 443 + } 444 444 + 445 445 + s, err := New(ctx, cfg, d, engines) 446 446 + if err != nil { 447 447 + return err 448 448 + } 449 449 + 450 450 + if m != nil { 451 451 + // resolve the chicken-and-egg: the engines (built above) hold the mill, 452 452 + // but the mill's db/notifier are created inside New 453 453 + m.Attach(s.DB(), s.Notifier()) 454 454 + s.mill = m 455 455 + if err := m.RestoreState(); err != nil { 456 456 + return fmt.Errorf("restoring mill state: %w", err) 457 457 + } 458 458 + } 459 459 + if cfg.Role == config.RoleExecutor { 460 460 + s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store) 461 461 + if err != nil { 462 462 + return err 463 463 + } 464 464 + } 465 465 + 466 466 + return s.Start(ctx) 467 467 + } 468 468 + 469 469 + func (s *Spindle) Router() http.Handler { 470 470 + mux := chi.NewRouter() 471 471 + 472 472 + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 473 473 + w.Write(s.GetMotdContent()) 474 474 + }) 475 475 + if s.cfg.Role == config.RoleExecutor { 476 476 + return mux 477 477 + } 478 478 + 479 479 + mux.HandleFunc("/events", s.Events) 480 480 + mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) 481 481 + 482 482 + // mill host: executors dial in here (plain ws, shared-secret auth) 483 483 + if s.mill != nil { 484 484 + mux.HandleFunc("/mill", s.mill.HandleExecutorConn) 485 485 + } 486 486 + 487 487 + mux.Mount("/xrpc", s.XrpcRouter()) 488 488 + return mux 489 489 + } 490 490 + 491 491 + func (s *Spindle) XrpcRouter() http.Handler { 492 492 + serviceAuth := serviceauth.NewServiceAuth(s.l, s.res.Directory(), s.cfg.Server.Did().String()) 493 493 + 494 494 + l := log.SubLogger(s.l, "xrpc") 495 495 + 496 496 + x := xrpc.Xrpc{ 497 497 + Logger: l, 498 498 + Db: s.db, 499 499 + Enforcer: s.e, 500 500 + Engines: s.engs, 501 501 + Config: s.cfg, 502 502 + ArtifactReader: s.reader, 503 503 + Resolver: s.res, 504 504 + Vault: s.vault, 505 505 + Notifier: s.Notifier(), 506 506 + ServiceAuth: serviceAuth, 507 507 + Trigger: s, 508 508 + } 509 509 + 510 510 + return x.Router() 511 511 + } 512 512 + 513 513 + func (s *Spindle) processKnotStream(ctx context.Context, src eventconsumer.Source, msg eventstream.Event) error { 514 514 + l := log.FromContext(ctx).With("handler", "processKnotStream") 515 515 + l = l.With("src", src.Key(), "msg.Nsid", msg.Nsid, "msg.Rkey", msg.Rkey) 516 516 + if msg.Nsid == knotdb.RepoCollaboratorUpdateNSID { 517 517 + return s.ingestKnotCollaborator(ctx, l, src, msg) 518 518 + } 519 519 + if msg.Nsid == tangled.GitRefUpdateNSID { 520 520 + event := tangled.GitRefUpdate{} 521 521 + if err := json.Unmarshal(msg.EventJson, &event); err != nil { 522 522 + l.Error("error unmarshalling", "err", err) 523 523 + return err 524 524 + } 525 525 + l = l.With("repo", event.Repo, "ref", event.Ref, "newSha", event.NewSha) 526 526 + l.Debug("debug") 527 527 + 528 528 + repoDid := syntax.DID(event.Repo) 529 529 + repo, err := s.db.GetRepoByDid(repoDid) 530 530 + if err != nil { 531 531 + return fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 532 532 + } 533 533 + 534 534 + if src.Host != repo.Knot { 535 535 + return fmt.Errorf("repo knot does not match event source: %s != %s", src.Host, repo.Knot) 536 536 + } 537 537 + 538 538 + if kgit.HasSkipCIPushOption(event.PushOptions) { 539 539 + l.Info("push event requested ci skip, skipping the event") 540 540 + return nil 541 541 + } 542 542 + 543 543 + // NOTE: we are blindly trusting the knot that it will return only repos it own 544 544 + repoCloneUri := s.newRepoCloneUrl(src.Host, repoDid) 545 545 + repoPath := s.newRepoPath(repoDid) 546 546 + if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, event.NewSha); err != nil { 547 547 + return fmt.Errorf("sync git repo: %w", err) 548 548 + } 549 549 + l.Info("synced git repo") 550 550 + 551 551 + triggerRepo, err := s.buildTriggerRepo(ctx, repo) 552 552 + if err != nil { 553 553 + return fmt.Errorf("building trigger repo: %w", err) 554 554 + } 555 555 + 556 556 + trigger := tangled.Pipeline_TriggerMetadata{ 557 557 + Kind: string(workflow.TriggerKindPush), 558 558 + Push: &tangled.Pipeline_PushTriggerData{ 559 559 + Ref: event.Ref, 560 560 + OldSha: event.OldSha, 561 561 + NewSha: event.NewSha, 562 562 + }, 563 563 + Repo: triggerRepo, 564 564 + } 565 565 + 566 566 + pipelineId, err := s.runPipeline(ctx, repoDid, trigger, event.ChangedFiles, repoCloneUri, repoPath, event.NewSha, nil, triggerRepo) 567 567 + if err != nil { 568 568 + return err 569 569 + } 570 570 + if pipelineId.Rkey == "" { 571 571 + l.Info("no workflow matched 'push' trigger, skipping the event") 572 572 + return nil 573 573 + } 574 574 + l.Info("pipeline triggered", "pipeline", pipelineId.AtUri()) 575 575 + } 576 576 + 577 577 + return nil 578 578 + } 579 579 + 580 580 + func (s *Spindle) ingestKnotCollaborator(ctx context.Context, l *slog.Logger, src eventconsumer.Source, msg eventstream.Event) error { 581 581 + var rec knotdb.RepoCollaboratorUpdate 582 582 + if err := json.Unmarshal(msg.EventJson, &rec); err != nil { 583 583 + l.Error("error unmarshalling collaboratorUpdate", "err", err) 584 584 + return err 585 585 + } 586 586 + 587 587 + subject, err := syntax.ParseDID(rec.Subject) 588 588 + if err != nil { 589 589 + l.Info("skipping collaboratorUpdate with malformed subject", "subject", rec.Subject, "err", err) 590 590 + return nil 591 591 + } 592 592 + repoDid, err := syntax.ParseDID(rec.Repo) 593 593 + if err != nil { 594 594 + l.Info("skipping collaboratorUpdate with malformed repo", "repo", rec.Repo, "err", err) 595 595 + return nil 596 596 + } 597 597 + 598 598 + repo, err := s.db.GetRepoByDid(repoDid) 599 599 + if errors.Is(err, sql.ErrNoRows) { 600 600 + l.Info("skipping collaboratorUpdate for unknown repo", "repo", repoDid) 601 601 + return nil 602 602 + } 603 603 + if err != nil { 604 604 + return fmt.Errorf("lookup repo %s: %w", repoDid, err) 605 605 + } 606 606 + if src.Host != repo.Knot { 607 607 + l.Warn("dropping collaboratorUpdate from non-owning knot", "src", src.Host, "repoKnot", repo.Knot) 608 608 + return nil 609 609 + } 610 610 + 611 611 + switch rec.Op { 612 612 + case knotdb.AclOpAdd: 613 613 + if err := s.e.AddCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { 614 614 + return fmt.Errorf("add collaborator policy: %w", err) 615 615 + } 616 616 + if err := s.db.AddKnotCollaborator(repoDid, subject); err != nil { 617 617 + return fmt.Errorf("track collaborator: %w", err) 618 618 + } 619 619 + l.Info("added knot-managed collaborator", "subject", subject, "repo", repoDid) 620 620 + case knotdb.AclOpRemove: 621 621 + if err := s.e.RemoveCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { 622 622 + return fmt.Errorf("remove collaborator policy: %w", err) 623 623 + } 624 624 + if err := s.db.DeleteRepoCollaboratorBySubjectRepo(subject, repoDid); err != nil { 625 625 + return fmt.Errorf("delete collaborator row: %w", err) 626 626 + } 627 627 + l.Info("removed knot-managed collaborator", "subject", subject, "repo", repoDid) 628 628 + default: 629 629 + return fmt.Errorf("collaboratorUpdate unknown op %q", rec.Op) 630 630 + } 631 631 + return nil 632 632 + } 633 633 + 634 634 + // buildTriggerRepo gathers trigger metadata, resolving default branch from the knot 635 635 + func (s *Spindle) buildTriggerRepo(ctx context.Context, repo *db.Repo) (*tangled.Pipeline_TriggerRepo, error) { 636 636 + rkey := string(repo.Rkey) 637 637 + repoDid := repo.RepoDid.String() 638 638 + return s.buildTriggerRepoFrom(ctx, repo.Knot, repo.Owner.String(), rkey, repoDid), nil 639 639 + } 640 640 + 641 641 + func (s *Spindle) buildTriggerRepoFrom(ctx context.Context, knot, did, rkey, repoDid string) *tangled.Pipeline_TriggerRepo { 642 642 + scheme := "https" 643 643 + if s.cfg.Server.Dev { 644 644 + scheme = "http" 645 645 + } 646 646 + client := &indigoxrpc.Client{Host: fmt.Sprintf("%s://%s", scheme, knot)} 647 647 + 648 648 + // this should maybe (?) be in the refUpdate event itself to save a roundtrip 649 649 + defaultBranch := "" 650 650 + if out, err := tangled.RepoGetDefaultBranch(ctx, client, repoDid); err == nil { 651 651 + defaultBranch = out.Name 652 652 + } 653 653 + 654 654 + var rkeyPtr *string 655 655 + if rkey != "" { 656 656 + rkeyPtr = &rkey 657 657 + } 658 658 + return &tangled.Pipeline_TriggerRepo{ 659 659 + Did: did, 660 660 + Knot: knot, 661 661 + Repo: rkeyPtr, 662 662 + RepoDid: &repoDid, 663 663 + DefaultBranch: defaultBranch, 664 664 + } 665 665 + } 666 666 + 667 667 + func (s *Spindle) resolvePipelineSourceRepo(ctx context.Context, trigger *tangled.Pipeline_TriggerMetadata) (*tangled.Pipeline_TriggerRepo, error) { 668 668 + if trigger == nil { 669 669 + return nil, nil 670 670 + } 671 671 + if trigger.SourceRepo == nil || *trigger.SourceRepo == "" { 672 672 + return trigger.Repo, nil 673 673 + } 674 674 + repoDid, err := syntax.ParseDID(*trigger.SourceRepo) 675 675 + if err != nil { 676 676 + return nil, fmt.Errorf("parse sourceRepo %s: %w", *trigger.SourceRepo, err) 677 677 + } 678 678 + return s.resolveSourceRepoInfo(ctx, repoDid) 679 679 + } 680 680 + 681 681 + // resolveSourceRepoInfo resolves trigger-repo metadata for a source repo DID. 682 682 + func (s *Spindle) resolveSourceRepoInfo(ctx context.Context, repoDid syntax.DID) (*tangled.Pipeline_TriggerRepo, error) { 683 683 + repo, err := s.db.GetRepoByDid(repoDid) 684 684 + if err == nil { 685 685 + return s.buildTriggerRepo(ctx, repo) 686 686 + } 687 687 + 688 688 + // verify repo, we don't want git sync to point to arbitrary endpoints 689 689 + res, err := s.verify(ctx, repoident.RepoDid(repoDid)) 690 690 + if err != nil { 691 691 + return nil, fmt.Errorf("verify sourceRepo %s: %w", repoDid, err) 692 692 + } 693 693 + return s.buildTriggerRepoFrom(ctx, res.KnotURL.Host, res.OwnerDid.String(), res.Rkey, repoDid.String()), nil 694 694 + } 695 695 + 696 696 + // runPipeline compiles and enqueues the pipeline for the given revision. 697 697 + // sourceRepo is the resolved repo the code was checked out from, forwarded to 698 698 + // processPipeline for env vars. 699 699 + func (s *Spindle) runPipeline(ctx context.Context, repoDid syntax.DID, trigger tangled.Pipeline_TriggerMetadata, changedFiles []string, repoCloneUri, repoPath, rev string, only []string, sourceRepo *tangled.Pipeline_TriggerRepo) (models.PipelineId, error) { 700 700 + l := log.FromContext(ctx) 701 701 + 702 702 + compiler := workflow.Compiler{ 703 703 + ChangedFiles: changedFiles, 704 704 + Trigger: trigger, 705 705 + } 706 706 + 707 707 + rawPipeline, err := s.loadPipeline(ctx, repoCloneUri, repoPath, rev) 708 708 + if err != nil { 709 709 + return models.PipelineId{}, fmt.Errorf("loading pipeline: %w", err) 710 710 + } 711 711 + if len(rawPipeline) == 0 { 712 712 + return models.PipelineId{}, nil 713 713 + } 714 714 + 715 715 + tpl := compiler.Compile(compiler.Parse(rawPipeline)) 716 716 + // todo(dawn): pass compile error to workflow log 717 717 + for _, w := range compiler.Diagnostics.Errors { 718 718 + l.Error(w.String()) 719 719 + } 720 720 + for _, w := range compiler.Diagnostics.Warnings { 721 721 + l.Warn(w.String()) 722 722 + } 723 723 + 724 724 + if len(only) > 0 { 725 725 + tpl.Workflows = filterWorkflows(tpl.Workflows, only) 726 726 + } 727 727 + if len(tpl.Workflows) == 0 { 728 728 + return models.PipelineId{}, nil 729 729 + } 730 730 + 731 731 + pipelineId := models.PipelineId{ 732 732 + Knot: trigger.Repo.Knot, 733 733 + Rkey: tid.TID(), 734 734 + } 735 735 + if err := s.db.CreatePipelineEvent(pipelineId.Rkey, tpl, s.n); err != nil { 736 736 + return models.PipelineId{}, fmt.Errorf("creating pipeline event: %w", err) 737 737 + } 738 738 + err = s.processPipeline(repoDid, tpl, pipelineId, sourceRepo) 739 739 + return pipelineId, err 740 740 + } 741 741 + 742 742 + // filterWorkflows filters workflows to the requested names 743 743 + func filterWorkflows(workflows []*tangled.Pipeline_Workflow, only []string) []*tangled.Pipeline_Workflow { 744 744 + allowed := make(map[string]struct{}, len(only)) 745 745 + for _, n := range only { 746 746 + allowed[n] = struct{}{} 747 747 + } 748 748 + var filtered []*tangled.Pipeline_Workflow 749 749 + for _, w := range workflows { 750 750 + if w == nil { 751 751 + continue 752 752 + } 753 753 + if _, ok := allowed[w.Name]; ok { 754 754 + filtered = append(filtered, w) 755 755 + } 756 756 + } 757 757 + return filtered 758 758 + } 759 759 + 760 760 + // TriggerManual dispatches a pipeline at sha, authorized against and recorded 761 761 + // under repoDid. sourceRepo, pull, and inputs are optional trigger payload. 762 762 + func (s *Spindle) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull xrpc.PullContext, inputs []*tangled.Pipeline_Pair) (syntax.ATURI, error) { 763 763 + repo, err := s.db.GetRepoByDid(repoDid) 764 764 + if err != nil { 765 765 + return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 766 766 + } 767 767 + 768 768 + triggerRepo, err := s.buildTriggerRepo(ctx, repo) 769 769 + if err != nil { 770 770 + return "", fmt.Errorf("building trigger repo: %w", err) 771 771 + } 772 772 + 773 773 + trigger := tangled.Pipeline_TriggerMetadata{Repo: triggerRepo} 774 774 + if pull.IsPullRequest { 775 775 + var pullAt *string 776 776 + if pull.Pull != "" { 777 777 + pullAtStr := pull.Pull.String() 778 778 + pullAt = &pullAtStr 779 779 + } 780 780 + trigger.Kind = string(workflow.TriggerKindPullRequest) 781 781 + trigger.PullRequest = &tangled.Pipeline_PullRequestTriggerData{ 782 782 + SourceBranch: pull.SourceBranch, 783 783 + TargetBranch: pull.TargetBranch, 784 784 + SourceSha: sha, 785 785 + Pull: pullAt, 786 786 + } 787 787 + } else { 788 788 + var refPtr *string 789 789 + if ref != "" { 790 790 + refPtr = &ref 791 791 + } 792 792 + trigger.Kind = string(workflow.TriggerKindManual) 793 793 + trigger.Manual = &tangled.Pipeline_ManualTriggerData{ 794 794 + Sha: sha, 795 795 + Ref: refPtr, 796 796 + Inputs: inputs, 797 797 + } 798 798 + } 799 799 + 800 800 + repoCloneUri := s.newRepoCloneUrl(repo.Knot, repoDid) 801 801 + repoPath := s.newRepoPath(repoDid) 802 802 + sourceInfo := triggerRepo // default: code comes from the repo itself 803 803 + if sourceRepo != "" && sourceRepo != repoDid { 804 804 + sourceInfo, err = s.resolveSourceRepoInfo(ctx, sourceRepo) 805 805 + if err != nil { 806 806 + return "", err 807 807 + } 808 808 + sourceRepoStr := sourceRepo.String() 809 809 + trigger.SourceRepo = &sourceRepoStr 810 810 + repoCloneUri = models.BuildRepoURL(sourceInfo) 811 811 + repoPath = s.newRepoPath(sourceRepo) 812 812 + } 813 813 + 814 814 + pipelineId, err := s.runPipeline(ctx, repoDid, trigger, nil, repoCloneUri, repoPath, sha, workflows, sourceInfo) 815 815 + if err != nil { 816 816 + return "", err 817 817 + } 818 818 + if pipelineId.Rkey == "" { 819 819 + return "", xrpc.ErrNoMatchingWorkflows 820 820 + } 821 821 + return pipelineId.AtUri(), nil 822 822 + } 823 823 + 824 824 + func (s *Spindle) loadPipeline(ctx context.Context, repoUri, repoPath, rev string) (workflow.RawPipeline, error) { 825 825 + if err := git.SparseSyncGitRepo(ctx, repoUri, repoPath, rev); err != nil { 826 826 + return nil, fmt.Errorf("syncing git repo: %w", err) 827 827 + } 828 828 + gr, err := kgit.Open(repoPath, rev) 829 829 + if err != nil { 830 830 + return nil, fmt.Errorf("opening git repo: %w", err) 831 831 + } 832 832 + 833 833 + workflowDir, err := gr.FileTree(ctx, workflow.WorkflowDir) 834 834 + if errors.Is(err, object.ErrDirectoryNotFound) { 835 835 + // return empty RawPipeline when directory doesn't exist 836 836 + return nil, nil 837 837 + } else if err != nil { 838 838 + return nil, fmt.Errorf("loading file tree: %w", err) 839 839 + } 840 840 + 841 841 + var rawPipeline workflow.RawPipeline 842 842 + for _, e := range workflowDir { 843 843 + if !e.IsFile() { 844 844 + continue 845 845 + } 846 846 + 847 847 + fpath := filepath.Join(workflow.WorkflowDir, e.Name) 848 848 + contents, err := gr.RawContent(fpath) 849 849 + if err != nil { 850 850 + return nil, fmt.Errorf("reading raw content of '%s': %w", fpath, err) 851 851 + } 852 852 + 853 853 + rawPipeline = append(rawPipeline, workflow.RawWorkflow{ 854 854 + Name: e.Name, 855 855 + Contents: contents, 856 856 + }) 857 857 + } 858 858 + 859 859 + return rawPipeline, nil 860 860 + } 861 861 + 862 862 + // processPipeline enqueues the workflows in tpl. 863 863 + func (s *Spindle) processPipeline(repoDid syntax.DID, tpl tangled.Pipeline, pipelineId models.PipelineId, sourceRepo *tangled.Pipeline_TriggerRepo) error { 864 864 + // derive security-relevant things like whether this run is trusted and can be passed 865 865 + // secrets to from the original metadata. 866 866 + pipelineEnv := models.PipelineEnvVarsForSource(tpl.TriggerMetadata, pipelineId, sourceRepo) 867 867 + trustedSource := true 868 868 + if tm := tpl.TriggerMetadata; tm != nil && tm.SourceRepo != nil && 869 869 + *tm.SourceRepo != "" && *tm.SourceRepo != repoDid.String() { 870 870 + trustedSource = false 871 871 + } 872 872 + 873 873 + // swap the repo with our sourceRepo if we are running a pipeline on a fork. 874 874 + // the metadata stays the same. we check whether the repo is trusted above, 875 875 + // so this only affects the clone URL. 876 876 + initTpl := tpl 877 877 + if sourceRepo != nil && tpl.TriggerMetadata != nil { 878 878 + tm := *tpl.TriggerMetadata 879 879 + tm.Repo = sourceRepo 880 880 + initTpl.TriggerMetadata = &tm 881 881 + } 882 882 + 883 883 + // filter & init workflows 884 884 + workflows := make(map[models.Engine][]models.Workflow) 885 885 + for _, w := range tpl.Workflows { 886 886 + if w == nil { 887 887 + continue 888 888 + } 889 889 + eng, ok := s.engs[w.Engine] 890 890 + if !ok { 891 891 + err := s.db.StatusFailed(models.WorkflowId{ 892 892 + PipelineId: pipelineId, 893 893 + Name: w.Name, 894 894 + }, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n) 895 895 + if err != nil { 896 896 + return fmt.Errorf("db.StatusFailed: %w", err) 897 897 + } 898 898 + 899 899 + continue 900 900 + } 901 901 + 902 902 + ewf, err := eng.InitWorkflow(*w, initTpl) 903 903 + if err != nil { 904 904 + err = s.db.StatusFailed(models.WorkflowId{ 905 905 + PipelineId: pipelineId, 906 906 + Name: w.Name, 907 907 + }, fmt.Sprintf("init workflow: %s", err), -1, s.n) 908 908 + if err != nil { 909 909 + return fmt.Errorf("db.StatusFailed: %w", err) 910 910 + } 911 911 + 912 912 + continue 913 913 + } 914 914 + 915 915 + // inject TANGLED_* env vars after InitWorkflow 916 916 + // This prevents user-defined env vars from overriding them 917 917 + if ewf.Environment == nil { 918 918 + ewf.Environment = make(map[string]string) 919 919 + } 920 920 + maps.Copy(ewf.Environment, pipelineEnv) 921 921 + 922 922 + workflows[eng] = append(workflows[eng], *ewf) 923 923 + } 924 924 + 925 925 + pipeline := &models.Pipeline{ 926 926 + RepoDid: repoDid, 927 927 + Workflows: workflows, 928 928 + TrustedSource: trustedSource, 929 929 + } 930 930 + 931 931 + if s.mill != nil { 932 932 + // mill host: no bounded pool. each job blocks in placement 933 933 + // (AcquireWorkflowSlot) which the user sees as pending. the only bound 934 934 + // is the mill's maxPending. rootCtx is the long-lived consumer context, 935 935 + // so the goroutine safely outlives this call 936 936 + go engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.rootCtx, pipeline, pipelineId) 937 937 + s.l.Info("pipeline handed to mill placement", "id", pipelineId) 938 938 + } else if s.jq != nil { 939 939 + ok := s.jq.Enqueue(repoDid, queue.Job{ 940 940 + Run: func() error { 941 941 + engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.rootCtx, pipeline, pipelineId) 942 942 + return nil 943 943 + }, 944 944 + OnFail: func(jobError error) { 945 945 + s.l.Error("pipeline run failed", "error", jobError) 946 946 + }, 947 947 + }) 948 948 + if !ok { 949 949 + return fmt.Errorf("failed to enqueue pipeline: queue is full") 950 950 + } 951 951 + s.l.Info("pipeline enqueued successfully", "id", pipelineId) 952 952 + } else { 953 953 + return fmt.Errorf("no queue or mill available to process pipeline") 954 954 + } 955 955 + 956 956 + // after successful enqueue, emit StatusPending for all workflows 957 957 + for _, ewfs := range workflows { 958 958 + for _, ewf := range ewfs { 959 959 + err := s.db.StatusPending(models.WorkflowId{ 960 960 + PipelineId: pipelineId, 961 961 + Name: ewf.Name, 962 962 + }, s.n) 963 963 + if err != nil { 964 964 + return fmt.Errorf("db.StatusPending: %w", err) 965 965 + } 966 966 + } 967 967 + } 968 968 + return nil 969 969 + } 970 970 + 971 971 + // newRepoPath creates a path to store repository by its did and rkey. 972 972 + // The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey 973 973 + func (s *Spindle) newRepoPath(repo syntax.DID) string { 974 974 + return filepath.Join(s.cfg.Server.RepoDir, repo.String()) 975 975 + } 976 976 + 977 977 + func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { 978 978 + scheme := "https://" 979 979 + if s.cfg.Server.Dev { 980 980 + scheme = "http://" 981 981 + } 982 982 + return fmt.Sprintf("%s%s/%s", scheme, knot, did) 983 983 + } 984 984 + 985 985 + const RequiredVersion = "2.49.0" 986 986 + 987 987 + func ensureGitVersion() error { 988 988 + v, err := git.Version() 989 989 + if err != nil { 990 990 + return fmt.Errorf("fetching git version: %w", err) 991 991 + } 992 992 + if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { 993 993 + return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) 994 994 + } 995 995 + return nil 996 996 + } 997 997 + 998 998 + func (s *Spindle) configureOwner() error { 999 999 + cfgOwner := s.cfg.Server.Owner 1000 1000 + 1001 1001 + existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) 1002 1002 + if err != nil { 1003 1003 + return err 1004 1004 + } 1005 1005 + 1006 1006 + switch len(existing) { 1007 1007 + case 0: 1008 1008 + // no owner configured, continue 1009 1009 + case 1: 1010 1010 + // find existing owner 1011 1011 + existingOwner := existing[0] 1012 1012 + 1013 1013 + // no ownership change, this is okay 1014 1014 + if existingOwner == s.cfg.Server.Owner { 1015 1015 + break 1016 1016 + } 1017 1017 + 1018 1018 + // remove existing owner 1019 1019 + err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) 1020 1020 + if err != nil { 1021 1021 + return nil 1022 1022 + } 1023 1023 + default: 1024 1024 + return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) 1025 1025 + } 1026 1026 + 1027 1027 + return s.e.AddSpindleOwner(rbacDomain, cfgOwner) 1028 1028 + }

+46

spindle/server_test.go

Reviewed

··· 1 1 package spindle 2 2 3 3 import ( 4 4 + "context" 5 5 + "net/http" 6 6 + "net/http/httptest" 7 7 + "path/filepath" 4 8 "testing" 5 9 6 10 kgit "tangled.org/core/knotserver/git" 11 11 + "tangled.org/core/spindle/config" 12 12 + "tangled.org/core/spindle/db" 13 13 + "tangled.org/core/spindle/models" 7 14 ) 8 15 9 16 func TestHasSkipCIPushOption(t *testing.T) { ··· 53 60 }) 54 61 } 55 62 } 63 63 + 64 64 + func TestExecutorRoleBuildsMinimalSpindle(t *testing.T) { 65 65 + ctx := context.Background() 66 66 + dbPath := filepath.Join(t.TempDir(), "spindle.db") 67 67 + d, err := db.Make(ctx, dbPath) 68 68 + if err != nil { 69 69 + t.Fatalf("db.Make() error = %v", err) 70 70 + } 71 71 + 72 72 + cfg := &config.Config{Role: config.RoleExecutor} 73 73 + cfg.Server.DBPath = dbPath 74 74 + cfg.Server.Hostname = "executor.test" 75 75 + cfg.Server.Tap.Embed = true 76 76 + cfg.ArtifactStores.Disk.Dir = t.TempDir() 77 77 + cfg.Mill.ArtifactStore = "disk" 78 78 + 79 79 + s, err := New(ctx, cfg, d, map[string]models.Engine{}) 80 80 + if err != nil { 81 81 + t.Fatalf("New() error = %v", err) 82 82 + } 83 83 + 84 84 + if s.jc != nil || s.tap != nil || s.e != nil || s.ks != nil || s.res != nil || s.vault != nil { 85 85 + t.Fatal("executor role built coordinator-only spindle dependencies") 86 86 + } 87 87 + 88 88 + rr := httptest.NewRecorder() 89 89 + req := httptest.NewRequest(http.MethodGet, "/", nil) 90 90 + s.Router().ServeHTTP(rr, req) 91 91 + if rr.Code != http.StatusOK { 92 92 + t.Fatalf("root status = %d, want %d", rr.Code, http.StatusOK) 93 93 + } 94 94 + 95 95 + rr = httptest.NewRecorder() 96 96 + req = httptest.NewRequest(http.MethodGet, "/xrpc/_health", nil) 97 97 + s.Router().ServeHTTP(rr, req) 98 98 + if rr.Code != http.StatusNotFound { 99 99 + t.Fatalf("executor xrpc status = %d, want %d", rr.Code, http.StatusNotFound) 100 100 + } 101 101 + }

+12 -26

spindle/stream.go

Reviewed

··· 4 4 "context" 5 5 "errors" 6 6 "fmt" 7 7 - "io" 8 7 "net/http" 9 8 "time" 10 9 11 10 "tangled.org/core/eventstream" 12 11 "tangled.org/core/log" 12 12 + "tangled.org/core/spindle/logview" 13 13 "tangled.org/core/spindle/models" 14 14 15 15 "github.com/go-chi/chi/v5" 16 16 "github.com/gorilla/websocket" 17 17 - "github.com/hpcloud/tail" 18 17 ) 19 18 20 19 var upgrader = websocket.Upgrader{ ··· 89 88 } 90 89 isFinished := models.StatusKind(status.Status).IsFinish() 91 90 92 92 - filePath := models.LogFilePath(s.cfg.Server.LogDir, wid) 93 93 - 94 94 - config := tail.Config{ 95 95 - Follow: !isFinished, 96 96 - ReOpen: !isFinished, 97 97 - MustExist: false, 98 98 - Location: &tail.SeekInfo{ 99 99 - Offset: 0, 100 100 - Whence: io.SeekStart, 101 101 - }, 102 102 - // Logger: tail.DiscardingLogger, 103 103 - } 104 104 - 105 105 - t, err := tail.TailFile(filePath, config) 91 91 + lines, stop, err := logview.Follow(ctx, s.db, s.reader, s.cfg.Server.LogDir, wid, isFinished) 106 92 if err != nil { 107 107 - return fmt.Errorf("failed to tail log file: %w", err) 93 93 + return fmt.Errorf("failed to follow workflow log: %w", err) 108 94 } 109 109 - defer t.Stop() 110 110 - 95 95 + defer stop() 111 96 for { 112 97 select { 113 98 case <-ctx.Done(): 114 99 return ctx.Err() 115 115 - case line := <-t.Lines: 116 116 - if line == nil && isFinished { 117 117 - return fmt.Errorf("tail completed") 100 100 + case line, ok := <-lines: 101 101 + if !ok && isFinished { 102 102 + return fmt.Errorf("log completed") 103 103 + } 104 104 + if !ok { 105 105 + return fmt.Errorf("log channel closed unexpectedly") 118 106 } 119 119 - 120 107 if line == nil { 121 121 - return fmt.Errorf("tail channel closed unexpectedly") 108 108 + continue 122 109 } 123 123 - 124 110 if line.Err != nil { 125 125 - return fmt.Errorf("error tailing log file: %w", line.Err) 111 111 + return fmt.Errorf("error following workflow log: %w", line.Err) 126 112 } 127 113 128 114 if err := conn.WriteMessage(websocket.TextMessage, []byte(line.Text)); err != nil {

+7 -20

spindle/xrpc/ci_pipeline_subscribe_logs.go

Reviewed

··· 4 4 "context" 5 5 "encoding/json" 6 6 "fmt" 7 7 - "io" 8 7 "net/http" 9 8 "sync" 10 9 "time" ··· 12 11 "github.com/bluesky-social/indigo/atproto/atclient" 13 12 "github.com/bluesky-social/indigo/atproto/syntax" 14 13 "github.com/gorilla/websocket" 15 15 - "github.com/hpcloud/tail" 16 14 "tangled.org/core/api/tangled" 15 15 + "tangled.org/core/spindle/logview" 17 16 "tangled.org/core/spindle/models" 18 17 ) 19 18 ··· 166 165 isFinished = models.StatusKind(status.Status).IsFinish() 167 166 } 168 167 169 169 - filePath := models.LogFilePath(x.Config.Server.LogDir, wid) 170 170 - 171 171 - tailConfig := tail.Config{ 172 172 - Follow: !isFinished, 173 173 - ReOpen: !isFinished, 174 174 - MustExist: false, 175 175 - Location: &tail.SeekInfo{ 176 176 - Offset: 0, 177 177 - Whence: io.SeekStart, 178 178 - }, 179 179 - } 180 180 - 181 181 - t, err := tail.TailFile(filePath, tailConfig) 168 168 + lines, stop, err := logview.Follow(ctx, x.Db, x.ArtifactReader, x.Config.Server.LogDir, wid, isFinished) 182 169 if err != nil { 183 183 - l.Error("failed to tail log file", "workflow", wfName, "err", err) 170 170 + l.Error("failed to follow workflow log", "workflow", wfName, "err", err) 184 171 return 185 172 } 186 186 - defer t.Stop() 173 173 + defer stop() 187 174 188 188 - // if we are following, poll status in database to stop tailing when finished 175 175 + // if we are following, poll status in database to stop when finished 189 176 if !isFinished { 190 177 go func() { 191 178 ticker := time.NewTicker(2 * time.Second) ··· 197 184 case <-ticker.C: 198 185 status, err := x.Db.GetStatus(wid) 199 186 if err == nil && models.StatusKind(status.Status).IsFinish() { 200 200 - t.Stop() 187 187 + stop() 201 188 return 202 189 } 203 190 } ··· 209 196 select { 210 197 case <-ctx.Done(): 211 198 return 212 212 - case line, ok := <-t.Lines: 199 199 + case line, ok := <-lines: 213 200 if !ok || line == nil { 214 201 return 215 202 }

+12 -10

spindle/xrpc/xrpc.go

Reviewed

··· 15 15 "tangled.org/core/idresolver" 16 16 "tangled.org/core/notifier" 17 17 "tangled.org/core/rbac" 18 18 + "tangled.org/core/spindle/artifactstore" 18 19 "tangled.org/core/spindle/config" 19 20 "tangled.org/core/spindle/db" 20 21 "tangled.org/core/spindle/models" ··· 41 42 } 42 43 43 44 type Xrpc struct { 44 44 - Logger *slog.Logger 45 45 - Db *db.DB 46 46 - Enforcer *rbac.Enforcer 47 47 - Engines map[string]models.Engine 48 48 - Config *config.Config 49 49 - Resolver *idresolver.Resolver 50 50 - Vault secrets.Manager 51 51 - Notifier *notifier.Notifier 52 52 - ServiceAuth *serviceauth.ServiceAuth 53 53 - Trigger PipelineTrigger 45 45 + Logger *slog.Logger 46 46 + Db *db.DB 47 47 + Enforcer *rbac.Enforcer 48 48 + Engines map[string]models.Engine 49 49 + Config *config.Config 50 50 + ArtifactReader artifactstore.Reader 51 51 + Resolver *idresolver.Resolver 52 52 + Vault secrets.Manager 53 53 + Notifier *notifier.Notifier 54 54 + ServiceAuth *serviceauth.ServiceAuth 55 55 + Trigger PipelineTrigger 54 56 } 55 57 56 58 func (x *Xrpc) Router() http.Handler {

+7 -8

workflow/compile.go

Reviewed

··· 112 112 } 113 113 114 114 func (compiler *Compiler) compileWorkflow(w Workflow) *tangled.Pipeline_Workflow { 115 115 - cw := &tangled.Pipeline_Workflow{} 116 116 - 117 115 matched, err := w.Match(compiler.Trigger, compiler.ChangedFiles) 118 116 if err != nil { 119 117 compiler.Diagnostics.AddError( ··· 134 132 // validate clone options 135 133 compiler.analyzeCloneOptions(w) 136 134 137 137 - cw.Name = w.Name 138 138 - 139 135 if w.Engine == "" { 140 136 compiler.Diagnostics.AddError(w.Name, MissingEngine) 141 137 return nil 142 138 } 143 139 144 144 - cw.Engine = w.Engine 145 145 - cw.Raw = w.Raw 146 146 - 147 140 o := w.CloneOpts.AsRecord() 148 148 - cw.Clone = &o 141 141 + cw := &tangled.Pipeline_Workflow{ 142 142 + Clone: &o, 143 143 + Engine: w.Engine, 144 144 + Name: w.Name, 145 145 + Raw: w.Raw, 146 146 + RunsOn: w.RunsOn, 147 147 + } 149 148 150 149 return cw 151 150 }

+45

workflow/compile_test.go

Reviewed

··· 1 1 package workflow 2 2 3 3 import ( 4 4 + "encoding/json" 4 5 "strings" 5 6 "testing" 6 7 ··· 41 42 assert.False(t, c.Diagnostics.IsErr()) 42 43 } 43 44 45 45 + func TestCompileWorkflow_RunsOnLabelsPersistWithoutFanout(t *testing.T) { 46 46 + wf := Workflow{ 47 47 + Name: ".tangled/workflows/arm64.yml", 48 48 + Engine: "microvm", 49 49 + When: when, 50 50 + RunsOn: []string{"linux/arm64", "kvm"}, 51 51 + } 52 52 + 53 53 + c := Compiler{Trigger: trigger} 54 54 + cp := c.Compile([]Workflow{wf}) 55 55 + 56 56 + assert.Len(t, cp.Workflows, 1) 57 57 + assert.Equal(t, []string{"linux/arm64", "kvm"}, cp.Workflows[0].RunsOn) 58 58 + 59 59 + raw, err := json.Marshal(cp.Workflows[0]) 60 60 + assert.NoError(t, err) 61 61 + 62 62 + var persisted map[string]any 63 63 + assert.NoError(t, json.Unmarshal(raw, &persisted)) 64 64 + assert.Equal(t, []any{"linux/arm64", "kvm"}, persisted["runsOn"]) 65 65 + } 66 66 + 67 67 + func TestCompileWorkflow_LegacyWorkflowOmitsRunsOn(t *testing.T) { 68 68 + wf := Workflow{ 69 69 + Name: ".tangled/workflows/legacy.yml", 70 70 + Engine: "microvm", 71 71 + When: when, 72 72 + } 73 73 + 74 74 + c := Compiler{Trigger: trigger} 75 75 + cp := c.Compile([]Workflow{wf}) 76 76 + 77 77 + assert.Len(t, cp.Workflows, 1) 78 78 + assert.Empty(t, cp.Workflows[0].RunsOn) 79 79 + 80 80 + raw, err := json.Marshal(cp.Workflows[0]) 81 81 + assert.NoError(t, err) 82 82 + 83 83 + var persisted map[string]any 84 84 + assert.NoError(t, json.Unmarshal(raw, &persisted)) 85 85 + _, ok := persisted["runsOn"] 86 86 + assert.False(t, ok, "legacy workflow JSON should not gain runsOn") 87 87 + } 88 88 + 44 89 func TestCompileWorkflow_TriggerMismatch(t *testing.T) { 45 90 wf := Workflow{ 46 91 Name: ".tangled/workflows/mismatch.yml",

+14 -7

workflow/def.go

Reviewed

··· 27 27 Workflow struct { 28 28 Name string `yaml:"-"` // name of the workflow file 29 29 Engine string `yaml:"engine"` 30 30 + RunsOn []string `yaml:"runs_on"` 30 31 When []Constraint `yaml:"when"` 31 32 CloneOpts CloneOpts `yaml:"clone"` 32 33 Raw string `yaml:"-"` ··· 121 122 } 122 123 123 124 func (c *Constraint) Match(trigger tangled.Pipeline_TriggerMetadata, changedFiles []string) (bool, error) { 124 124 - match := true 125 125 - 126 125 // manual triggers always pass this constraint 127 126 if trigger.Manual != nil { 128 127 return true, nil 129 128 } 130 129 131 130 // apply event constraints 132 132 - match = match && c.MatchEvent(trigger.Kind) 131 131 + if !c.MatchEvent(trigger.Kind) { 132 132 + return false, nil 133 133 + } 133 134 134 135 // apply branch constraints for PRs 135 136 if trigger.PullRequest != nil { ··· 137 138 if err != nil { 138 139 return false, err 139 140 } 140 140 - match = match && matched 141 141 + if !matched { 142 142 + return false, nil 143 143 + } 141 144 } 142 145 143 146 // apply ref constraints for pushes ··· 146 149 if err != nil { 147 150 return false, err 148 151 } 149 149 - match = match && matched 152 152 + if !matched { 153 153 + return false, nil 154 154 + } 150 155 } 151 156 152 157 // apply paths filter: if specified, at least one changed file must match ··· 155 160 if err != nil { 156 161 return false, err 157 162 } 158 158 - match = match && matched 163 163 + if !matched { 164 164 + return false, nil 165 165 + } 159 166 } 160 167 161 161 - return match, nil 168 168 + return true, nil 162 169 } 163 170 164 171 // matchesAnyFile returns true if any file in files matches any of the glob patterns.

+13

workflow/def_test.go

Reviewed

··· 23 23 assert.False(t, wf.CloneOpts.Skip, "Skip should default to false") 24 24 } 25 25 26 26 + func TestUnmarshalWorkflowWithRunsOnLabels(t *testing.T) { 27 27 + yamlData := ` 28 28 + engine: microvm 29 29 + runs_on: [linux/arm64, kvm] 30 30 + when: 31 31 + - event: push` 32 32 + 33 33 + wf, err := FromFile("test.yml", []byte(yamlData)) 34 34 + assert.NoError(t, err) 35 35 + 36 36 + assert.Equal(t, []string{"linux/arm64", "kvm"}, wf.RunsOn) 37 37 + } 38 38 + 26 39 func TestUnmarshalCloneFalse(t *testing.T) { 27 40 yamlData := ` 28 41 when: