spindle: add artifact store #3

+1 -82

api/tangled/cbor_gen.go

Reviewed

··· 9749 9749 } 9750 9750 9751 9751 cw := cbg.NewCborWriter(w) 9752 9752 - fieldCount := 5 9753 9752 9753 9753 + if _, err := cw.Write([]byte{164}); err != nil { 9754 9754 - if t.RunsOn == nil { 9755 9755 - fieldCount-- 9756 9756 - } 9757 9757 - 9758 9758 - if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { 9759 9754 return err 9760 9755 } 9761 9756 ··· 9843 9838 if _, err := cw.WriteString(string(t.Engine)); err != nil { 9844 9839 return err 9845 9840 } 9846 9846 - 9847 9847 - // t.RunsOn ([]string) (slice) 9848 9848 - if t.RunsOn != nil { 9849 9849 - 9850 9850 - if len("runsOn") > 1000000 { 9851 9851 - return xerrors.Errorf("Value in field \"runsOn\" was too long") 9852 9852 - } 9853 9853 - 9854 9854 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("runsOn"))); err != nil { 9855 9855 - return err 9856 9856 - } 9857 9857 - if _, err := cw.WriteString(string("runsOn")); err != nil { 9858 9858 - return err 9859 9859 - } 9860 9860 - 9861 9861 - if len(t.RunsOn) > 8192 { 9862 9862 - return xerrors.Errorf("Slice value in field t.RunsOn was too long") 9863 9863 - } 9864 9864 - 9865 9865 - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.RunsOn))); err != nil { 9866 9866 - return err 9867 9867 - } 9868 9868 - for _, v := range t.RunsOn { 9869 9869 - if len(v) > 1000000 { 9870 9870 - return xerrors.Errorf("Value in field v was too long") 9871 9871 - } 9872 9872 - 9873 9873 - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { 9874 9874 - return err 9875 9875 - } 9876 9876 - if _, err := cw.WriteString(string(v)); err != nil { 9877 9877 - return err 9878 9878 - } 9879 9879 - 9880 9880 - } 9881 9881 - } 9882 9841 return nil 9883 9842 } 9884 9843 ··· 9976 9935 9977 9936 t.Engine = string(sval) 9978 9937 } 9979 9979 - // t.RunsOn ([]string) (slice) 9980 9980 - case "runsOn": 9981 9981 - 9982 9982 - maj, extra, err = cr.ReadHeader() 9983 9983 - if err != nil { 9984 9984 - return err 9985 9985 - } 9986 9986 - 9987 9987 - if extra > 8192 { 9988 9988 - return fmt.Errorf("t.RunsOn: array too large (%d)", extra) 9989 9989 - } 9990 9990 - 9991 9991 - if maj != cbg.MajArray { 9992 9992 - return fmt.Errorf("expected cbor array") 9993 9993 - } 9994 9994 - 9995 9995 - if extra > 0 { 9996 9996 - t.RunsOn = make([]string, extra) 9997 9997 - } 9998 9998 - 9999 9999 - for i := 0; i < int(extra); i++ { 10000 10000 - { 10001 10001 - var maj byte 10002 10002 - var extra uint64 10003 10003 - var err error 10004 10004 - _ = maj 10005 10005 - _ = extra 10006 10006 - _ = err 10007 10007 - 10008 10008 - { 10009 10009 - sval, err := cbg.ReadStringWithMax(cr, 1000000) 10010 10010 - if err != nil { 10011 10011 - return err 10012 10012 - } 10013 10013 - 10014 10014 - t.RunsOn[i] = string(sval) 10015 10015 - } 10016 10016 - 10017 10017 - } 10018 10018 - } 10019 9938 10020 9939 default: 10021 9940 // Field doesn't exist on this type, so ignore it

-1

api/tangled/tangledpipeline.go

Reviewed

··· 90 90 Engine string `json:"engine" cborgen:"engine"` 91 91 Name string `json:"name" cborgen:"name"` 92 92 Raw string `json:"raw" cborgen:"raw"` 93 93 - RunsOn []string `json:"runsOn,omitempty" cborgen:"runsOn,omitempty"` 94 93 }

-4

buf.gen.yaml

Reviewed

··· 9 9 out: shuttle/src/gen 10 10 opt: 11 11 - bytes=. 12 12 - # the fleet protocol is broker<->executor only (both Go); shuttle (Rust) 13 13 - # only speaks the agent protocol, so keep fleet types out of its gen tree. 14 14 - exclude_types: 15 15 - - spindle.mill.v1

-1

buf.yaml

Reviewed

··· 1 1 version: v2 2 2 modules: 3 3 - path: spindle/agentproto 4 4 - - path: spindle/mill/proto 5 4 deps: 6 5 - buf.build/bufbuild/protovalidate

-160

cmd/spindle/main.go

Reviewed

··· 2 2 3 3 import ( 4 4 "context" 5 5 - "fmt" 6 5 "log/slog" 7 6 "os" 8 8 - "strings" 9 9 - "text/tabwriter" 10 10 - "time" 11 7 12 8 "github.com/urfave/cli/v3" 13 9 tlog "tangled.org/core/log" 14 10 "tangled.org/core/spindle" 15 15 - "tangled.org/core/spindle/db" 16 16 - "tangled.org/core/spindle/mill" 17 11 ) 18 12 19 13 func main() { ··· 22 16 Usage: "spindle continuous integration runner", 23 17 Commands: []*cli.Command{ 24 18 Command(), 25 25 - millCommand(), 26 19 }, 27 20 DefaultCommand: "run", 28 21 } ··· 48 41 }, 49 42 } 50 43 } 51 51 - 52 52 - // for mill host administration and executor management 53 53 - func millCommand() *cli.Command { 54 54 - dbFlag := &cli.StringFlag{ 55 55 - Name: "db", 56 56 - Usage: "path to the spindle sqlite db", 57 57 - Value: "spindle.db", 58 58 - Sources: cli.EnvVars("SPINDLE_SERVER_DB_PATH"), 59 59 - } 60 60 - openDB := func(ctx context.Context, cmd *cli.Command) (*db.DB, error) { 61 61 - return db.Make(ctx, cmd.String("db")) 62 62 - } 63 63 - return &cli.Command{ 64 64 - Name: "mill", 65 65 - Usage: "mill host administration", 66 66 - Commands: []*cli.Command{ 67 67 - { 68 68 - Name: "executor", 69 69 - Usage: "manage executors allowed to join this mill", 70 70 - Commands: []*cli.Command{ 71 71 - { 72 72 - Name: "add", 73 73 - Usage: "register an executor and print its token", 74 74 - ArgsUsage: "<name>", 75 75 - Flags: []cli.Flag{ 76 76 - dbFlag, 77 77 - &cli.DurationFlag{ 78 78 - Name: "ttl", 79 79 - Usage: "token lifetime (e.g. 720h); omit for no expiry", 80 80 - }, 81 81 - &cli.StringSliceFlag{ 82 82 - Name: "label", 83 83 - Usage: "authorized labels for this executor", 84 84 - }, 85 85 - }, 86 86 - Action: func(ctx context.Context, cmd *cli.Command) error { 87 87 - name := cmd.Args().First() 88 88 - if name == "" { 89 89 - return fmt.Errorf("usage: spindle mill executor add <name>") 90 90 - } 91 91 - d, err := openDB(ctx, cmd) 92 92 - if err != nil { 93 93 - return err 94 94 - } 95 95 - token, err := mill.GenerateToken() 96 96 - if err != nil { 97 97 - return err 98 98 - } 99 99 - var expiresAt *time.Time 100 100 - if ttl := cmd.Duration("ttl"); ttl > 0 { 101 101 - exp := time.Now().UTC().Add(ttl) 102 102 - expiresAt = &exp 103 103 - } 104 104 - labels := cmd.StringSlice("label") 105 105 - if err := d.AddExecutorToken(name, mill.HashToken(token), expiresAt, labels); err != nil { 106 106 - return fmt.Errorf("registering executor %q: %w", name, err) 107 107 - } 108 108 - fmt.Println(token) 109 109 - return nil 110 110 - }, 111 111 - }, 112 112 - { 113 113 - Name: "list", 114 114 - Usage: "list registered executors", 115 115 - Flags: []cli.Flag{dbFlag}, 116 116 - Action: func(ctx context.Context, cmd *cli.Command) error { 117 117 - d, err := openDB(ctx, cmd) 118 118 - if err != nil { 119 119 - return err 120 120 - } 121 121 - tokens, err := d.ListExecutorTokens() 122 122 - if err != nil { 123 123 - return err 124 124 - } 125 125 - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 126 126 - fmt.Fprintln(w, "NAME\tCREATED\tEXPIRES\tLABELS\tQUARANTINE") 127 127 - for _, t := range tokens { 128 128 - expires := "never" 129 129 - if t.ExpiresAt != nil { 130 130 - expires = t.ExpiresAt.Format(time.RFC3339) 131 131 - if time.Now().After(*t.ExpiresAt) { 132 132 - expires += " (expired)" 133 133 - } 134 134 - } 135 135 - labels := strings.Join(t.Labels, ",") 136 136 - if labels == "" { 137 137 - labels = "-" 138 138 - } 139 139 - quarantine := "-" 140 140 - if t.QuarantineReason != nil { 141 141 - quarantine = *t.QuarantineReason 142 142 - if t.QuarantinedAt != nil { 143 143 - quarantine = *t.QuarantinedAt + ": " + quarantine 144 144 - } 145 145 - } 146 146 - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", t.Name, t.CreatedAt, expires, labels, quarantine) 147 147 - } 148 148 - return w.Flush() 149 149 - }, 150 150 - }, 151 151 - { 152 152 - Name: "unquarantine", 153 153 - Usage: "allow a quarantined executor to reconnect", 154 154 - ArgsUsage: "<name>", 155 155 - Flags: []cli.Flag{dbFlag}, 156 156 - Action: func(ctx context.Context, cmd *cli.Command) error { 157 157 - name := cmd.Args().First() 158 158 - if name == "" { 159 159 - return fmt.Errorf("usage: spindle mill executor unquarantine <name>") 160 160 - } 161 161 - d, err := openDB(ctx, cmd) 162 162 - if err != nil { 163 163 - return err 164 164 - } 165 165 - ok, err := d.ClearExecutorQuarantine(name) 166 166 - if err != nil { 167 167 - return err 168 168 - } 169 169 - if !ok { 170 170 - return fmt.Errorf("no such executor identity %q", name) 171 171 - } 172 172 - return nil 173 173 - }, 174 174 - }, 175 175 - { 176 176 - Name: "revoke", 177 177 - Usage: "revoke an executor's token", 178 178 - ArgsUsage: "<name>", 179 179 - Flags: []cli.Flag{dbFlag}, 180 180 - Action: func(ctx context.Context, cmd *cli.Command) error { 181 181 - name := cmd.Args().First() 182 182 - if name == "" { 183 183 - return fmt.Errorf("usage: spindle mill executor revoke <name>") 184 184 - } 185 185 - d, err := openDB(ctx, cmd) 186 186 - if err != nil { 187 187 - return err 188 188 - } 189 189 - ok, err := d.RevokeExecutorToken(name) 190 190 - if err != nil { 191 191 - return err 192 192 - } 193 193 - if !ok { 194 194 - return fmt.Errorf("no such executor identity %q", name) 195 195 - } 196 196 - return nil 197 197 - }, 198 198 - }, 199 199 - }, 200 200 - }, 201 201 - }, 202 202 - } 203 203 - }

-157

docker-compose.mill.yml

Reviewed

··· 1 1 - # turns the primary spindle into a mill host and runs a three-executor fleet 2 2 - # against it. the executors differ on two axes so placement is testable: 3 3 - # 4 4 - # executor labels images can run 5 5 - # --------- ------------- ----------- ------------------------ 6 6 - # executor-a linux, fast full image/alpine, image/nixos 7 7 - # executor-b linux, slow full image/alpine, image/nixos 8 8 - # executor-c linux, gpu alpine only image/alpine 9 9 - 10 10 - x-mill-executor: &mill-executor 11 11 - profiles: ["linux"] 12 12 - build: 13 13 - context: . 14 14 - dockerfile: localinfra/spindle.Dockerfile 15 15 - restart: unless-stopped 16 16 - environment: &mill-executor-env 17 17 - SPINDLE_ROLE: executor 18 18 - SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555 19 19 - SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db 20 20 - SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev 21 21 - SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe 22 22 - SPINDLE_SERVER_DEV: "true" 23 23 - SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev 24 24 - SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db 25 25 - SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev 26 26 - SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images 27 27 - SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays 28 28 - SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts 29 29 - SPINDLE_MILL_ARTIFACT_STORE: disk 30 30 - SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" 31 31 - SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 32 32 - SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= 33 33 - SPINDLE_NIX_CACHE_UPLOAD_URL: http://ncps:8501/upload 34 34 - # dials the mill container directly using ws 35 35 - SPINDLE_MILL_URL: ws://spindle:6555/mill 36 36 - devices: 37 37 - - /dev/vsock:/dev/vsock 38 38 - - /dev/kvm:/dev/kvm 39 39 - - /dev/vhost-vsock:/dev/vhost-vsock 40 40 - - /dev/net/tun:/dev/net/tun 41 41 - cap_add: 42 42 - - NET_ADMIN 43 43 - - SYS_ADMIN 44 44 - security_opt: 45 45 - - label=disable 46 46 - - seccomp=unconfined 47 47 - healthcheck: 48 48 - test: ["CMD", "wget", "-qO-", "http://localhost:6555/"] 49 49 - interval: 2s 50 50 - timeout: 2s 51 51 - retries: 30 52 52 - start_period: 5s 53 53 - depends_on: &mill-executor-deps 54 54 - plc: 55 55 - condition: service_started 56 56 - jetstream: 57 57 - condition: service_started 58 58 - init-accounts: 59 59 - condition: service_completed_successfully 60 60 - ncps: 61 61 - condition: service_started 62 62 - spindle: 63 63 - condition: service_healthy 64 64 - mill-tokens: 65 65 - condition: service_completed_successfully 66 66 - networks: [tngl] 67 67 - 68 68 - services: 69 69 - # configures the primary spindle container as a mill host 70 70 - spindle: 71 71 - environment: 72 72 - SPINDLE_ROLE: mill 73 73 - SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts 74 74 - SPINDLE_MILL_ARTIFACT_STORE: disk 75 75 - volumes: 76 76 - - spindle-artifacts:/var/lib/spindle/artifacts 77 77 - 78 78 - mill-tokens: 79 79 - profiles: ["linux"] 80 80 - build: 81 81 - context: . 82 82 - dockerfile: localinfra/spindle.Dockerfile 83 83 - restart: "no" 84 84 - entrypoint: ["/bin/sh", "-c"] 85 85 - environment: 86 86 - SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db 87 87 - command: 88 88 - - | 89 89 - set -eu 90 90 - seed() { 91 91 - spindle mill executor revoke "$$1" >/dev/null 2>&1 || true 92 92 - args="" 93 93 - for l in $$(echo "$$2" | tr ',' ' '); do args="$$args --label $$l"; done 94 94 - spindle mill executor add "$$1" $$args > "/shared/$$1.mill-token" 95 95 - } 96 96 - seed executor-a linux,fast 97 97 - seed executor-b linux,slow 98 98 - seed executor-c linux,gpu 99 99 - echo "seeded executor tokens" 100 100 - volumes: 101 101 - - spindle-data:/var/lib/spindle 102 102 - - init-state:/shared 103 103 - depends_on: 104 104 - spindle: 105 105 - condition: service_healthy 106 106 - networks: [tngl] 107 107 - 108 108 - spindle-executor-a: 109 109 - <<: *mill-executor 110 110 - environment: 111 111 - <<: *mill-executor-env 112 112 - SPINDLE_SERVER_HOSTNAME: executor-a.tngl.boltless.dev 113 113 - SPINDLE_MILL_LABELS: linux,fast 114 114 - SPINDLE_MILL_TOKEN_FILE: /shared/executor-a.mill-token 115 115 - SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11241" 116 116 - volumes: 117 117 - - spindle-executor-a-data:/var/lib/spindle 118 118 - - spindle-artifacts:/var/lib/spindle/artifacts 119 119 - - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro 120 120 - - init-state:/shared:ro 121 121 - - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro 122 122 - 123 123 - spindle-executor-b: 124 124 - <<: *mill-executor 125 125 - environment: 126 126 - <<: *mill-executor-env 127 127 - SPINDLE_SERVER_HOSTNAME: executor-b.tngl.boltless.dev 128 128 - SPINDLE_MILL_LABELS: linux,slow 129 129 - SPINDLE_MILL_TOKEN_FILE: /shared/executor-b.mill-token 130 130 - SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11242" 131 131 - volumes: 132 132 - - spindle-executor-b-data:/var/lib/spindle 133 133 - - spindle-artifacts:/var/lib/spindle/artifacts 134 134 - - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro 135 135 - - init-state:/shared:ro 136 136 - - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro 137 137 - 138 138 - spindle-executor-c: 139 139 - <<: *mill-executor 140 140 - environment: 141 141 - <<: *mill-executor-env 142 142 - SPINDLE_SERVER_HOSTNAME: executor-c.tngl.boltless.dev 143 143 - SPINDLE_MILL_LABELS: linux,gpu 144 144 - SPINDLE_MILL_TOKEN_FILE: /shared/executor-c.mill-token 145 145 - SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11243" 146 146 - volumes: 147 147 - - spindle-executor-c-data:/var/lib/spindle 148 148 - - spindle-artifacts:/var/lib/spindle/artifacts 149 149 - - ./out/localinfra-spindle-images-alpine:/var/lib/spindle/images:ro 150 150 - - init-state:/shared:ro 151 151 - - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro 152 152 - 153 153 - volumes: 154 154 - spindle-executor-a-data: 155 155 - spindle-executor-b-data: 156 156 - spindle-executor-c-data: 157 157 - spindle-artifacts:

docker-compose.yml

Reviewed

This file has not been changed.

-49

eventstream/eventstream_test.go

Reviewed

··· 293 293 } 294 294 } 295 295 296 296 - func TestHighWaterSeedsClockFromStoredEvents(t *testing.T) { 297 297 - db, err := sql.Open("sqlite3", t.TempDir()+"/events.db") 298 298 - if err != nil { 299 299 - t.Fatalf("open: %v", err) 300 300 - } 301 301 - t.Cleanup(func() { db.Close() }) 302 302 - if _, err := db.Exec(`create table events ( 303 303 - rkey text not null, 304 304 - nsid text not null, 305 305 - event text not null, 306 306 - created integer not null, 307 307 - primary key (rkey, nsid) 308 308 - )`); err != nil { 309 309 - t.Fatalf("schema: %v", err) 310 310 - } 311 311 - 312 312 - stored := time.Now().Add(time.Hour).UnixNano() 313 313 - if _, err := db.Exec( 314 314 - `insert into events (rkey, nsid, event, created) values (?, ?, ?, ?)`, 315 315 - "stored", "sh.tangled.test", "{}", stored, 316 316 - ); err != nil { 317 317 - t.Fatalf("seed event: %v", err) 318 318 - } 319 319 - 320 320 - cut, err := HighWater(db) 321 321 - if err != nil { 322 322 - t.Fatalf("HighWater() error = %v", err) 323 323 - } 324 324 - if cut < stored { 325 325 - t.Fatalf("HighWater() = %d, want at least stored cursor %d", cut, stored) 326 326 - } 327 327 - 328 328 - n := notifier.New() 329 329 - if err := Insert(db, Event{ 330 330 - Rkey: "new", 331 331 - Nsid: "sh.tangled.test", 332 332 - EventJson: json.RawMessage("{}"), 333 333 - }, &n); err != nil { 334 334 - t.Fatalf("Insert() error = %v", err) 335 335 - } 336 336 - events, err := List(db, cut, 10) 337 337 - if err != nil { 338 338 - t.Fatalf("List() error = %v", err) 339 339 - } 340 340 - if len(events) != 1 || events[0].Rkey != "new" || events[0].Created <= cut { 341 341 - t.Fatalf("events after cut = %+v, want only new event above %d", events, cut) 342 342 - } 343 343 - } 344 344 - 345 296 func isCloseErr(err error) bool { 346 297 if err == nil { 347 298 return false

-29

eventstream/store.go

Reviewed

··· 19 19 lastNanos int64 20 20 ) 21 21 22 22 - func HighWater(s Store) (int64, error) { 23 23 - clockMu.Lock() 24 24 - defer clockMu.Unlock() 25 25 - 26 26 - rows, err := s.Query(`select coalesce(max(created), 0) from events`) 27 27 - if err != nil { 28 28 - return 0, err 29 29 - } 30 30 - defer rows.Close() 31 31 - 32 32 - var created int64 33 33 - if !rows.Next() { 34 34 - if err := rows.Err(); err != nil { 35 35 - return 0, err 36 36 - } 37 37 - return 0, sql.ErrNoRows 38 38 - } 39 39 - if err := rows.Scan(&created); err != nil { 40 40 - return 0, err 41 41 - } 42 42 - if err := rows.Err(); err != nil { 43 43 - return 0, err 44 44 - } 45 45 - if created > lastNanos { 46 46 - lastNanos = created 47 47 - } 48 48 - return lastNanos, nil 49 49 - } 50 50 - 51 22 func Insert(s Store, ev Event, n *notifier.Notifier) error { 52 23 clockMu.Lock() 53 24 defer clockMu.Unlock()

-6

lexicons/pipeline/pipeline.json

Reviewed

··· 184 184 "engine": { 185 185 "type": "string" 186 186 }, 187 187 - "runsOn": { 188 188 - "type": "array", 189 189 - "items": { 190 190 - "type": "string" 191 191 - } 192 192 - }, 193 187 "clone": { 194 188 "type": "ref", 195 189 "ref": "#cloneOpts"

-20

localinfra/readme.md

Reviewed

··· 51 51 This writes the image directory under `out/localinfra-spindle-images`. 52 52 5. `docker compose up` 53 53 6. AppView will be running on `127.0.0.1:3000` with two test users: `alice.pds.tngl.boltless.dev` and `bob.pds.tngl.boltless.dev`. Both with password `password`. 54 54 - 55 55 - ## Mill mode 56 56 - 57 57 - The default stack runs one standalone spindle. `docker-compose.mill.yml` is an overlay that splits that into the distributed arch: the primary spindle becomes the mill host (`role=mill`, it only places jobs) and a three-executor fleet (`role=executor`) runs the engines. 58 58 - 59 59 - ```bash 60 60 - docker compose -f docker-compose.yml -f docker-compose.mill.yml --profile linux up 61 61 - ``` 62 62 - 63 63 - The executors differ on the two axes placement cares about, so you can watch candidates get filtered and ranked: 64 64 - 65 65 - | executor | labels | images | runs | 66 66 - |------------|---------------|-------------|-----------------------------| 67 67 - | executor-a | `linux, fast` | full | `image/alpine, image/nixos` | 68 68 - | executor-b | `linux, slow` | full | `image/alpine, image/nixos` | 69 69 - | executor-c | `linux, gpu` | alpine only | `image/alpine` | 70 70 - 71 71 - So `image: nixos` has two candidates, `image: alpine` has three, and `runs_on: [gpu]` pins to executor-c. The alpine-only image set is staged by `prepare-spindle-images.sh` (step 4) alongside the full one, no extra step. 72 72 - 73 73 - Each executor needs its own identity (one live session per token), so the `mill-tokens` service registers a token per executor in the mill db and drops it into the shared volume for the executor to read.

-10

localinfra/scripts/prepare-spindle-images.sh

Reviewed

··· 29 29 extract_image spindle-nixos-image-tarball nixos-x86_64 nixos 30 30 extract_image spindle-alpine-image-tarball alpine-x86_64 alpine 31 31 32 32 - # a reduced image set (alpine only) for the mill overlay: an executor mounting 33 33 - # this advertises image/alpine but not image/nixos, so capability-based 34 34 - # placement is testable across a mixed fleet 35 35 - alpine_only="$repo/out/localinfra-spindle-images-alpine" 36 36 - [ -d "$alpine_only" ] && chmod -R +w "$alpine_only" || true 37 37 - rm -rf "$alpine_only" 38 38 - mkdir -p "$alpine_only" 39 39 - cp -r "$image_root/alpine-x86_64" "$alpine_only/alpine-x86_64" 40 40 - ln -s alpine-x86_64 "$alpine_only/alpine" 41 41 - 42 32 echo "prepared spindle microVM images in $image_root"

-5

localinfra/spindle.Dockerfile

Reviewed

··· 49 49 export SPINDLE_SERVER_OWNER="$(cat /shared/owner-did)" 50 50 : "${SPINDLE_SERVER_OWNER:?set via env or /shared/owner-did}" 51 51 52 52 - # executors read their mill token from a file seeded by the mill-tokens 53 53 - # service (command substitution strips the trailing newline) 54 54 - [ -z "${SPINDLE_MILL_SHARED_SECRET:-}" ] && [ -n "${SPINDLE_MILL_TOKEN_FILE:-}" ] && [ -r "${SPINDLE_MILL_TOKEN_FILE}" ] && \ 55 55 - export SPINDLE_MILL_SHARED_SECRET="$(cat "${SPINDLE_MILL_TOKEN_FILE}")" 56 56 - 57 52 mkdir -p /var/lib/spindle /var/lib/spindle/overlays /var/log/spindle 58 53 59 54 if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then

+3 -3

netutil/ssrf.go

Reviewed

··· 10 10 "github.com/gorilla/websocket" 11 11 ) 12 12 13 13 + // SSRFDialer returns a net.Dialer that refuses non-public IPs. 13 13 - // refuses non-public ips to prevent ssrf 14 14 func SSRFDialer(dev bool) *net.Dialer { 15 15 if dev { 16 16 return &net.Dialer{} ··· 18 18 return ssrf.PublicOnlyDialer() 19 19 } 20 20 21 21 + // SSRFTransport returns an http.Transport that refuses non-public IPs. 21 21 - // refuses non-public ips to prevent ssrf 22 22 func SSRFTransport(dev bool) *http.Transport { 23 23 if dev { 24 24 return &http.Transport{} ··· 26 26 return ssrf.PublicOnlyTransport() 27 27 } 28 28 29 29 + // SSRFWebsocketDialer returns a websocket.Dialer that refuses non-public IPs. 29 29 - // refuses non-public ips to prevent ssrf 30 30 func SSRFWebsocketDialer(dev bool) *websocket.Dialer { 31 31 dialer := *websocket.DefaultDialer 32 32 dialer.NetDialContext = SSRFDialer(dev).DialContext

+1 -1

nix/modules/spindle.nix

Reviewed

··· 156 156 }; 157 157 }; 158 158 159 159 + 159 160 pipelines = { 160 161 workflowTimeout = mkOption { 161 162 type = types.str; ··· 406 407 "SPINDLE_ARTIFACT_STORES_DISK_DIR=${cfg.artifactStores.disk.dir}" 407 408 "SPINDLE_ARTIFACT_STORES_S3_BUCKET=${cfg.artifactStores.s3.bucket}" 408 409 "SPINDLE_ARTIFACT_STORES_S3_REGION=${cfg.artifactStores.s3.region}" 409 409 - "SPINDLE_MILL_ARTIFACT_STORE=s3" 410 410 ]; 411 411 ExecStart = "${cfg.package}/bin/spindle"; 412 412 Restart = "always";

nix/vm.nix

Reviewed

This file has not been changed.

spindle/artifactstore/artifactstore.go

Reviewed

This file has not been changed.

spindle/artifactstore/artifactstore_test.go

Reviewed

This file has not been changed.

+1 -46

spindle/config/config.go

Reviewed

··· 2 2 3 3 import ( 4 4 "context" 5 5 - "fmt" 6 5 "time" 7 6 8 7 "github.com/bluesky-social/indigo/atproto/syntax" ··· 78 77 79 78 type MicroVMPipelines struct { 80 79 ImageDir string `env:"IMAGE_DIR"` 81 81 - OverlayDir string `env:"OVERLAY_DIR"` // where microVM temporary disks will live 80 80 + OverlayDir string `env:"OVERLAY_DIR, default="` // where microVM temporary disks will live 82 81 DefaultImage string `env:"DEFAULT_IMAGE, default=nixos-x86_64"` 83 82 AgentPort uint32 `env:"AGENT_PORT, default=10240"` 84 83 EnableKVM bool `env:"ENABLE_KVM, default=true"` ··· 108 107 UploadURL string `env:"UPLOAD_URL"` 109 108 } 110 109 111 111 - // governs how spindle places and runs jobs 112 112 - type Role string 113 113 - 114 114 - const ( 115 115 - RoleStandalone Role = "standalone" 116 116 - RoleMill Role = "mill" 117 117 - RoleExecutor Role = "executor" 118 118 - ) 119 119 - 120 120 - // fields are selectively active depending on the role 121 121 - type Mill struct { 122 122 - URL string `env:"URL"` // mill websocket endpoint dialled by the executor 123 123 - SharedSecret string `env:"SHARED_SECRET"` // the executor's token for dialing the mill 124 124 - MaxPending int `env:"MAX_PENDING, default=100"` // mill pending job queue limit 125 125 - ReconnectGrace time.Duration `env:"RECONNECT_GRACE, default=45s"` // reconnect window before leases are failed 126 126 - Seats int `env:"SEATS, default=4"` // executor seats advertised to the mill 127 127 - Labels []string `env:"LABELS"` // executor capability labels 128 128 - ArtifactStore string `env:"ARTIFACT_STORE"` // store shared by mill and its executors 129 129 - } 130 130 - 131 110 type Config struct { 132 132 - Role Role `env:"SPINDLE_ROLE, default=standalone"` 133 111 Server Server `env:",prefix=SPINDLE_SERVER_"` 134 112 NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` 135 113 MicroVMPipelines MicroVMPipelines `env:",prefix=SPINDLE_MICROVM_PIPELINES_"` 136 114 NixCache NixCache `env:",prefix=SPINDLE_NIX_CACHE_"` 137 115 ArtifactStores ArtifactStores `env:",prefix=SPINDLE_ARTIFACT_STORES_"` 138 116 LegacyS3 LegacyS3 `env:",prefix=SPINDLE_S3_"` 139 139 - Mill Mill `env:",prefix=SPINDLE_MILL_"` 140 117 } 141 118 142 142 - func (c *Config) validate() error { 143 143 - switch c.Role { 144 144 - case RoleStandalone, RoleMill: 145 145 - if c.Mill.URL != "" { 146 146 - return fmt.Errorf("SPINDLE_MILL_URL is set but SPINDLE_ROLE=%s; only an executor dials a mill", c.Role) 147 147 - } 148 148 - case RoleExecutor: 149 149 - if c.Mill.URL == "" { 150 150 - return fmt.Errorf("SPINDLE_ROLE=executor requires SPINDLE_MILL_URL (the mill to dial)") 151 151 - } 152 152 - if c.Mill.SharedSecret == "" { 153 153 - return fmt.Errorf("SPINDLE_ROLE=executor requires SPINDLE_MILL_SHARED_SECRET (its executor token)") 154 154 - } 155 155 - default: 156 156 - return fmt.Errorf("unknown SPINDLE_ROLE %q (want standalone, mill, or executor)", c.Role) 157 157 - } 158 158 - return nil 159 159 - } 160 160 - 161 119 func Load(ctx context.Context) (*Config, error) { 162 120 163 121 164 122 if err != nil { 165 165 - return nil, err 166 166 - } 167 167 - if err := cfg.validate(); err != nil { 168 123 return nil, err 169 124 } 170 125

+12 -56

spindle/db/db.go

Reviewed

··· 79 79 ); 80 80 81 81 create table if not exists spindle_members ( 82 82 + -- identifiers for the record 82 83 id integer primary key autoincrement, 83 84 did text not null, 84 85 rkey text not null, 86 86 + 87 87 + -- data 85 88 instance text not null, 86 89 subject text not null, 87 90 created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), 91 91 + 92 92 + -- constraints 88 93 unique (did, rkey) 89 94 ); 90 95 96 96 + -- status event for a single workflow 91 97 create table if not exists events ( 92 98 rkey text not null, 93 99 nsid text not null, 94 94 - event text not null, 95 95 - created integer not null 100 100 + event text not null, -- json 101 101 + created integer not null -- unix nanos 96 102 ); 97 103 98 104 create table if not exists nixos_toplevel_cache ( ··· 126 132 foreign key (pipeline_id) references pipelines(id) on delete cascade 127 133 ); 128 134 129 129 - create table if not exists mill_executors ( 130 130 - name text primary key, 131 131 - token_hash text not null unique, 132 132 - created_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), 133 133 - expires_at text, 134 134 - labels text, 135 135 - quarantine_reason text, 136 136 - quarantined_at text 137 137 - ); 138 138 - 139 139 - create table if not exists mill_leases ( 140 140 - lease_id text primary key, 141 141 - node_id text not null, 142 142 - epoch text not null, 143 143 - engine text not null, 144 144 - knot text not null, 145 145 - rkey text not null, 146 146 - workflow text not null, 147 147 - state text not null 148 148 - ); 149 149 - 150 150 - create table if not exists mill_executor_cursors ( 151 151 - node_id text not null, 152 152 - epoch text not null, 153 153 - acked_seqno integer not null, 154 154 - primary key (node_id, epoch) 155 155 - ); 156 156 - 157 157 - create table if not exists mill_outbox_state ( 158 158 - epoch text not null, 159 159 - next_seqno integer not null, 160 160 - primary key (epoch) 161 161 - ); 162 162 - 163 163 - create table if not exists mill_outbox_rows ( 164 164 - epoch text not null, 165 165 - seqno integer not null, 166 166 - payload blob not null, 167 167 - byte_size integer not null, 168 168 - control integer not null, 169 169 - primary key (epoch, seqno), 170 170 - foreign key (epoch) references mill_outbox_state(epoch) on delete cascade 171 171 - ); 172 172 - 173 135 create table if not exists mill_artifacts ( 174 136 id integer primary key autoincrement, 175 137 lease_id text not null, ··· 178 140 hash text not null 179 141 ); 180 142 181 181 - create table if not exists executor_pending_artifacts ( 182 182 - lease_id text primary key, 183 183 - workflow text not null, 184 184 - status text not null, 185 185 - error text not null default '', 186 186 - exit_code integer not null default 0, 187 187 - ref text not null, 188 188 - hash text not null 189 189 - ); 190 190 - 191 143 create table if not exists migrations ( 192 144 id integer primary key autoincrement, 193 145 name text unique 194 146 ); 195 147 `) 148 148 + if err != nil { 149 149 + return nil, err 150 150 + } 151 151 + 196 152 if err := runMigrations(ctx, conn, logger); err != nil { 197 153 return nil, err 198 154 }

+15 -59

spindle/db/events.go

Reviewed

··· 19 19 return eventstream.List(d, cursor, limit) 20 20 } 21 21 22 22 - func (d *DB) EventHighWater() (int64, error) { 23 23 - return eventstream.HighWater(d) 24 24 - } 25 25 - 26 22 func (d *DB) CreatePipelineEvent(rkey string, pipeline tangled.Pipeline, n *notifier.Notifier) error { 27 23 eventJson, err := json.Marshal(pipeline) 28 24 if err != nil { ··· 36 32 return d.insertEvent(event, n) 37 33 } 38 34 35 35 + func (d *DB) createStatusEvent( 36 36 + workflowId models.WorkflowId, 37 37 + statusKind models.StatusKind, 38 38 + workflowError *string, 39 39 + exitCode *int64, 40 40 + n *notifier.Notifier, 41 41 + ) error { 42 42 + now := time.Now() 43 43 + pipelineAtUri := workflowId.PipelineId.AtUri() 39 39 - // leaves created at zero so insertevent stamps the local clock 40 40 - func statusEvent(pipelineAtUri, workflow, status string, workflowError *string, exitCode *int64) (eventstream.Event, error) { 41 44 s := tangled.PipelineStatus{ 45 45 + CreatedAt: now.Format(time.RFC3339), 42 42 - CreatedAt: time.Now().Format(time.RFC3339), 43 46 Error: workflowError, 44 47 ExitCode: exitCode, 48 48 + Pipeline: string(pipelineAtUri), 49 49 + Workflow: workflowId.Name, 50 50 + Status: string(statusKind), 45 45 - Pipeline: pipelineAtUri, 46 46 - Workflow: workflow, 47 47 - Status: status, 48 51 } 49 52 50 53 eventJson, err := json.Marshal(s) 51 54 if err != nil { 55 55 + return err 52 52 - return eventstream.Event{}, err 53 56 } 54 57 58 58 + event := eventstream.Event{ 55 55 - return eventstream.Event{ 56 59 Rkey: tid.TID(), 57 60 Nsid: tangled.PipelineStatusNSID, 58 61 EventJson: eventJson, 59 59 - }, nil 60 60 - } 61 61 - 62 62 - func (d *DB) createStatusEvent( 63 63 - workflowId models.WorkflowId, 64 64 - statusKind models.StatusKind, 65 65 - workflowError *string, 66 66 - exitCode *int64, 67 67 - n *notifier.Notifier, 68 68 - ) error { 69 69 - event, err := statusEvent(string(workflowId.PipelineId.AtUri()), workflowId.Name, string(statusKind), workflowError, exitCode) 70 70 - if err != nil { 71 71 - return err 72 62 } 73 73 - return d.insertEvent(event, n) 74 74 - } 75 63 76 76 - // stamps the mill's own clock so it orders against the cursor like a local write 77 77 - func (d *DB) InsertEventStatus( 78 78 - pipelineAtUri string, 79 79 - workflow string, 80 80 - status string, 81 81 - workflowError *string, 82 82 - exitCode *int64, 83 83 - n *notifier.Notifier, 84 84 - ) error { 85 85 - event, err := statusEvent(pipelineAtUri, workflow, status, workflowError, exitCode) 86 86 - if err != nil { 87 87 - return err 88 88 - } 89 64 return d.insertEvent(event, n) 90 65 } 91 66 92 92 - // deleting the lease in the same transaction prevents the terminal event 93 93 - // from replaying 94 94 - func (d *DB) CompleteMillLease( 95 95 - leaseID string, 96 96 - pipelineAtUri string, 97 97 - workflow string, 98 98 - status string, 99 99 - workflowError *string, 100 100 - exitCode *int64, 101 101 - n *notifier.Notifier, 102 102 - ) error { 103 103 - return d.ApplyEventBatch(n, func(tx *EventBatchTx) error { 104 104 - if err := tx.InsertStatusEvent(pipelineAtUri, workflow, status, workflowError, exitCode); err != nil { 105 105 - return err 106 106 - } 107 107 - return tx.DeleteLease(leaseID) 108 108 - }) 109 109 - } 110 110 - 111 67 func (d *DB) GetStatus(workflowId models.WorkflowId) (*tangled.PipelineStatus, error) { 112 68 pipelineAtUri := workflowId.PipelineId.AtUri() 113 69

-392

spindle/db/mill_state.go

Reviewed

··· 1 1 - package db 2 2 - 3 3 - import ( 4 4 - "database/sql" 5 5 - "fmt" 6 6 - 7 7 - "tangled.org/core/eventstream" 8 8 - "tangled.org/core/notifier" 9 9 - ) 10 10 - 11 11 - // enough to rebuild the fencing token and workflow identity after a restart 12 12 - type MillLease struct { 13 13 - LeaseID string 14 14 - NodeID string 15 15 - Epoch string 16 16 - Engine string 17 17 - Knot string 18 18 - Rkey string 19 19 - Workflow string 20 20 - State string 21 21 - } 22 22 - 23 23 - type ExecutorCursor struct { 24 24 - NodeID string 25 25 - Epoch string 26 26 - AckedSeqno uint64 27 27 - } 28 28 - 29 29 - type OutboxRow struct { 30 30 - Epoch string 31 31 - Seqno uint64 32 32 - Payload []byte 33 33 - ByteSize int64 34 34 - Control bool 35 35 - } 36 36 - type OutboxDeletion struct { 37 37 - Rows int64 38 38 - Bytes int64 39 39 - } 40 40 - 41 41 - func (d *DB) SaveMillLease(l MillLease) error { 42 42 - _, err := d.Exec( 43 43 - `insert into mill_leases ( 44 44 - lease_id, node_id, epoch, engine, knot, rkey, workflow, state 45 45 - ) values (?, ?, ?, ?, ?, ?, ?, ?) 46 46 - on conflict(lease_id) do update set state = excluded.state`, 47 47 - l.LeaseID, l.NodeID, l.Epoch, l.Engine, l.Knot, l.Rkey, l.Workflow, l.State, 48 48 - ) 49 49 - return err 50 50 - } 51 51 - 52 52 - func (d *DB) DeleteMillLease(leaseID string) error { 53 53 - _, err := d.Exec(`delete from mill_leases where lease_id = ?`, leaseID) 54 54 - return err 55 55 - } 56 56 - 57 57 - func (d *DB) ListMillLeases() ([]MillLease, error) { 58 58 - rows, err := d.Query(` 59 59 - select lease_id, node_id, epoch, engine, knot, rkey, workflow, state 60 60 - from mill_leases 61 61 - `) 62 62 - if err != nil { 63 63 - return nil, err 64 64 - } 65 65 - defer rows.Close() 66 66 - 67 67 - var leases []MillLease 68 68 - for rows.Next() { 69 69 - var l MillLease 70 70 - if err := rows.Scan( 71 71 - &l.LeaseID, &l.NodeID, &l.Epoch, &l.Engine, &l.Knot, &l.Rkey, &l.Workflow, &l.State, 72 72 - ); err != nil { 73 73 - return nil, err 74 74 - } 75 75 - leases = append(leases, l) 76 76 - } 77 77 - return leases, rows.Err() 78 78 - } 79 79 - 80 80 - func (d *DB) ListExecutorCursors() ([]ExecutorCursor, error) { 81 81 - rows, err := d.Query(`select node_id, epoch, acked_seqno from mill_executor_cursors`) 82 82 - if err != nil { 83 83 - return nil, err 84 84 - } 85 85 - defer rows.Close() 86 86 - 87 87 - var cursors []ExecutorCursor 88 88 - for rows.Next() { 89 89 - var c ExecutorCursor 90 90 - if err := rows.Scan(&c.NodeID, &c.Epoch, &c.AckedSeqno); err != nil { 91 91 - return nil, err 92 92 - } 93 93 - cursors = append(cursors, c) 94 94 - } 95 95 - return cursors, rows.Err() 96 96 - } 97 97 - 98 98 - func (d *DB) SetOutboxEpoch(epoch string) error { 99 99 - tx, err := d.Begin() 100 100 - if err != nil { 101 101 - return err 102 102 - } 103 103 - defer tx.Rollback() 104 104 - 105 105 - if _, err := tx.Exec(`delete from mill_outbox_rows`); err != nil { 106 106 - return err 107 107 - } 108 108 - if _, err := tx.Exec(`delete from mill_outbox_state`); err != nil { 109 109 - return err 110 110 - } 111 111 - if _, err := tx.Exec(`insert into mill_outbox_state (epoch, next_seqno) values (?, 1)`, epoch); err != nil { 112 112 - return err 113 113 - } 114 114 - return tx.Commit() 115 115 - } 116 116 - 117 117 - func (d *DB) AppendOutboxRow(payload []byte, control bool) (uint64, error) { 118 118 - tx, err := d.Begin() 119 119 - if err != nil { 120 120 - return 0, err 121 121 - } 122 122 - defer tx.Rollback() 123 123 - 124 124 - var epoch string 125 125 - var nextSeqno uint64 126 126 - err = tx.QueryRow(`select epoch, next_seqno from mill_outbox_state limit 1`).Scan(&epoch, &nextSeqno) 127 127 - if err == sql.ErrNoRows { 128 128 - return 0, fmt.Errorf("no outbox epoch set") 129 129 - } else if err != nil { 130 130 - return 0, err 131 131 - } 132 132 - 133 133 - byteSize := int64(len(payload)) 134 134 - controlVal := 0 135 135 - if control { 136 136 - controlVal = 1 137 137 - } 138 138 - 139 139 - if _, err := tx.Exec( 140 140 - `insert into mill_outbox_rows (epoch, seqno, payload, byte_size, control) values (?, ?, ?, ?, ?)`, 141 141 - epoch, nextSeqno, payload, byteSize, controlVal, 142 142 - ); err != nil { 143 143 - return 0, err 144 144 - } 145 145 - 146 146 - if _, err := tx.Exec( 147 147 - `update mill_outbox_state set next_seqno = ? where epoch = ?`, 148 148 - nextSeqno+1, epoch, 149 149 - ); err != nil { 150 150 - return 0, err 151 151 - } 152 152 - 153 153 - if err := tx.Commit(); err != nil { 154 154 - return 0, err 155 155 - } 156 156 - return nextSeqno, nil 157 157 - } 158 158 - 159 159 - func (d *DB) DeleteOutboxPrefix(ackedSeqno uint64) (OutboxDeletion, error) { 160 160 - tx, err := d.Begin() 161 161 - if err != nil { 162 162 - return OutboxDeletion{}, err 163 163 - } 164 164 - defer tx.Rollback() 165 165 - 166 166 - var epoch string 167 167 - err = tx.QueryRow(`select epoch from mill_outbox_state limit 1`).Scan(&epoch) 168 168 - if err == sql.ErrNoRows { 169 169 - return OutboxDeletion{}, nil 170 170 - } 171 171 - if err != nil { 172 172 - return OutboxDeletion{}, err 173 173 - } 174 174 - 175 175 - var deleted OutboxDeletion 176 176 - if err := tx.QueryRow(` 177 177 - select count(*), 178 178 - coalesce(sum(byte_size), 0) 179 179 - from mill_outbox_rows 180 180 - where epoch = ? and seqno <= ? 181 181 - `, epoch, ackedSeqno).Scan(&deleted.Rows, &deleted.Bytes); err != nil { 182 182 - return OutboxDeletion{}, err 183 183 - } 184 184 - if _, err := tx.Exec( 185 185 - `delete from mill_outbox_rows where epoch = ? and seqno <= ?`, 186 186 - epoch, ackedSeqno, 187 187 - ); err != nil { 188 188 - return OutboxDeletion{}, err 189 189 - } 190 190 - if err := tx.Commit(); err != nil { 191 191 - return OutboxDeletion{}, err 192 192 - } 193 193 - return deleted, nil 194 194 - } 195 195 - 196 196 - func (d *DB) ListOutboxRows() ([]OutboxRow, error) { 197 197 - rows, err := d.Query(`select epoch, seqno, payload, byte_size, control from mill_outbox_rows order by seqno`) 198 198 - if err != nil { 199 199 - return nil, err 200 200 - } 201 201 - defer rows.Close() 202 202 - 203 203 - var out []OutboxRow 204 204 - for rows.Next() { 205 205 - var r OutboxRow 206 206 - var controlVal int 207 207 - if err := rows.Scan(&r.Epoch, &r.Seqno, &r.Payload, &r.ByteSize, &controlVal); err != nil { 208 208 - return nil, err 209 209 - } 210 210 - r.Control = (controlVal != 0) 211 211 - out = append(out, r) 212 212 - } 213 213 - return out, rows.Err() 214 214 - } 215 215 - func (d *DB) ListOutboxRowsAfter(seqno uint64, limit int) ([]OutboxRow, error) { 216 216 - rows, err := d.Query(` 217 217 - select epoch, seqno, payload, byte_size, control 218 218 - from mill_outbox_rows 219 219 - where epoch = (select epoch from mill_outbox_state limit 1) 220 220 - and seqno > ? 221 221 - order by seqno 222 222 - limit ? 223 223 - `, seqno, limit) 224 224 - if err != nil { 225 225 - return nil, err 226 226 - } 227 227 - defer rows.Close() 228 228 - 229 229 - var out []OutboxRow 230 230 - for rows.Next() { 231 231 - var row OutboxRow 232 232 - var control int 233 233 - if err := rows.Scan(&row.Epoch, &row.Seqno, &row.Payload, &row.ByteSize, &control); err != nil { 234 234 - return nil, err 235 235 - } 236 236 - row.Control = control != 0 237 237 - out = append(out, row) 238 238 - } 239 239 - return out, rows.Err() 240 240 - } 241 241 - 242 242 - func (d *DB) GetOutboxState() (string, uint64, error) { 243 243 - var epoch string 244 244 - var nextSeqno uint64 245 245 - err := d.QueryRow(`select epoch, next_seqno from mill_outbox_state limit 1`).Scan(&epoch, &nextSeqno) 246 246 - if err == sql.ErrNoRows { 247 247 - return "", 0, nil 248 248 - } 249 249 - return epoch, nextSeqno, err 250 250 - } 251 251 - 252 252 - type FinishedLog struct { 253 253 - LeaseID string 254 254 - Workflow string 255 255 - Ref string 256 256 - Hash string 257 257 - } 258 258 - 259 259 - func (d *DB) GetFinishedLog(workflow string) (*FinishedLog, error) { 260 260 - var fl FinishedLog 261 261 - err := d.QueryRow( 262 262 - `select lease_id, workflow, ref, hash 263 263 - from mill_artifacts 264 264 - where workflow = ? 265 265 - order by id desc limit 1`, 266 266 - workflow, 267 267 - ).Scan(&fl.LeaseID, &fl.Workflow, &fl.Ref, &fl.Hash) 268 268 - if err != nil { 269 269 - return nil, err 270 270 - } 271 271 - return &fl, nil 272 272 - } 273 273 - 274 274 - type EventBatchTx struct { 275 275 - tx *sql.Tx 276 276 - db *DB 277 277 - } 278 278 - 279 279 - func (tx *EventBatchTx) InsertStatusEvent(pipelineAtUri, workflow, status string, workflowError *string, exitCode *int64) error { 280 280 - event, err := statusEvent(pipelineAtUri, workflow, status, workflowError, exitCode) 281 281 - if err != nil { 282 282 - return err 283 283 - } 284 284 - return eventstream.Insert(tx.tx, event, nil) 285 285 - } 286 286 - 287 287 - func (tx *EventBatchTx) DeleteLease(leaseID string) error { 288 288 - _, err := tx.tx.Exec(`delete from mill_leases where lease_id = ?`, leaseID) 289 289 - return err 290 290 - } 291 291 - 292 292 - func (tx *EventBatchTx) AdvanceCursor(nodeID, epoch string, seqno uint64) error { 293 293 - _, err := tx.tx.Exec( 294 294 - `insert into mill_executor_cursors (node_id, epoch, acked_seqno) values (?, ?, ?) 295 295 - on conflict(node_id, epoch) do update set acked_seqno = excluded.acked_seqno`, 296 296 - nodeID, epoch, seqno, 297 297 - ) 298 298 - return err 299 299 - } 300 300 - 301 301 - func (tx *EventBatchTx) InsertArtifactRef(leaseID, workflow, ref, hash string) error { 302 302 - _, err := tx.tx.Exec( 303 303 - `insert into mill_artifacts (lease_id, workflow, ref, hash) 304 304 - values (?, ?, ?, ?)`, 305 305 - leaseID, workflow, ref, hash, 306 306 - ) 307 307 - return err 308 308 - } 309 309 - 310 310 - func (d *DB) SaveArtifactRef(leaseID, workflow, ref, hash string) error { 311 311 - _, err := d.Exec( 312 312 - `insert into mill_artifacts (lease_id, workflow, ref, hash) 313 313 - values (?, ?, ?, ?)`, 314 314 - leaseID, workflow, ref, hash, 315 315 - ) 316 316 - return err 317 317 - } 318 318 - 319 319 - type PendingArtifact struct { 320 320 - LeaseID string 321 321 - Workflow string 322 322 - Status string 323 323 - Error string 324 324 - ExitCode int64 325 325 - Ref string 326 326 - Hash string 327 327 - } 328 328 - 329 329 - func (d *DB) SavePendingArtifact(leaseID, workflow, status, errStr string, exitCode int64, ref, hash string) error { 330 330 - _, err := d.Exec( 331 331 - `insert into executor_pending_artifacts (lease_id, workflow, status, error, exit_code, ref, hash) 332 332 - values (?, ?, ?, ?, ?, ?, ?) 333 333 - on conflict(lease_id) do update set 334 334 - workflow = excluded.workflow, 335 335 - status = excluded.status, 336 336 - error = excluded.error, 337 337 - exit_code = excluded.exit_code, 338 338 - ref = excluded.ref, 339 339 - hash = excluded.hash`, 340 340 - leaseID, workflow, status, errStr, exitCode, ref, hash, 341 341 - ) 342 342 - return err 343 343 - } 344 344 - 345 345 - func (d *DB) RemovePendingArtifact(leaseID string) error { 346 346 - _, err := d.Exec(`delete from executor_pending_artifacts where lease_id = ?`, leaseID) 347 347 - return err 348 348 - } 349 349 - 350 350 - func (d *DB) ListPendingArtifacts() ([]PendingArtifact, error) { 351 351 - rows, err := d.Query(`select lease_id, workflow, status, error, exit_code, ref, hash from executor_pending_artifacts`) 352 352 - if err != nil { 353 353 - return nil, err 354 354 - } 355 355 - defer rows.Close() 356 356 - 357 357 - var res []PendingArtifact 358 358 - for rows.Next() { 359 359 - var p PendingArtifact 360 360 - if err := rows.Scan(&p.LeaseID, &p.Workflow, &p.Status, &p.Error, &p.ExitCode, &p.Ref, &p.Hash); err != nil { 361 361 - return nil, err 362 362 - } 363 363 - res = append(res, p) 364 364 - } 365 365 - return res, rows.Err() 366 366 - } 367 367 - 368 368 - func (d *DB) ApplyEventBatch(n *notifier.Notifier, fn func(tx *EventBatchTx) error) error { 369 369 - tx, err := d.Begin() 370 370 - if err != nil { 371 371 - return err 372 372 - } 373 373 - defer tx.Rollback() 374 374 - 375 375 - batchTx := &EventBatchTx{ 376 376 - tx: tx, 377 377 - db: d, 378 378 - } 379 379 - 380 380 - if err := fn(batchTx); err != nil { 381 381 - return err 382 382 - } 383 383 - 384 384 - if err := tx.Commit(); err != nil { 385 385 - return err 386 386 - } 387 387 - 388 388 - if n != nil { 389 389 - n.NotifyAll() 390 390 - } 391 391 - return nil 392 392 - }

-394

spindle/db/mill_state_test.go

Reviewed

··· 1 1 - package db 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "fmt" 6 6 - "path/filepath" 7 7 - "testing" 8 8 - 9 9 - "tangled.org/core/notifier" 10 10 - ) 11 11 - 12 12 - func TestMillLeaseRoundTrip(t *testing.T) { 13 13 - d := newTestDB(t) 14 14 - 15 15 - lease := MillLease{ 16 16 - LeaseID: "lease-1", 17 17 - NodeID: "node-1", 18 18 - Epoch: "inc-1", 19 19 - Engine: "dummy", 20 20 - Knot: "knot.example", 21 21 - Rkey: "rkey1", 22 22 - Workflow: "build", 23 23 - State: "reserved", 24 24 - } 25 25 - if err := d.SaveMillLease(lease); err != nil { 26 26 - t.Fatalf("SaveMillLease: %v", err) 27 27 - } 28 28 - 29 29 - lease.State = "running" 30 30 - if err := d.SaveMillLease(lease); err != nil { 31 31 - t.Fatalf("SaveMillLease(transition): %v", err) 32 32 - } 33 33 - 34 34 - leases, err := d.ListMillLeases() 35 35 - if err != nil { 36 36 - t.Fatalf("ListMillLeases: %v", err) 37 37 - } 38 38 - if len(leases) != 1 { 39 39 - t.Fatalf("ListMillLeases returned %d leases, want 1 (state transition must replace, not duplicate)", len(leases)) 40 40 - } 41 41 - if leases[0].LeaseID != lease.LeaseID || leases[0].State != "running" { 42 42 - t.Fatalf("ListMillLeases[0] = %+v, want %+v", leases[0], lease) 43 43 - } 44 44 - 45 45 - if err := d.DeleteMillLease("lease-1"); err != nil { 46 46 - t.Fatalf("DeleteMillLease: %v", err) 47 47 - } 48 48 - if leases, _ = d.ListMillLeases(); len(leases) != 0 { 49 49 - t.Fatalf("lease survived deletion: %+v", leases) 50 50 - } 51 51 - } 52 52 - 53 53 - func TestExecutorCursors(t *testing.T) { 54 54 - d := newTestDB(t) 55 55 - 56 56 - if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 5) }); err != nil { 57 57 - t.Fatalf("AdvanceCursor: %v", err) 58 58 - } 59 59 - if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 9) }); err != nil { 60 60 - t.Fatalf("AdvanceCursor(advance): %v", err) 61 61 - } 62 62 - if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-2", "inc-1", 1) }); err != nil { 63 63 - t.Fatalf("AdvanceCursor(node-2): %v", err) 64 64 - } 65 65 - 66 66 - cursors, err := d.ListExecutorCursors() 67 67 - if err != nil { 68 68 - t.Fatalf("ListExecutorCursors: %v", err) 69 69 - } 70 70 - if len(cursors) != 2 { 71 71 - t.Fatalf("ListExecutorCursors = %v, want 2 cursors", cursors) 72 72 - } 73 73 - 74 74 - cursorMap := make(map[string]uint64) 75 75 - for _, c := range cursors { 76 76 - cursorMap[c.NodeID+"/"+c.Epoch] = c.AckedSeqno 77 77 - } 78 78 - 79 79 - if cursorMap["node-1/inc-1"] != 9 || cursorMap["node-2/inc-1"] != 1 { 80 80 - t.Fatalf("unexpected cursors: %v", cursorMap) 81 81 - } 82 82 - 83 83 - } 84 84 - 85 85 - func TestCompleteMillLeaseIsAtomic(t *testing.T) { 86 86 - d := newTestDB(t) 87 87 - lease := MillLease{ 88 88 - LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 89 89 - Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: "running", 90 90 - } 91 91 - if err := d.SaveMillLease(lease); err != nil { 92 92 - t.Fatalf("SaveMillLease: %v", err) 93 93 - } 94 94 - if _, err := d.Exec(` 95 95 - create trigger reject_mill_lease_delete 96 96 - before delete on mill_leases 97 97 - begin 98 98 - select raise(abort, 'forced delete failure'); 99 99 - end 100 100 - `); err != nil { 101 101 - t.Fatalf("create failure trigger: %v", err) 102 102 - } 103 103 - 104 104 - n := notifier.New() 105 105 - notifications := n.Subscribe() 106 106 - defer n.Unsubscribe(notifications) 107 107 - err := d.CompleteMillLease( 108 108 - "lease-1", 109 109 - "at://knot.example/sh.tangled.pipeline/rkey1", 110 110 - "build", 111 111 - "failed", 112 112 - nil, 113 113 - nil, 114 114 - &n, 115 115 - ) 116 116 - if err == nil { 117 117 - t.Fatal("CompleteMillLease succeeded despite forced lease deletion failure") 118 118 - } 119 119 - var eventCount int 120 120 - if err := d.QueryRow(`select count(*) from events`).Scan(&eventCount); err != nil { 121 121 - t.Fatalf("count events after rollback: %v", err) 122 122 - } 123 123 - if eventCount != 0 { 124 124 - t.Fatalf("terminal event count after rollback = %d, want 0", eventCount) 125 125 - } 126 126 - if leases, listErr := d.ListMillLeases(); listErr != nil || len(leases) != 1 { 127 127 - t.Fatalf("leases after rollback = %+v, err = %v; want original lease", leases, listErr) 128 128 - } 129 129 - select { 130 130 - case <-notifications: 131 131 - t.Fatal("rollback notified event subscribers") 132 132 - default: 133 133 - } 134 134 - 135 135 - if _, err := d.Exec(`drop trigger reject_mill_lease_delete`); err != nil { 136 136 - t.Fatalf("drop failure trigger: %v", err) 137 137 - } 138 138 - if err := d.CompleteMillLease( 139 139 - "lease-1", 140 140 - "at://knot.example/sh.tangled.pipeline/rkey1", 141 141 - "build", 142 142 - "failed", 143 143 - nil, 144 144 - nil, 145 145 - &n, 146 146 - ); err != nil { 147 147 - t.Fatalf("CompleteMillLease retry: %v", err) 148 148 - } 149 149 - if err := d.QueryRow(`select count(*) from events`).Scan(&eventCount); err != nil { 150 150 - t.Fatalf("count committed events: %v", err) 151 151 - } 152 152 - if eventCount != 1 { 153 153 - t.Fatalf("terminal event count after commit = %d, want 1", eventCount) 154 154 - } 155 155 - if leases, listErr := d.ListMillLeases(); listErr != nil || len(leases) != 0 { 156 156 - t.Fatalf("leases after commit = %+v, err = %v; want none", leases, listErr) 157 157 - } 158 158 - select { 159 159 - case <-notifications: 160 160 - default: 161 161 - t.Fatal("committed terminal event did not notify subscribers") 162 162 - } 163 163 - } 164 164 - 165 165 - func TestRestartPersistence(t *testing.T) { 166 166 - dbPath := filepath.Join(t.TempDir(), "persist.db") 167 167 - ctx := context.Background() 168 168 - d, err := Make(ctx, dbPath) 169 169 - if err != nil { 170 170 - t.Fatalf("Make: %v", err) 171 171 - } 172 172 - 173 173 - lease := MillLease{ 174 174 - LeaseID: "lease-p", NodeID: "node-p", Epoch: "inc-p", Engine: "dummy", 175 175 - Knot: "k", Rkey: "r", Workflow: "w", State: "running", 176 176 - } 177 177 - if err := d.SaveMillLease(lease); err != nil { 178 178 - t.Fatalf("SaveMillLease: %v", err) 179 179 - } 180 180 - 181 181 - if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-p", "inc-p", 42) }); err != nil { 182 182 - t.Fatalf("AdvanceCursor: %v", err) 183 183 - } 184 184 - 185 185 - if err := d.SetOutboxEpoch("inc-p"); err != nil { 186 186 - t.Fatalf("SetOutboxEpoch: %v", err) 187 187 - } 188 188 - if _, err := d.AppendOutboxRow([]byte("hello world"), true); err != nil { 189 189 - t.Fatalf("AppendOutboxRow: %v", err) 190 190 - } 191 191 - 192 192 - if err := d.Close(); err != nil { 193 193 - t.Fatalf("Close: %v", err) 194 194 - } 195 195 - 196 196 - d2, err := Make(ctx, dbPath) 197 197 - if err != nil { 198 198 - t.Fatalf("Make reopen: %v", err) 199 199 - } 200 200 - defer d2.Close() 201 201 - 202 202 - leases, err := d2.ListMillLeases() 203 203 - if err != nil { 204 204 - t.Fatalf("ListMillLeases: %v", err) 205 205 - } 206 206 - if len(leases) != 1 || leases[0].LeaseID != "lease-p" || leases[0].Epoch != "inc-p" { 207 207 - t.Fatalf("unexpected leases: %+v", leases) 208 208 - } 209 209 - 210 210 - cursors, err := d2.ListExecutorCursors() 211 211 - if err != nil { 212 212 - t.Fatalf("ListExecutorCursors: %v", err) 213 213 - } 214 214 - if len(cursors) != 1 || cursors[0].NodeID != "node-p" || cursors[0].Epoch != "inc-p" || cursors[0].AckedSeqno != 42 { 215 215 - t.Fatalf("unexpected cursors: %+v", cursors) 216 216 - } 217 217 - 218 218 - inc, nextSeqno, err := d2.GetOutboxState() 219 219 - if err != nil { 220 220 - t.Fatalf("GetOutboxState: %v", err) 221 221 - } 222 222 - if inc != "inc-p" || nextSeqno != 2 { 223 223 - t.Fatalf("unexpected outbox state: inc=%q, next=%d", inc, nextSeqno) 224 224 - } 225 225 - rows, err := d2.ListOutboxRows() 226 226 - if err != nil { 227 227 - t.Fatalf("ListOutboxRows: %v", err) 228 228 - } 229 229 - if len(rows) != 1 || string(rows[0].Payload) != "hello world" || !rows[0].Control { 230 230 - t.Fatalf("unexpected outbox rows: %+v", rows) 231 231 - } 232 232 - } 233 233 - 234 234 - func TestOutboxPrefixAck(t *testing.T) { 235 235 - d := newTestDB(t) 236 236 - 237 237 - if err := d.SetOutboxEpoch("inc-1"); err != nil { 238 238 - t.Fatalf("SetOutboxEpoch: %v", err) 239 239 - } 240 240 - 241 241 - o1, err := d.AppendOutboxRow([]byte("msg1"), false) 242 242 - if err != nil || o1 != 1 { 243 243 - t.Fatalf("AppendOutboxRow 1: %v, seqno=%d", err, o1) 244 244 - } 245 245 - o2, err := d.AppendOutboxRow([]byte("msg2"), false) 246 246 - if err != nil || o2 != 2 { 247 247 - t.Fatalf("AppendOutboxRow 2: %v, seqno=%d", err, o2) 248 248 - } 249 249 - o3, err := d.AppendOutboxRow([]byte("msg3"), true) 250 250 - if err != nil || o3 != 3 { 251 251 - t.Fatalf("AppendOutboxRow 3: %v, seqno=%d", err, o3) 252 252 - } 253 253 - 254 254 - n, err := d.DeleteOutboxPrefix(2) 255 255 - if err != nil { 256 256 - t.Fatalf("DeleteOutboxPrefix: %v", err) 257 257 - } 258 258 - if n.Rows != 2 || n.Bytes != 8 { 259 259 - t.Fatalf("deleted prefix = %+v, want 2 rows and 8 bytes", n) 260 260 - } 261 261 - 262 262 - rows, err := d.ListOutboxRows() 263 263 - if err != nil { 264 264 - t.Fatalf("ListOutboxRows: %v", err) 265 265 - } 266 266 - if len(rows) != 1 || rows[0].Seqno != 3 || string(rows[0].Payload) != "msg3" { 267 267 - t.Fatalf("expected only msg3 (seqno 3) to remain, got: %+v", rows) 268 268 - } 269 269 - } 270 270 - 271 271 - func TestCompositeCursors(t *testing.T) { 272 272 - d := newTestDB(t) 273 273 - 274 274 - if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 10) }); err != nil { 275 275 - t.Fatalf("AdvanceCursor: %v", err) 276 276 - } 277 277 - if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-2", 20) }); err != nil { 278 278 - t.Fatalf("AdvanceCursor: %v", err) 279 279 - } 280 280 - if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-2", "inc-1", 5) }); err != nil { 281 281 - t.Fatalf("AdvanceCursor: %v", err) 282 282 - } 283 283 - 284 284 - cursors, err := d.ListExecutorCursors() 285 285 - if err != nil { 286 286 - t.Fatalf("ListExecutorCursors: %v", err) 287 287 - } 288 288 - if len(cursors) != 3 { 289 289 - t.Fatalf("expected 3 cursors, got %d", len(cursors)) 290 290 - } 291 291 - 292 292 - cursorMap := make(map[string]uint64) 293 293 - for _, c := range cursors { 294 294 - key := c.NodeID + "/" + c.Epoch 295 295 - cursorMap[key] = c.AckedSeqno 296 296 - } 297 297 - 298 298 - if cursorMap["node-1/inc-1"] != 10 || cursorMap["node-1/inc-2"] != 20 || cursorMap["node-2/inc-1"] != 5 { 299 299 - t.Fatalf("unexpected cursor values: %v", cursorMap) 300 300 - } 301 301 - 302 302 - } 303 303 - 304 304 - func TestBatchRollback(t *testing.T) { 305 305 - d := newTestDB(t) 306 306 - 307 307 - lease := MillLease{ 308 308 - LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 309 309 - Knot: "k", Rkey: "r", Workflow: "w", State: "running", 310 310 - } 311 311 - if err := d.SaveMillLease(lease); err != nil { 312 312 - t.Fatalf("SaveMillLease: %v", err) 313 313 - } 314 314 - 315 315 - n := notifier.New() 316 316 - err := d.ApplyEventBatch(&n, func(tx *EventBatchTx) error { 317 317 - if err := tx.DeleteLease("lease-1"); err != nil { 318 318 - return err 319 319 - } 320 320 - if err := tx.AdvanceCursor("node-1", "inc-1", 100); err != nil { 321 321 - return err 322 322 - } 323 323 - return fmt.Errorf("forced batch failure") 324 324 - }) 325 325 - 326 326 - if err == nil { 327 327 - t.Fatal("expected ApplyEventBatch to return error") 328 328 - } 329 329 - 330 330 - leases, err := d.ListMillLeases() 331 331 - if err != nil { 332 332 - t.Fatalf("ListMillLeases: %v", err) 333 333 - } 334 334 - if len(leases) != 1 { 335 335 - t.Fatalf("lease was deleted despite rollback: %+v", leases) 336 336 - } 337 337 - 338 338 - cursors, err := d.ListExecutorCursors() 339 339 - if err != nil { 340 340 - t.Fatalf("ListExecutorCursors: %v", err) 341 341 - } 342 342 - if len(cursors) != 0 { 343 343 - t.Fatalf("cursor was advanced despite rollback: %+v", cursors) 344 344 - } 345 345 - } 346 346 - 347 347 - func TestTerminalCursorAtomicity(t *testing.T) { 348 348 - d := newTestDB(t) 349 349 - 350 350 - lease := MillLease{ 351 351 - LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 352 352 - Knot: "k", Rkey: "r", Workflow: "w", State: "running", 353 353 - } 354 354 - if err := d.SaveMillLease(lease); err != nil { 355 355 - t.Fatalf("SaveMillLease: %v", err) 356 356 - } 357 357 - 358 358 - n := notifier.New() 359 359 - notifications := n.Subscribe() 360 360 - defer n.Unsubscribe(notifications) 361 361 - 362 362 - err := d.ApplyEventBatch(&n, func(tx *EventBatchTx) error { 363 363 - if err := tx.DeleteLease("lease-1"); err != nil { 364 364 - return err 365 365 - } 366 366 - return tx.AdvanceCursor("node-1", "inc-1", 100) 367 367 - }) 368 368 - 369 369 - if err != nil { 370 370 - t.Fatalf("ApplyEventBatch: %v", err) 371 371 - } 372 372 - 373 373 - leases, err := d.ListMillLeases() 374 374 - if err != nil { 375 375 - t.Fatalf("ListMillLeases: %v", err) 376 376 - } 377 377 - if len(leases) != 0 { 378 378 - t.Fatalf("lease not deleted: %+v", leases) 379 379 - } 380 380 - 381 381 - cursors, err := d.ListExecutorCursors() 382 382 - if err != nil { 383 383 - t.Fatalf("ListExecutorCursors: %v", err) 384 384 - } 385 385 - if len(cursors) != 1 || cursors[0].AckedSeqno != 100 { 386 386 - t.Fatalf("cursor not advanced correctly: %+v", cursors) 387 387 - } 388 388 - 389 389 - select { 390 390 - case <-notifications: 391 391 - default: 392 392 - t.Fatal("notifier was not fired after batch commit") 393 393 - } 394 394 - }

-167

spindle/db/mill_tokens.go

Reviewed

··· 1 1 - package db 2 2 - 3 3 - import ( 4 4 - "database/sql" 5 5 - "encoding/json" 6 6 - "slices" 7 7 - "strings" 8 8 - "time" 9 9 - ) 10 10 - 11 11 - // the raw token is never stored, only its hash 12 12 - type ExecutorToken struct { 13 13 - Name string 14 14 - CreatedAt string 15 15 - ExpiresAt *time.Time 16 16 - Labels []string 17 17 - QuarantineReason *string 18 18 - QuarantinedAt *string 19 19 - } 20 20 - 21 21 - func (d *DB) AddExecutorToken(name, tokenHash string, expiresAt *time.Time, labels []string) error { 22 22 - normalized := normalizeLabels(labels) 23 23 - labelsBytes, err := json.Marshal(normalized) 24 24 - if err != nil { 25 25 - return err 26 26 - } 27 27 - _, err = d.Exec( 28 28 - `insert into mill_executors (name, token_hash, expires_at, labels) values (?, ?, ?, ?)`, 29 29 - name, tokenHash, expiryArg(expiresAt), string(labelsBytes), 30 30 - ) 31 31 - return err 32 32 - } 33 33 - 34 34 - func (d *DB) ResolveExecutorToken(tokenHash string) (string, []string, bool, error) { 35 35 - var name string 36 36 - var expires sql.NullString 37 37 - var labelsRaw, quarantineReason sql.NullString 38 38 - err := d.QueryRow( 39 39 - `select name, expires_at, labels, quarantine_reason from mill_executors where token_hash = ?`, tokenHash, 40 40 - ).Scan(&name, &expires, &labelsRaw, &quarantineReason) 41 41 - if err == sql.ErrNoRows { 42 42 - return "", nil, false, nil 43 43 - } 44 44 - if err != nil { 45 45 - return "", nil, false, err 46 46 - } 47 47 - if quarantineReason.Valid { 48 48 - return "", nil, false, nil 49 49 - } 50 50 - exp, hasExpiry, err := parseExpiry(expires) 51 51 - if err != nil { 52 52 - return "", nil, false, err 53 53 - } 54 54 - if hasExpiry && time.Now().After(exp) { 55 55 - return "", nil, false, nil 56 56 - } 57 57 - var labels []string 58 58 - if labelsRaw.Valid && labelsRaw.String != "" { 59 59 - if err := json.Unmarshal([]byte(labelsRaw.String), &labels); err != nil { 60 60 - return "", nil, false, err 61 61 - } 62 62 - } 63 63 - return name, labels, true, nil 64 64 - } 65 65 - 66 66 - func (d *DB) QuarantineExecutor(name, reason string) error { 67 67 - _, err := d.Exec( 68 68 - `update mill_executors 69 69 - set quarantine_reason = ?, quarantined_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') 70 70 - where name = ?`, 71 71 - reason, name, 72 72 - ) 73 73 - return err 74 74 - } 75 75 - 76 76 - func (d *DB) ClearExecutorQuarantine(name string) (bool, error) { 77 77 - res, err := d.Exec( 78 78 - `update mill_executors set quarantine_reason = null, quarantined_at = null where name = ?`, 79 79 - name, 80 80 - ) 81 81 - if err != nil { 82 82 - return false, err 83 83 - } 84 84 - n, err := res.RowsAffected() 85 85 - return n > 0, err 86 86 - } 87 87 - 88 88 - func (d *DB) RevokeExecutorToken(name string) (bool, error) { 89 89 - res, err := d.Exec(`delete from mill_executors where name = ?`, name) 90 90 - if err != nil { 91 91 - return false, err 92 92 - } 93 93 - n, err := res.RowsAffected() 94 94 - return n > 0, err 95 95 - } 96 96 - 97 97 - func (d *DB) ListExecutorTokens() ([]ExecutorToken, error) { 98 98 - rows, err := d.Query(`select name, created_at, expires_at, labels, quarantine_reason, quarantined_at from mill_executors order by name`) 99 99 - if err != nil { 100 100 - return nil, err 101 101 - } 102 102 - defer rows.Close() 103 103 - 104 104 - var out []ExecutorToken 105 105 - for rows.Next() { 106 106 - var t ExecutorToken 107 107 - var expires, labelsRaw, quarantineReason, quarantinedAt sql.NullString 108 108 - if err := rows.Scan(&t.Name, &t.CreatedAt, &expires, &labelsRaw, &quarantineReason, &quarantinedAt); err != nil { 109 109 - return nil, err 110 110 - } 111 111 - exp, hasExpiry, err := parseExpiry(expires) 112 112 - if err != nil { 113 113 - return nil, err 114 114 - } 115 115 - if hasExpiry { 116 116 - t.ExpiresAt = &exp 117 117 - } 118 118 - if labelsRaw.Valid && labelsRaw.String != "" { 119 119 - if err := json.Unmarshal([]byte(labelsRaw.String), &t.Labels); err != nil { 120 120 - return nil, err 121 121 - } 122 122 - } 123 123 - if quarantineReason.Valid { 124 124 - t.QuarantineReason = &quarantineReason.String 125 125 - } 126 126 - if quarantinedAt.Valid { 127 127 - t.QuarantinedAt = &quarantinedAt.String 128 128 - } 129 129 - out = append(out, t) 130 130 - } 131 131 - return out, rows.Err() 132 132 - } 133 133 - 134 134 - func normalizeLabels(labels []string) []string { 135 135 - var out []string 136 136 - seen := make(map[string]bool) 137 137 - for _, l := range labels { 138 138 - trimmed := strings.TrimSpace(l) 139 139 - if trimmed == "" { 140 140 - continue 141 141 - } 142 142 - if !seen[trimmed] { 143 143 - seen[trimmed] = true 144 144 - out = append(out, trimmed) 145 145 - } 146 146 - } 147 147 - slices.Sort(out) 148 148 - return out 149 149 - } 150 150 - 151 151 - func expiryArg(t *time.Time) any { 152 152 - if t == nil { 153 153 - return nil 154 154 - } 155 155 - return t.UTC().Format(time.RFC3339) 156 156 - } 157 157 - 158 158 - func parseExpiry(s sql.NullString) (time.Time, bool, error) { 159 159 - if !s.Valid { 160 160 - return time.Time{}, false, nil 161 161 - } 162 162 - t, err := time.Parse(time.RFC3339, s.String) 163 163 - if err != nil { 164 164 - return time.Time{}, false, err 165 165 - } 166 166 - return t, true, nil 167 167 - }

-272

spindle/db/mill_tokens_test.go

Reviewed

··· 1 1 - package db 2 2 - 3 3 - import ( 4 4 - "slices" 5 5 - "testing" 6 6 - "time" 7 7 - ) 8 8 - 9 9 - func TestAddExecutorTokenRejectsDuplicateName(t *testing.T) { 10 10 - d := newTestDB(t) 11 11 - 12 12 - if err := d.AddExecutorToken("exec-1", "hash-a", nil, nil); err != nil { 13 13 - t.Fatalf("AddExecutorToken: %v", err) 14 14 - } 15 15 - if err := d.AddExecutorToken("exec-1", "hash-b", nil, nil); err == nil { 16 16 - t.Fatal("AddExecutorToken re-registered an existing name; a duplicate must be rejected") 17 17 - } 18 18 - 19 19 - name, _, ok, err := d.ResolveExecutorToken("hash-a") 20 20 - if err != nil { 21 21 - t.Fatalf("ResolveExecutorToken(hash-a): %v", err) 22 22 - } 23 23 - if !ok || name != "exec-1" { 24 24 - t.Fatalf("ResolveExecutorToken(hash-a) = (%q, %v), want (exec-1, true)", name, ok) 25 25 - } 26 26 - if _, _, ok, _ := d.ResolveExecutorToken("hash-b"); ok { 27 27 - t.Fatal("rejected duplicate's token resolved; the failed insert leaked a credential") 28 28 - } 29 29 - } 30 30 - 31 31 - func TestResolveExecutorTokenMissAndHit(t *testing.T) { 32 32 - d := newTestDB(t) 33 33 - 34 34 - name, _, ok, err := d.ResolveExecutorToken("no-such-hash") 35 35 - if err != nil { 36 36 - t.Fatalf("ResolveExecutorToken(miss): %v", err) 37 37 - } 38 38 - if ok || name != "" { 39 39 - t.Fatalf("ResolveExecutorToken(miss) = (%q, %v), want (\"\", false)", name, ok) 40 40 - } 41 41 - 42 42 - if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil { 43 43 - t.Fatalf("AddExecutorToken: %v", err) 44 44 - } 45 45 - 46 46 - name, _, ok, err = d.ResolveExecutorToken("hash-1") 47 47 - if err != nil { 48 48 - t.Fatalf("ResolveExecutorToken(hit): %v", err) 49 49 - } 50 50 - if !ok || name != "exec-1" { 51 51 - t.Fatalf("ResolveExecutorToken(hash-1) = (%q, %v), want (exec-1, true)", name, ok) 52 52 - } 53 53 - 54 54 - if _, _, ok, _ := d.ResolveExecutorToken("hash-unregistered"); ok { 55 55 - t.Fatal("ResolveExecutorToken matched an unregistered hash") 56 56 - } 57 57 - } 58 58 - 59 59 - func TestRevokeExecutorToken(t *testing.T) { 60 60 - d := newTestDB(t) 61 61 - 62 62 - if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil { 63 63 - t.Fatalf("AddExecutorToken: %v", err) 64 64 - } 65 65 - 66 66 - deleted, err := d.RevokeExecutorToken("exec-1") 67 67 - if err != nil { 68 68 - t.Fatalf("RevokeExecutorToken: %v", err) 69 69 - } 70 70 - if !deleted { 71 71 - t.Fatal("RevokeExecutorToken reported no deletion for an existing identity") 72 72 - } 73 73 - 74 74 - if _, _, ok, _ := d.ResolveExecutorToken("hash-1"); ok { 75 75 - t.Fatal("revoked token still resolves; revocation is not enforced") 76 76 - } 77 77 - 78 78 - if deleted, err := d.RevokeExecutorToken("exec-1"); err != nil || deleted { 79 79 - t.Fatalf("RevokeExecutorToken(already-gone) = (%v, %v), want (false, nil)", deleted, err) 80 80 - } 81 81 - 82 82 - if deleted, err := d.RevokeExecutorToken("ghost"); err != nil || deleted { 83 83 - t.Fatalf("RevokeExecutorToken(unknown) = (%v, %v), want (false, nil)", deleted, err) 84 84 - } 85 85 - } 86 86 - 87 87 - func TestExecutorQuarantineIsVisibleAndReversible(t *testing.T) { 88 88 - d := newTestDB(t) 89 89 - if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil { 90 90 - t.Fatalf("AddExecutorToken: %v", err) 91 91 - } 92 92 - if err := d.QuarantineExecutor("exec-1", "missed cancel deadline"); err != nil { 93 93 - t.Fatalf("QuarantineExecutor: %v", err) 94 94 - } 95 95 - if _, _, ok, err := d.ResolveExecutorToken("hash-1"); err != nil || ok { 96 96 - t.Fatalf("ResolveExecutorToken(quarantined) = (ok=%v, err=%v), want (false, nil)", ok, err) 97 97 - } 98 98 - tokens, err := d.ListExecutorTokens() 99 99 - if err != nil { 100 100 - t.Fatalf("ListExecutorTokens: %v", err) 101 101 - } 102 102 - if len(tokens) != 1 || tokens[0].QuarantineReason == nil || *tokens[0].QuarantineReason != "missed cancel deadline" || tokens[0].QuarantinedAt == nil { 103 103 - t.Fatalf("quarantined token not surfaced: %+v", tokens) 104 104 - } 105 105 - if cleared, err := d.ClearExecutorQuarantine("exec-1"); err != nil || !cleared { 106 106 - t.Fatalf("ClearExecutorQuarantine = (%v, %v), want (true, nil)", cleared, err) 107 107 - } 108 108 - if _, _, ok, err := d.ResolveExecutorToken("hash-1"); err != nil || !ok { 109 109 - t.Fatalf("ResolveExecutorToken(cleared) = (ok=%v, err=%v), want (true, nil)", ok, err) 110 110 - } 111 111 - if cleared, err := d.ClearExecutorQuarantine("missing"); err != nil || cleared { 112 112 - t.Fatalf("ClearExecutorQuarantine(missing) = (%v, %v), want (false, nil)", cleared, err) 113 113 - } 114 114 - } 115 115 - 116 116 - func TestListExecutorTokens(t *testing.T) { 117 117 - d := newTestDB(t) 118 118 - 119 119 - tokens, err := d.ListExecutorTokens() 120 120 - if err != nil { 121 121 - t.Fatalf("ListExecutorTokens(empty): %v", err) 122 122 - } 123 123 - if len(tokens) != 0 { 124 124 - t.Fatalf("ListExecutorTokens on empty table = %d rows, want 0", len(tokens)) 125 125 - } 126 126 - 127 127 - // insert out of alphabetical order to test query sorting 128 128 - for _, name := range []string{"charlie", "alice", "bob"} { 129 129 - if err := d.AddExecutorToken(name, "hash-"+name, nil, nil); err != nil { 130 130 - t.Fatalf("AddExecutorToken(%s): %v", name, err) 131 131 - } 132 132 - } 133 133 - 134 134 - tokens, err = d.ListExecutorTokens() 135 135 - if err != nil { 136 136 - t.Fatalf("ListExecutorTokens: %v", err) 137 137 - } 138 138 - want := []string{"alice", "bob", "charlie"} 139 139 - if len(tokens) != len(want) { 140 140 - t.Fatalf("ListExecutorTokens = %d rows, want %d", len(tokens), len(want)) 141 141 - } 142 142 - for i := range want { 143 143 - if tokens[i].Name != want[i] { 144 144 - t.Fatalf("ListExecutorTokens[%d].Name = %q, want %q (ordered by name)", i, tokens[i].Name, want[i]) 145 145 - } 146 146 - } 147 147 - } 148 148 - 149 149 - // expired tokens must fail closed 150 150 - func TestResolveExecutorTokenExpiry(t *testing.T) { 151 151 - cases := []struct { 152 152 - name string 153 153 - seqno time.Duration 154 154 - noExpiry bool 155 155 - wantOK bool 156 156 - }{ 157 157 - {"future expiry resolves", time.Hour, false, true}, 158 158 - {"past expiry fails closed", -time.Hour, false, false}, 159 159 - {"nil expiry never expires", 0, true, true}, 160 160 - } 161 161 - for _, tc := range cases { 162 162 - t.Run(tc.name, func(t *testing.T) { 163 163 - d := newTestDB(t) 164 164 - 165 165 - var expires *time.Time 166 166 - if !tc.noExpiry { 167 167 - exp := time.Now().Add(tc.seqno) 168 168 - expires = &exp 169 169 - } 170 170 - if err := d.AddExecutorToken("exec-1", "hash-1", expires, nil); err != nil { 171 171 - t.Fatalf("AddExecutorToken: %v", err) 172 172 - } 173 173 - 174 174 - name, _, ok, err := d.ResolveExecutorToken("hash-1") 175 175 - if err != nil { 176 176 - t.Fatalf("ResolveExecutorToken: %v", err) 177 177 - } 178 178 - if ok != tc.wantOK { 179 179 - t.Fatalf("ResolveExecutorToken ok = %v, want %v", ok, tc.wantOK) 180 180 - } 181 181 - if tc.wantOK && name != "exec-1" { 182 182 - t.Fatalf("ResolveExecutorToken name = %q, want exec-1", name) 183 183 - } 184 184 - if !tc.wantOK && name != "" { 185 185 - t.Fatalf("ResolveExecutorToken name = %q, want \"\" when failing closed", name) 186 186 - } 187 187 - }) 188 188 - } 189 189 - } 190 190 - 191 191 - func TestResolveExecutorTokenRejectsMalformedExpiry(t *testing.T) { 192 192 - d := newTestDB(t) 193 193 - if _, err := d.Exec( 194 194 - `insert into mill_executors (name, token_hash, expires_at) values (?, ?, ?)`, 195 195 - "exec-1", "hash-1", "not-a-timestamp", 196 196 - ); err != nil { 197 197 - t.Fatalf("insert malformed token: %v", err) 198 198 - } 199 199 - 200 200 - name, _, ok, err := d.ResolveExecutorToken("hash-1") 201 201 - if err == nil { 202 202 - t.Fatal("ResolveExecutorToken accepted a malformed non-NULL expiry") 203 203 - } 204 204 - if ok || name != "" { 205 205 - t.Fatalf("ResolveExecutorToken = (%q, %v, %v), want (\"\", false, error)", name, ok, err) 206 206 - } 207 207 - } 208 208 - 209 209 - // expiry storage is RFC3339 second precision UTC so 210 210 - // round trips compare to the second 211 211 - func TestListExecutorTokensSurfacesExpiry(t *testing.T) { 212 212 - d := newTestDB(t) 213 213 - 214 214 - exp := time.Now().Add(24 * time.Hour) 215 215 - if err := d.AddExecutorToken("expiring", "hash-exp", &exp, nil); err != nil { 216 216 - t.Fatalf("AddExecutorToken(expiring): %v", err) 217 217 - } 218 218 - if err := d.AddExecutorToken("forever", "hash-forever", nil, nil); err != nil { 219 219 - t.Fatalf("AddExecutorToken(forever): %v", err) 220 220 - } 221 221 - 222 222 - tokens, err := d.ListExecutorTokens() 223 223 - if err != nil { 224 224 - t.Fatalf("ListExecutorTokens: %v", err) 225 225 - } 226 226 - 227 227 - got := make(map[string]*time.Time, len(tokens)) 228 228 - for _, tok := range tokens { 229 229 - got[tok.Name] = tok.ExpiresAt 230 230 - } 231 231 - 232 232 - e, present := got["forever"] 233 233 - if !present { 234 234 - t.Fatal("ListExecutorTokens omitted the non-expiring identity") 235 235 - } 236 236 - if e != nil { 237 237 - t.Fatalf("forever.ExpiresAt = %v, want nil (never expires)", e) 238 238 - } 239 239 - 240 240 - e, present = got["expiring"] 241 241 - if !present { 242 242 - t.Fatal("ListExecutorTokens omitted the expiring identity") 243 243 - } 244 244 - if e == nil { 245 245 - t.Fatal("expiring.ExpiresAt = nil, want the stored expiry") 246 246 - } 247 247 - if e.Unix() != exp.Unix() { 248 248 - t.Fatalf("expiring.ExpiresAt = %d (unix), want %d", e.Unix(), exp.Unix()) 249 249 - } 250 250 - } 251 251 - 252 252 - func TestExecutorTokenLabels(t *testing.T) { 253 253 - d := newTestDB(t) 254 254 - 255 255 - labels := []string{" foo ", "bar", " foo", ""} 256 256 - if err := d.AddExecutorToken("exec-1", "hash-1", nil, labels); err != nil { 257 257 - t.Fatalf("AddExecutorToken: %v", err) 258 258 - } 259 259 - 260 260 - name, resolvedLabels, ok, err := d.ResolveExecutorToken("hash-1") 261 261 - if err != nil { 262 262 - t.Fatalf("ResolveExecutorToken: %v", err) 263 263 - } 264 264 - if !ok || name != "exec-1" { 265 265 - t.Fatalf("ResolveExecutorToken: ok=%v, name=%q, want true, exec-1", ok, name) 266 266 - } 267 267 - 268 268 - wantLabels := []string{"bar", "foo"} 269 269 - if !slices.Equal(resolvedLabels, wantLabels) { 270 270 - t.Fatalf("resolved labels = %v, want %v", resolvedLabels, wantLabels) 271 271 - } 272 272 - }

+24 -58

spindle/engine/engine.go

Reviewed

··· 67 67 } 68 68 } 69 69 70 70 - // the mill streams the executor's real log file into place itself 71 71 - // a local logger here would only write competing lines 72 72 - type workflowLoggerProvider interface { 73 73 - WorkflowLogger(wid models.WorkflowId) models.WorkflowLogger 70 70 + type workflowFinalizer interface { 71 71 + FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error 74 72 } 75 73 76 76 - // for engines that manage status updates outside StartWorkflows 77 77 - type RemoteStatusEngine interface { 78 78 - AuthorsRemoteStatus() 79 79 - } 80 80 - 81 81 - func reportWorkflowStatusError(l *slog.Logger, database *db.DB, n *notifier.Notifier, wid models.WorkflowId, err error) { 82 82 - if errors.Is(err, ErrTimedOut) { 83 83 - dbErr := database.StatusTimeout(wid, n) 84 84 - if dbErr != nil { 85 85 - l.Error("failed to set workflow status to timeout", "wid", wid, "err", dbErr) 86 86 - } 87 87 - } else if errors.Is(err, ErrWorkflowCanceled) { 88 88 - dbErr := database.StatusCancelled(wid, err.Error(), -1, n) 89 89 - if dbErr != nil { 90 90 - l.Error("failed to set workflow status to cancelled", "wid", wid, "err", dbErr) 91 91 - } 92 92 - } else { 93 93 - dbErr := database.StatusFailed(wid, err.Error(), -1, n) 94 94 - if dbErr != nil { 95 95 - l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr) 96 96 - } 97 97 - } 98 98 - } 99 99 - 100 74 func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, stores *artifactstore.Stores, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { 101 75 l.Info("starting all workflows in parallel", "pipeline", pipelineId) 102 76 ··· 127 101 wfCounts[wid.String()]++ 128 102 } 129 103 } 104 104 + 130 105 var wg sync.WaitGroup 131 106 for eng, wfs := range pipeline.Workflows { 132 107 workflowTimeout := eng.WorkflowTimeout() ··· 153 128 l.Info("skipping finished workflow", "wid", wid, "status", st.Status) 154 129 return 155 130 } 156 156 - var err error 157 157 - var wfLogger models.WorkflowLogger 158 158 - if p, ok := eng.(workflowLoggerProvider); ok { 159 159 - wfLogger = p.WorkflowLogger(wid) 160 160 - } else if fileLogger, err := models.NewFileWorkflowLogger(cfg.Server.LogDir, wid, secretValues); err != nil { 131 131 + wfLogger, err := models.NewFileWorkflowLogger(cfg.Server.LogDir, wid, secretValues) 132 132 + if err != nil { 161 133 l.Warn("failed to setup step logger; logs will not be persisted", "error", err) 162 134 wfLogger = models.NullLogger{} 163 135 } else { 164 136 l.Info("setup step logger; logs will be persisted", "logDir", cfg.Server.LogDir, "wid", wid) 165 165 - wfLogger = fileLogger 166 137 defer archiveWorkflowLog(l, stores, db, cfg.Server.LogDir, wid) 167 167 - defer fileLogger.Close() 138 138 + defer wfLogger.Close() 168 139 } 169 140 170 141 timeoutCtx, timeoutCancel := context.WithTimeout(ctx, workflowTimeout) ··· 185 156 186 157 l.Info("waiting for slot", "wid", wid) 187 158 slot := WorkflowSlot(NoopSlot{}) 188 188 - _, remoteStatus := eng.(RemoteStatusEngine) 189 189 - 190 159 if s, ok := eng.(WorkflowSlotter); ok { 191 191 - slot, err = s.AcquireWorkflowSlot(wfCtx, wid, &w, Wait) 160 160 + slot, err = s.AcquireWorkflowSlot(wfCtx, wid, &w) 192 161 if err != nil { 193 162 writeWfError(db, n, l, wfCtx, wid, "waiting for slot", err) 194 163 return ··· 196 165 } 197 166 defer slot.Release() 198 167 199 199 - if !remoteStatus { 200 200 - err := db.StatusRunning(wid, n) 201 201 - if err != nil { 202 202 - l.Error("failed to set workflow status to running", "wid", wid, "err", err) 203 203 - return 204 204 - } 168 168 + err = db.StatusRunning(wid, n) 169 169 + if err != nil { 170 170 + l.Error("failed to set workflow status to running", "wid", wid, "err", err) 171 171 + return 205 172 } 206 173 207 174 err = eng.SetupWorkflow(wfCtx, wid, &w, wfLogger) ··· 211 178 l.Error("failed to destroy workflow after setup failure", "error", destroyErr) 212 179 } 213 180 } 214 214 - if !remoteStatus { 215 215 - writeWfError(db, n, l, wfCtx, wid, "setting up workflow", err) 216 216 - } 181 181 + writeWfError(db, n, l, wfCtx, wid, "setting up workflow", err) 217 182 return 218 183 } 219 184 defer eng.DestroyWorkflow(ctx, wid) ··· 234 199 } 235 200 236 201 if err != nil { 237 237 - if !remoteStatus { 238 238 - writeWfError(db, n, l, wfCtx, wid, "running step", err) 239 239 - } 202 202 + writeWfError(db, n, l, wfCtx, wid, "running step", err) 240 203 return 241 204 } 242 205 } 243 206 244 244 - if isCanceled(wfCtx) { 245 245 - if !remoteStatus { 246 246 - writeWfError(db, n, l, wfCtx, wid, "before success", nil) 207 207 + if finalizer, ok := eng.(workflowFinalizer); ok { 208 208 + if err := finalizer.FinalizeWorkflow(wfCtx, wid, &w, wfLogger); err != nil { 209 209 + writeWfError(db, n, l, wfCtx, wid, "finalizing", err) 210 210 + return 247 211 } 212 212 + } 213 213 + 214 214 + if isCanceled(wfCtx) { 215 215 + writeWfError(db, n, l, wfCtx, wid, "before success", nil) 248 216 return 249 217 } 250 218 251 251 - if !remoteStatus { 252 252 - err = db.StatusSuccess(wid, n) 253 253 - if err != nil { 254 254 - l.Error("failed to set workflow status to success", "wid", wid, "err", err) 255 255 - } 219 219 + err = db.StatusSuccess(wid, n) 220 220 + if err != nil { 221 221 + l.Error("failed to set workflow status to success", "wid", wid, "err", err) 256 222 } 257 223 }) 258 224 }

spindle/engine/engine_test.go

Reviewed

This file has not been changed.

-8

spindle/engine/manifest_test.go

Reviewed

··· 79 79 } 80 80 } 81 81 82 82 - func TestDescribeManifestErrorAcceptsRunsOnGenericWorkflowKey(t *testing.T) { 83 83 - raw := "engine: microvm\nruns_on: [linux/arm64, kvm]\nimage: nixos\n" 84 84 - 85 85 - if err := DescribeManifestError(raw, testManifest{}); err != nil { 86 86 - t.Fatalf("DescribeManifestError(%q) = %v, want nil", raw, err) 87 87 - } 88 88 - } 89 89 - 90 82 func TestDescribeManifestErrorNoFalsePositives(t *testing.T) { 91 83 cases := []string{ 92 84 // well-formed manifest

-10

spindle/engine/placement.go

Reviewed

··· 1 1 - package engine 2 2 - 3 3 - import ( 4 4 - "tangled.org/core/spindle/models" 5 5 - ) 6 6 - 7 7 - // checked before an executor accepts and holds a remote lease 8 8 - type WorkflowPlacementValidator interface { 9 9 - ValidateWorkflowPlacement(wf *models.Workflow) error 10 10 - }

spindle/engine/s3.go

Reviewed

This file has not been changed.

+8 -12

spindle/engine/scheduler.go

Reviewed

··· 50 50 } 51 51 } 52 52 53 53 + func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R) (WorkflowSlot, error) { 53 53 - // the mill owns the backlog so a NoWait caller must have room immediately or fail 54 54 - func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R, mode AcquireMode) (WorkflowSlot, error) { 55 54 if s == nil { 56 55 return NoopSlot{}, nil 57 56 } ··· 61 60 s.mu.Unlock() 62 61 return nil, fmt.Errorf("%w: request=%v budget=%v max=%v", ErrNoWorkflowSlots, req, s.budget, s.max) 63 62 } 63 63 + if len(s.queue) == 0 && s.used.Add(req).Fits(s.budget) { 64 64 - // NoWait ignores the queue because it never blocks 65 65 - // Wait only bypasses empty queues to prevent starvation 66 66 - if s.used.Add(req).Fits(s.budget) && (mode == NoWait || len(s.queue) == 0) { 67 64 s.used = s.used.Add(req) 68 65 s.mu.Unlock() 69 66 return &resourceLease[R]{scheduler: s, req: req}, nil 70 67 } 71 71 - if mode == NoWait { 72 72 - used := s.used 73 73 - s.mu.Unlock() 74 74 - return nil, fmt.Errorf("%w: request=%v used=%v budget=%v", ErrNoWorkflowSlots, req, used, s.budget) 75 75 - } 76 68 77 69 waiter := &resourceWaiter[R]{req: req, ready: make(chan struct{}), enqueuedAt: s.now()} 78 70 s.queue = append(s.queue, waiter) ··· 116 108 117 109 // start every waiter whose request fits. once a waiter is older than 118 110 // agingThreshold, count its request as already used so younger waiters 111 111 + // stop being scheduled ahead of it. 119 119 - // stop being scheduled ahead of it 120 112 func (s *ResourceScheduler[R]) schedule() { 121 113 var reserved R 122 114 now := s.now() ··· 137 129 } 138 130 139 131 func (s *ResourceScheduler[R]) remove(waiter *resourceWaiter[R]) { 132 132 + for i, candidate := range s.queue { 133 133 + if candidate != waiter { 134 134 + continue 135 135 + } 140 140 - if i := slices.Index(s.queue, waiter); i >= 0 { 141 136 s.queue = slices.Delete(s.queue, i, i+1) 137 137 + return 142 138 } 143 139 }

+9 -75

spindle/engine/scheduler_test.go

Reviewed

··· 36 36 37 37 scheduler := NewResourceScheduler(ru{}, ru{}, 0) 38 38 39 39 + slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}) 39 39 - slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}, Wait) 40 40 if err != nil { 41 41 t.Fatalf("Acquire() error = %v", err) 42 42 } ··· 48 48 49 49 scheduler := NewResourceScheduler(ru{a: 1024, b: 10_000}, ru{a: 512, b: 5_000}, 0) 50 50 51 51 + _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}) 51 51 - _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}, Wait) 52 52 if !errors.Is(err, ErrNoWorkflowSlots) { 53 53 t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err) 54 54 } 55 55 56 56 + _, err = scheduler.Acquire(context.Background(), ru{a: 128, b: 12_000}) 56 56 - _, err = scheduler.Acquire(context.Background(), ru{a: 128, b: 12_000}, Wait) 57 57 if !errors.Is(err, ErrNoWorkflowSlots) { 58 58 t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err) 59 59 } ··· 64 64 65 65 scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 0) 66 66 67 67 + first, err := scheduler.Acquire(context.Background(), ru{a: 1024}) 67 67 - first, err := scheduler.Acquire(context.Background(), ru{a: 1024}, Wait) 68 68 if err != nil { 69 69 t.Fatalf("first Acquire() error = %v", err) 70 70 } ··· 85 85 86 86 scheduler := NewResourceScheduler(ru{a: 1}, ru{}, 0) 87 87 88 88 + slot, err := scheduler.Acquire(context.Background(), ru{a: 1}) 88 88 - slot, err := scheduler.Acquire(context.Background(), ru{a: 1}, Wait) 89 89 if err != nil { 90 90 t.Fatalf("Acquire() error = %v", err) 91 91 } ··· 93 93 slot.Release() 94 94 slot.Release() 95 95 96 96 + second, err := scheduler.Acquire(context.Background(), ru{a: 1}) 96 96 - second, err := scheduler.Acquire(context.Background(), ru{a: 1}, Wait) 97 97 if err != nil { 98 98 t.Fatalf("Acquire() after double release error = %v", err) 99 99 } ··· 105 105 106 106 scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, time.Hour) // disable aging so we test pure backfill 107 107 108 108 + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}) 108 108 - hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait) 109 109 if err != nil { 110 110 t.Fatalf("hold Acquire() error = %v", err) 111 111 } ··· 128 128 fakeNow := time.Now() 129 129 scheduler.now = func() time.Time { return fakeNow } 130 130 131 131 + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}) 131 131 - hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait) 132 132 if err != nil { 133 133 t.Fatalf("hold Acquire() error = %v", err) 134 134 } ··· 151 151 big.Release() 152 152 } 153 153 154 154 - func TestResourceSchedulerTryRejectsWhenNoRoomNow(t *testing.T) { 155 155 - t.Parallel() 156 156 - 157 157 - scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 0) 158 158 - 159 159 - first, err := scheduler.Acquire(context.Background(), ru{a: 1024}, NoWait) 160 160 - if err != nil { 161 161 - t.Fatalf("first Acquire(NoWait) error = %v", err) 162 162 - } 163 163 - 164 164 - if _, err := scheduler.Acquire(context.Background(), ru{a: 1}, NoWait); !errors.Is(err, ErrNoWorkflowSlots) { 165 165 - t.Fatalf("Acquire(NoWait) error = %v, want ErrNoWorkflowSlots", err) 166 166 - } 167 167 - 168 168 - first.Release() 169 169 - 170 170 - second, err := scheduler.Acquire(context.Background(), ru{a: 1}, NoWait) 171 171 - if err != nil { 172 172 - t.Fatalf("Acquire(NoWait) after release error = %v", err) 173 173 - } 174 174 - second.Release() 175 175 - } 176 176 - 177 177 - func TestResourceSchedulerTryRejectsRequestsThatCanNeverFit(t *testing.T) { 178 178 - t.Parallel() 179 179 - 180 180 - scheduler := NewResourceScheduler(ru{a: 1024, b: 10_000}, ru{a: 512, b: 5_000}, 0) 181 181 - 182 182 - if _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}, NoWait); !errors.Is(err, ErrNoWorkflowSlots) { 183 183 - t.Fatalf("Acquire(NoWait) error = %v, want ErrNoWorkflowSlots", err) 184 184 - } 185 185 - } 186 186 - 187 187 - func TestResourceSchedulerTryIgnoresQueuedWaiters(t *testing.T) { 188 188 - t.Parallel() 189 189 - 190 190 - scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, time.Hour) 191 191 - 192 192 - hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait) 193 193 - if err != nil { 194 194 - t.Fatalf("hold Acquire() error = %v", err) 195 195 - } 196 196 - defer hold.Release() 197 197 - 198 198 - bigCh := acquireAsync(context.Background(), scheduler, ru{a: 768}) 199 199 - assertAcquireBlocked(t, bigCh) 200 200 - 201 201 - slot, err := scheduler.Acquire(context.Background(), ru{a: 256}, NoWait) 202 202 - if err != nil { 203 203 - t.Fatalf("Acquire(NoWait) error = %v, want success past queued waiter", err) 204 204 - } 205 205 - slot.Release() 206 206 - } 207 207 - 208 208 - func TestResourceSchedulerZeroLimitsTryDoesNotReject(t *testing.T) { 209 209 - t.Parallel() 210 210 - 211 211 - scheduler := NewResourceScheduler(ru{}, ru{}, 0) 212 212 - 213 213 - slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}, NoWait) 214 214 - if err != nil { 215 215 - t.Fatalf("Acquire(NoWait) error = %v", err) 216 216 - } 217 217 - slot.Release() 218 218 - } 219 219 - 220 154 func acquireAsync(ctx context.Context, scheduler *ResourceScheduler[ru], req ru) <-chan acquireResult { 221 155 ch := make(chan acquireResult, 1) 222 156 go func() { 157 157 + slot, err := scheduler.Acquire(ctx, req) 223 223 - slot, err := scheduler.Acquire(ctx, req, Wait) 224 158 ch <- acquireResult{slot: slot, err: err} 225 159 }() 226 160 return ch

+2 -21

spindle/engine/slot.go

Reviewed

··· 13 13 Release() 14 14 } 15 15 16 16 - // governs blocking behaviour when acquiring a slot 17 17 - type AcquireMode int 18 18 - 19 19 - const ( 20 20 - // blocks until a slot is free 21 21 - Wait AcquireMode = iota 22 22 - // fails immediately if full 23 23 - // executors use this because the mill owns the backlog 24 24 - NoWait 25 25 - ) 26 26 - 27 16 type WorkflowSlotter interface { 17 17 + AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) 28 28 - AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode AcquireMode) (WorkflowSlot, error) 29 18 } 30 19 31 20 type releaseFunc func() ··· 52 41 return &SemaphoreSlotter{slots: make(chan struct{}, maxConcurrent)} 53 42 } 54 43 44 44 + func (a *SemaphoreSlotter) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) { 55 55 - func (a *SemaphoreSlotter) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode AcquireMode) (WorkflowSlot, error) { 56 45 if a == nil || a.slots == nil { 57 46 return NoopSlot{}, nil 58 47 } 59 59 - if mode == NoWait { 60 60 - select { 61 61 - case a.slots <- struct{}{}: 62 62 - return releaseFunc(func() { <-a.slots }), nil 63 63 - default: 64 64 - return nil, ErrNoWorkflowSlots 65 65 - } 66 66 - } 67 48 select { 68 49 case a.slots <- struct{}{}: 69 50 return releaseFunc(func() { <-a.slots }), nil

+5 -42

spindle/engine/slot_test.go

Reviewed

··· 15 15 slotter := NewSemaphoreSlotter(0) 16 16 17 17 for range 10 { 18 18 + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) 18 18 - slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) 19 19 if err != nil { 20 20 t.Fatalf("AcquireWorkflowSlot() error = %v", err) 21 21 } ··· 28 28 29 29 slotter := NewSemaphoreSlotter(1) 30 30 31 31 + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) 31 31 - first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) 32 32 if err != nil { 33 33 t.Fatalf("first AcquireWorkflowSlot() error = %v", err) 34 34 } ··· 42 42 acquired := make(chan WorkflowSlot, 1) 43 43 errs := make(chan error, 1) 44 44 go func() { 45 45 + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) 45 45 - slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) 46 46 if err != nil { 47 47 errs <- err 48 48 return ··· 64 64 65 65 slotter := NewSemaphoreSlotter(1) 66 66 67 67 + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) 67 67 - first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) 68 68 if err != nil { 69 69 t.Fatalf("first AcquireWorkflowSlot() error = %v", err) 70 70 } ··· 73 73 ctx, cancel := context.WithCancel(context.Background()) 74 74 cancel() 75 75 76 76 + _, err = slotter.AcquireWorkflowSlot(ctx, zeroWorkflowID(), nil) 76 76 - _, err = slotter.AcquireWorkflowSlot(ctx, zeroWorkflowID(), nil, Wait) 77 77 if !errors.Is(err, context.Canceled) { 78 78 t.Fatalf("AcquireWorkflowSlot() error = %v, want context.Canceled", err) 79 79 } 80 80 } 81 81 82 82 - func TestSemaphoreSlotterTryDisabledDoesNotReject(t *testing.T) { 83 83 - t.Parallel() 84 84 - 85 85 - slotter := NewSemaphoreSlotter(0) 86 86 - 87 87 - for range 10 { 88 88 - slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait) 89 89 - if err != nil { 90 90 - t.Fatalf("AcquireWorkflowSlot(NoWait) error = %v", err) 91 91 - } 92 92 - slot.Release() 93 93 - } 94 94 - } 95 95 - 96 96 - func TestSemaphoreSlotterTryRejectsWhenFull(t *testing.T) { 97 97 - t.Parallel() 98 98 - 99 99 - slotter := NewSemaphoreSlotter(1) 100 100 - 101 101 - first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait) 102 102 - if err != nil { 103 103 - t.Fatalf("first AcquireWorkflowSlot(NoWait) error = %v", err) 104 104 - } 105 105 - 106 106 - if _, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait); !errors.Is(err, ErrNoWorkflowSlots) { 107 107 - t.Fatalf("AcquireWorkflowSlot(NoWait) error = %v, want ErrNoWorkflowSlots", err) 108 108 - } 109 109 - 110 110 - first.Release() 111 111 - 112 112 - second, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait) 113 113 - if err != nil { 114 114 - t.Fatalf("AcquireWorkflowSlot(NoWait) after release error = %v", err) 115 115 - } 116 116 - second.Release() 117 117 - } 118 118 - 119 82 func assertNotAcquired(t *testing.T, acquired <-chan WorkflowSlot, errs <-chan error) { 120 83 t.Helper() 121 84

+2 -16

spindle/engines/dummy/engine.go

Reviewed

··· 8 8 9 9 "gopkg.in/yaml.v3" 10 10 "tangled.org/core/api/tangled" 11 11 - "tangled.org/core/spindle/engine" 12 11 "tangled.org/core/spindle/models" 13 12 "tangled.org/core/spindle/secrets" 14 13 ) ··· 17 16 // step output to the workflow logger. Useful for testing pipeline plumbing 18 17 // without a real execution backend. 19 18 type DummyEngine struct { 19 19 + l *slog.Logger 20 20 - l *slog.Logger 21 21 - StepDelay time.Duration 22 20 } 23 21 24 22 func New(l *slog.Logger) *DummyEngine { ··· 81 79 return 5 * time.Minute 82 80 } 83 81 84 84 - // no capacity limit, so always a no-op slot regardless of mode 85 85 - func (e *DummyEngine) AcquireWorkflowSlot(_ context.Context, _ models.WorkflowId, _ *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) { 86 86 - return engine.NoopSlot{}, nil 87 87 - } 88 88 - 89 82 func (e *DummyEngine) DestroyWorkflow(_ context.Context, wid models.WorkflowId) error { 90 83 e.l.Info("destroying workflow", "wid", wid) 91 84 return nil 92 85 } 93 86 87 87 + func (e *DummyEngine) RunStep(_ context.Context, wid models.WorkflowId, w *models.Workflow, idx int, _ []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { 94 94 - func (e *DummyEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, _ []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { 95 88 step := w.Steps[idx] 96 89 e.l.Info("running step", "wid", wid, "step", step.Name(), "command", step.Command()) 97 90 fmt.Fprintf(wfLogger.DataWriter(idx, "stdout"), "$ %s", step.Command()) 98 98 - if e.StepDelay > 0 { 99 99 - select { 100 100 - case <-ctx.Done(): 101 101 - return ctx.Err() 102 102 - case <-time.After(e.StepDelay): 103 103 - } 104 104 - } 105 91 return nil 106 92 }

+5 -2

spindle/engines/microvm/budget.go

Reviewed

··· 68 68 return budget, maxReq, cfg.AgingThreshold 69 69 } 70 70 71 71 + func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (engine.WorkflowSlot, error) { 71 71 - func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode engine.AcquireMode) (engine.WorkflowSlot, error) { 72 72 state, ok := wf.Data.(*workflowState) 73 73 if !ok || state == nil { 74 74 return nil, fmt.Errorf("microVM workflow state is not initialized") ··· 77 77 return engine.NoopSlot{}, nil 78 78 } 79 79 req := resourcesForImage(state.ImageSpec) 80 80 + if req.MemoryMiB < 0 || req.VCPUs < 0 || req.DiskMiB < 0 { 81 81 + return nil, fmt.Errorf("microVM resource request must not be negative: %s", req) 82 82 + } 83 83 + return e.scheduler.Acquire(ctx, req) 80 80 - return e.scheduler.Acquire(ctx, req, mode) 81 84 } 82 85 83 86 func resourcesForImage(spec ImageSpec) Resources {

+4 -4

spindle/engines/microvm/engine.go

Reviewed

··· 52 52 agent *agentHub 53 53 scheduler *engine.ResourceScheduler[Resources] 54 54 cgroupParent *CgroupParent 55 55 - budget Resources 56 56 - maxWorkflow Resources 57 55 58 56 cleanupMu sync.Mutex 59 57 cleanup map[string][]cleanupFunc ··· 93 91 db: d, 94 92 scheduler: engine.NewResourceScheduler(budget, max, agingThreshold), 95 93 cgroupParent: cgroupParent, 96 96 - budget: budget, 97 97 - maxWorkflow: max, 98 94 cleanup: make(map[string][]cleanupFunc), 99 95 }, nil 100 96 } ··· 565 561 return cleanupErr 566 562 } 567 563 564 564 + func (e *Engine) FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, w *models.Workflow, wfLogger models.WorkflowLogger) error { 565 565 + return nil 566 566 + } 567 567 + 568 568 func (e *Engine) WorkflowTimeout() time.Duration { 569 569 d, err := time.ParseDuration(e.cfg.MicroVMPipelines.WorkflowTimeout) 570 570 if err != nil {

+13 -14

spindle/engines/microvm/image.go

Reviewed

··· 240 240 } 241 241 return "", false, err 242 242 } 243 243 + if !info.IsDir() { 244 244 + return candidate, true, nil 243 243 - if info.IsDir() { 244 244 - candidate = filepath.Join(candidate, imageSpecFileName) 245 245 - info, err = os.Stat(candidate) 246 246 - if err != nil { 247 247 - if errors.Is(err, os.ErrNotExist) { 248 248 - return "", false, fmt.Errorf("microVM image directory %q does not contain %s", filepath.Dir(candidate), imageSpecFileName) 249 249 - } 250 250 - return "", false, err 251 251 - } 252 252 - if info.IsDir() { 253 253 - return "", false, fmt.Errorf("microVM image spec %q is a directory", candidate) 254 254 - } 255 245 } 256 246 247 247 + spec := filepath.Join(candidate, imageSpecFileName) 248 248 + info, err = os.Stat(spec) 257 257 - path, err := filepath.EvalSymlinks(candidate) 258 249 if err != nil { 250 250 + if errors.Is(err, os.ErrNotExist) { 251 251 + return "", false, fmt.Errorf("microVM image directory %q does not contain %s", candidate, imageSpecFileName) 252 252 + } 259 253 return "", false, err 260 254 } 255 255 + // this only happens if there is a directory named `spec.json` which would be very silly. 256 256 + // but better output an error for it anyway :p 257 257 + if info.IsDir() { 258 258 + return "", false, fmt.Errorf("microVM image spec %q is a directory", spec) 259 259 + } 260 260 + return spec, true, nil 261 261 - return path, true, nil 262 261 }

-58

spindle/engines/microvm/image_test.go

Reviewed

··· 65 65 }) 66 66 } 67 67 } 68 68 - func TestImageSpecPathPinsSymlinkTarget(t *testing.T) { 69 69 - dir := t.TempDir() 70 70 - imageA := filepath.Join(dir, "image-a") 71 71 - imageB := filepath.Join(dir, "image-b") 72 72 - writeSpecFile(t, filepath.Join(imageA, imageSpecFileName)) 73 73 - writeSpecFile(t, filepath.Join(imageB, imageSpecFileName)) 74 74 - 75 75 - alias := filepath.Join(dir, "default") 76 76 - if err := os.Symlink(imageA, alias); err != nil { 77 77 - t.Fatal(err) 78 78 - } 79 79 - got, ok, err := imageSpecPath(alias) 80 80 - if err != nil { 81 81 - t.Fatal(err) 82 82 - } 83 83 - if !ok { 84 84 - t.Fatal("imageSpecPath() did not resolve symlinked image") 85 85 - } 86 86 - 87 87 - if err := os.Remove(alias); err != nil { 88 88 - t.Fatal(err) 89 89 - } 90 90 - if err := os.Symlink(imageB, alias); err != nil { 91 91 - t.Fatal(err) 92 92 - } 93 93 - want := filepath.Join(imageA, imageSpecFileName) 94 94 - if got != want { 95 95 - t.Fatalf("imageSpecPath() = %q after alias resolution, want pinned target %q", got, want) 96 96 - } 97 97 - } 98 68 99 69 func TestResolveImageDirectoryMissingSpec(t *testing.T) { 100 70 dir := t.TempDir() ··· 165 135 t.Fatalf("spec without shell should fail validation, got: %v", err) 166 136 } 167 137 } 168 168 - func TestMkfsExt4ForVolumesSkipsLookupWithoutVolumes(t *testing.T) { 169 169 - t.Setenv("PATH", "") 170 170 - path, err := mkfsExt4ForVolumes(nil, "") 171 171 - if err != nil { 172 172 - t.Fatalf("mkfsExt4ForVolumes() with no volumes returned error: %v", err) 173 173 - } 174 174 - if path != "" { 175 175 - t.Fatalf("mkfsExt4ForVolumes() = %q with no volumes, want empty path", path) 176 176 - } 177 177 - 178 178 - _, err = mkfsExt4ForVolumes([]Volume{{Image: "workspace"}}, "") 179 179 - if err == nil || !strings.Contains(err.Error(), "mkfs.ext4") { 180 180 - t.Fatalf("mkfsExt4ForVolumes() with a volume and no formatter returned %v", err) 181 181 - } 182 182 - } 183 183 - 184 184 - func TestQEMURunnerValidateRejectsUnsupportedNetworkTypeBeforeHostChecks(t *testing.T) { 185 185 - spec := validImageSpec() 186 186 - spec.NetworkInterfaces = []NetworkInterface{{ 187 187 - Type: "tap", 188 188 - ID: "net0", 189 189 - MAC: "02:00:00:00:00:01", 190 190 - }} 191 191 - err := (qemuRunner{}).Validate(spec, false) 192 192 - if err == nil || !strings.Contains(err.Error(), `unsupported microvm network interface type "tap"`) { 193 193 - t.Fatalf("Validate() error = %v, want unsupported network type", err) 194 194 - } 195 195 - }

-15

spindle/engines/microvm/placement/placement.go

Reviewed

··· 1 1 - package placement 2 2 - 3 3 - func IsNativeArchitecture(imageArch, goArch string) bool { 4 4 - normalize := func(arch string) string { 5 5 - switch arch { 6 6 - case "x86_64", "amd64": 7 7 - return "amd64" 8 8 - case "aarch64", "arm64": 9 9 - return "arm64" 10 10 - default: 11 11 - return arch 12 12 - } 13 13 - } 14 14 - return imageArch != "" && normalize(imageArch) == normalize(goArch) 15 15 - }

-26

spindle/engines/microvm/placement/placement_test.go

Reviewed

··· 1 1 - package placement 2 2 - 3 3 - import ( 4 4 - "testing" 5 5 - ) 6 6 - 7 7 - func TestIsNativeArchitecture(t *testing.T) { 8 8 - tests := []struct { 9 9 - image string 10 10 - host string 11 11 - want bool 12 12 - }{ 13 13 - {image: "x86_64", host: "amd64", want: true}, 14 14 - {image: "amd64", host: "amd64", want: true}, 15 15 - {image: "aarch64", host: "arm64", want: true}, 16 16 - {image: "arm64", host: "arm64", want: true}, 17 17 - {image: "x86_64", host: "arm64", want: false}, 18 18 - {image: "aarch64", host: "amd64", want: false}, 19 19 - {image: "", host: "amd64", want: false}, 20 20 - } 21 21 - for _, tt := range tests { 22 22 - if got := IsNativeArchitecture(tt.image, tt.host); got != tt.want { 23 23 - t.Errorf("IsNativeArchitecture(%q, %q) = %v, want %v", tt.image, tt.host, got, tt.want) 24 24 - } 25 25 - } 26 26 - }

-61

spindle/engines/microvm/placement_linux.go

Reviewed

··· 1 1 - //go:build linux 2 2 - 3 3 - package microvm 4 4 - 5 5 - import ( 6 6 - "fmt" 7 7 - "os/exec" 8 8 - "runtime" 9 9 - 10 10 - "tangled.org/core/spindle/engines/microvm/placement" 11 11 - "tangled.org/core/spindle/models" 12 12 - ) 13 13 - 14 14 - func (e *Engine) ValidateWorkflowPlacement(wf *models.Workflow) error { 15 15 - state, ok := wf.Data.(*workflowState) 16 16 - if !ok || state == nil { 17 17 - return fmt.Errorf("microVM workflow state is not initialized") 18 18 - } 19 19 - return e.validateImagePlacement(state.ImageSpec) 20 20 - } 21 21 - 22 22 - func (e *Engine) validateImagePlacement(spec ImageSpec) error { 23 23 - if err := spec.Validate(); err != nil { 24 24 - return err 25 25 - } 26 26 - if !placement.IsNativeArchitecture(spec.Arch, runtime.GOARCH) { 27 27 - return fmt.Errorf("microVM image architecture %q is not native to executor architecture %q", spec.Arch, runtime.GOARCH) 28 28 - } 29 29 - if err := spec.validateImageFiles(); err != nil { 30 30 - return err 31 31 - } 32 32 - runner, err := runnerFor(spec.RunnerType) 33 33 - if err != nil { 34 34 - return err 35 35 - } 36 36 - if err := runner.Validate(spec, e.cfg.MicroVMPipelines.EnableKVM); err != nil { 37 37 - return err 38 38 - } 39 39 - if len(spec.Volumes) > 0 { 40 40 - if _, err := exec.LookPath("mkfs.ext4"); err != nil { 41 41 - return fmt.Errorf("required host command %q not found in PATH: %w", "mkfs.ext4", err) 42 42 - } 43 43 - for _, volume := range spec.Volumes { 44 44 - if volume.ReadOnly { 45 45 - return fmt.Errorf("read-only microvm volume %q is not supported yet", volume.Image) 46 46 - } 47 47 - if volume.FSType != "ext4" { 48 48 - return fmt.Errorf("microvm volume %q uses unsupported fsType %q", volume.Image, volume.FSType) 49 49 - } 50 50 - if volume.ImageType != "" && volume.ImageType != "raw" { 51 51 - return fmt.Errorf("microvm volume %q uses unsupported imageType %q", volume.Image, volume.ImageType) 52 52 - } 53 53 - } 54 54 - } 55 55 - 56 56 - request := resourcesForImage(spec) 57 57 - if !request.Fits(e.budget) || !request.Fits(e.maxWorkflow) { 58 58 - return fmt.Errorf("microVM image resources exceed executor limits: request=%v budget=%v max=%v", request, e.budget, e.maxWorkflow) 59 59 - } 60 60 - return nil 61 61 - }

-4

spindle/engines/microvm/qemu.go

Reviewed

··· 73 73 type qemuRunner struct{} 74 74 75 75 func (qemuRunner) Validate(spec ImageSpec, enableKVM bool) error { 76 76 - b := newArgBuilder(len(spec.NetworkInterfaces) * 4) 77 77 - if err := addQEMUNetworkArgs(&b, spec.NetworkInterfaces); err != nil { 78 78 - return err 79 79 - } 80 76 if _, err := exec.LookPath(spec.RunnerCmd()); err != nil { 81 77 return fmt.Errorf("required host command %q not found in PATH: %w", spec.RunnerCmd(), err) 82 78 }

+6 -14

spindle/engines/microvm/vm.go

Reviewed

··· 47 47 return nil 48 48 } 49 49 50 50 - func mkfsExt4ForVolumes(volumes []Volume, configured string) (string, error) { 51 51 - if len(volumes) == 0 || configured != "" { 52 52 - return configured, nil 53 53 - } 54 54 - path, err := exec.LookPath("mkfs.ext4") 55 55 - if err != nil { 56 56 - return "", fmt.Errorf("mkfs.ext4 command not found in PATH: %w", err) 57 57 - } 58 58 - return path, nil 59 59 - } 60 60 - 61 50 func prepareVolumes(ctx context.Context, workDir string, volumes []Volume, mkfsExt4 string) (map[string]string, error) { 62 51 paths := make(map[string]string, len(volumes)) 63 52 for _, volume := range volumes { ··· 372 361 return nil, err 373 362 } 374 363 364 364 + mkfsExt4 := cfg.MkfsExt4 365 365 + if mkfsExt4 == "" { 366 366 + mkfsExt4, err = exec.LookPath("mkfs.ext4") 367 367 + if err != nil { 368 368 + return nil, fmt.Errorf("mkfs.ext4 command not found in PATH: %w", err) 369 369 + } 375 375 - mkfsExt4, err := mkfsExt4ForVolumes(cfg.Image.Volumes, cfg.MkfsExt4) 376 376 - if err != nil { 377 377 - return nil, err 378 370 } 379 371 volumePaths, err := prepareVolumes(ctx, cfg.WorkDir, cfg.Image.Volumes, mkfsExt4) 380 372 if err != nil {

+1 -2

spindle/engines/nixery/engine.go

Reviewed

··· 198 198 ctx context.Context, 199 199 wid models.WorkflowId, 200 200 wf *models.Workflow, 201 201 - mode engine.AcquireMode, 202 201 ) (engine.WorkflowSlot, error) { 203 202 if e.slotter == nil { 204 203 return engine.NoopSlot{}, nil 205 204 } 206 205 206 206 + return e.slotter.AcquireWorkflowSlot(ctx, wid, wf) 207 207 - return e.slotter.AcquireWorkflowSlot(ctx, wid, wf, mode) 208 207 } 209 208 210 209 func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) (err error) {

spindle/logview/logview.go

Reviewed

This file has not been changed.

-159

spindle/mill/README.md

Reviewed

··· 1 1 - # spindle mill 2 2 - 3 3 - This document describes the architecture of the mill, spindle's distributed 4 4 - job placement layer. A mill is a spindle that doesn't run jobs itself: it 5 5 - places them on remote executors, which run the real engines (microvm, nixery, 6 6 - dummy) exactly as they would standalone. 7 7 - 8 8 - ## The engine mirroring model 9 9 - 10 10 - Neither side's execution path changes between local and remote runs. 11 11 - 12 12 - On the mill host, engines are registered under the real engine names 13 13 - ("microvm", "nixery", "dummy"), but each is a stand-in that places jobs 14 14 - remotely instead of running them. `InitWorkflow` on the stand-in parses 15 15 - nothing, it stashes the raw pipeline and workflow and builds a synthetic 16 16 - one-step workflow, so everything upstream (trigger handling, pending status, 17 17 - `TANGLED_*` env) behaves exactly like a local run. The real `InitWorkflow` 18 18 - runs exactly once, on the executor. 19 19 - 20 20 - On the executor, the reserve handler wraps the real engine in a 21 21 - `reservedEngine`: the slot acquired when the reservation is accepted is the 22 22 - same slot `StartWorkflows` later runs on. Nothing acquires twice, and from the 23 23 - engine's point of view the job is indistinguishable from a local one. 24 24 - 25 25 - The same mirroring applies to output: the executor's jobs write status rows 26 26 - and log lines exactly like a standalone spindle, and the mill re-authors them 27 27 - into its own event stream and log store, so appview and the log endpoints see 28 28 - no difference between a local and a remote job. 29 29 - 30 30 - The only real divergence: secrets are withheld from untrusted pipelines at the 31 31 - mill, and they cross the wire exactly once, inside `CommitLease`, to the one 32 32 - node that won the bid. Losing bidders never see them. 33 33 - 34 34 - ## Placement: labels, seats, and rejection 35 35 - 36 36 - Placement decides which executors can run a workflow right now, using one 37 37 - kind of fact: 38 38 - 39 39 - - **labels** are operator-defined strings on the executor's token, checked 40 40 - against the workflow's `runs_on`. they're for coarse fleet partitioning 41 41 - ("this pool is for trusted jobs", "this box has a gpu"), nothing more 42 42 - 43 43 - Matching is exact intersection: candidate sessions whose snapshot has the 44 44 - engine available, a free seat, and labels covering `runs_on`. Among the 45 45 - eligible candidates the mill ranks least-loaded first and bids the top-K 46 46 - concurrently with `ReserveSeat`. The best-ranked accept wins, losers get 47 47 - `ReleaseLease` so they free their held seats immediately. If nobody 48 48 - accepts, the job waits for a change: a new executor connecting, a snapshot 49 49 - flipping availability, or a lease finishing somewhere. 50 50 - 51 51 - Image and arch compatibility is deliberately *not* a mill concern. The mill 52 52 - never parses an image name or an arch string; the executor re-validates 53 53 - everything at reserve time against its own disk (engine exists, workflow 54 54 - parses, the image spec validates and is natively runnable, the runner is 55 55 - usable) and rejects what it can't run with `incompatible`. A reject rotates 56 56 - the bid to the next candidate, and if every candidate rejects, the user 57 57 - gets the collected reasons as the placement error. Multi-arch falls out of 58 58 - this: an arm64 box simply cannot accept an x86_64 image, so it never holds 59 59 - one. If you *want* to pin an arch or an image explicitly, label the nodes 60 60 - and use `runs_on`. 61 61 - 62 62 - ### Example: an alpine microvm job 63 63 - 64 64 - Say a workflow asks for the microvm engine with `image: alpine`, no 65 65 - `runs_on`, and the fleet looks like this: 66 66 - 67 67 - | node | arch | labels | engines | load | 68 68 - |---------|---------|---------|-------------------|--------| 69 69 - | ci-1 | x86_64 | [linux] | microvm | busy | 70 70 - | ci-2 | aarch64 | [linux] | microvm | idle | 71 71 - | ci-3 | x86_64 | [linux] | microvm, nixery | idle | 72 72 - | ci-4 | x86_64 | [gpu] | microvm | idle | 73 73 - 74 74 - The walkthrough: 75 75 - 76 76 - ```mermaid 77 77 - flowchart TD 78 78 - W["workflow<br/><small>engine: microvm, image: alpine</small>"] --> F{"candidate filter"} 79 79 - F -->|"all pass runs_on (empty)"| L{"has microvm<br/>available?"} 80 80 - L -->|all four| R{"rank by load"} 81 81 - R -->|"ci-1 busy"| X2["ci-1 ranked last"] 82 82 - R --> B["bid top-K: ci-2, ci-4, ci-1"] 83 83 - ``` 84 84 - 85 85 - Say ci-2's alpine image is actually x86_64-only: the mill offers the bid 86 86 - anyway, ci-2's reserve-time validation rejects it as incompatible, and the 87 87 - bid rotates to ci-4. The mill learns "ci-2 can't run alpine" from the 88 88 - rejection, not from any advertisement, and the reason reaches the user if 89 89 - every candidate fails the same way. 90 90 - 91 91 - The bid then runs concurrently: 92 92 - 93 93 - ```mermaid 94 94 - sequenceDiagram 95 95 - participant M as mill 96 96 - participant C2 as ci-2 97 97 - participant C4 as ci-4 98 98 - participant C1 as ci-1 99 99 - 100 100 - par bids 101 101 - M->>C2: ReserveSeat (raw pipeline+workflow) 102 102 - M->>C4: ReserveSeat 103 103 - M->>C1: ReserveSeat 104 104 - end 105 105 - C2-->>M: accept (idle) 106 106 - C4-->>M: accept 107 107 - C1-->>M: reject (transient, seats full) 108 108 - Note over M: ci-2 ranked above ci-4,<br/>ci-4 gets ReleaseLease 109 109 - M->>C2: CommitLease (secrets) 110 110 - C2-->>M: Committed 111 111 - M->>C4: ReleaseLease 112 112 - ``` 113 113 - 114 114 - Both idle nodes accepted, so rank breaks the tie: ci-2 keeps the seat, ci-4 115 115 - frees its immediately, and only ci-2 ever sees the secrets. From here ci-2 116 116 - runs the job exactly like a standalone spindle would, booting the alpine 117 117 - image under QEMU, while the mill blocks on the terminal event. 118 118 - 119 119 - ## The protocol's durability model 120 120 - 121 121 - Executor to mill is a single websocket per node, and everything the executor 122 122 - reports (status, logs, terminal results) travels as sequenced `Event`s. Two 123 123 - identifiers keep it all consistent: 124 124 - 125 125 - - the **epoch** names one lifetime of the executor process. a restarted 126 126 - executor connects with a fresh epoch, and anything arriving for an old one 127 127 - is invalid. leases are bound to node+epoch, so a zombie from a previous 128 128 - process can't act on them 129 129 - - the **seqno** is a dense per-epoch counter on events. the executor persists 130 130 - events in a local outbox before sending and trims it only when the mill 131 131 - acks. on reconnect it resumes from the mill's acked position and replays. 132 132 - the mill applies idempotently: replays drop, gaps kill the session 133 133 - 134 134 - The mill is equally restartable: leases, acked positions and the canonical 135 135 - log all live in its db. Restored leases start as orphans and must be 136 136 - reclaimed by the executor's first snapshot, or a sweep fails them after one 137 137 - grace window. The invariant throughout: at any moment, for any lease, exactly 138 138 - one of {executor outbox, mill db} holds the newest state, and the seqno/epoch 139 139 - pair says who. 140 140 - 141 141 - ## Leases 142 142 - 143 143 - A lease is the mill-side handle for one placed job: 144 144 - 145 145 - ```mermaid 146 146 - stateDiagram-v2 147 147 - [*] --> reserved: bid won 148 148 - reserved --> committing: CommitLease sent 149 149 - committing --> running: Committed 150 150 - running --> done: terminal arrived 151 151 - reserved --> done: released / expired 152 152 - committing --> done: released / expired 153 153 - ``` 154 154 - 155 155 - Commit retries ride reconnects: a reservation outlives one disconnect, so a 156 156 - lost session means wait and retry, not a failed job. What ends a job is the 157 157 - job timeout, a terminal event, or the executor being declared dead after 158 158 - reconnect grace expires, at which point every lease the node held fails 159 159 - (preserving a pending cancellation as the reason).

-482

spindle/mill/auth_test.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "io" 6 6 - "log/slog" 7 7 - "net/http" 8 8 - "net/http/httptest" 9 9 - "path/filepath" 10 10 - "strings" 11 11 - "testing" 12 12 - "time" 13 13 - 14 14 - "github.com/gorilla/websocket" 15 15 - "tangled.org/core/notifier" 16 16 - "tangled.org/core/spindle/db" 17 17 - "tangled.org/core/spindle/models" 18 18 - 19 19 - millproto "tangled.org/core/spindle/mill/proto" 20 20 - millv1 "tangled.org/core/spindle/mill/proto/gen" 21 21 - ) 22 22 - 23 23 - func discardLogger() *slog.Logger { 24 24 - return slog.New(slog.NewTextHandler(io.Discard, nil)) 25 25 - } 26 26 - 27 27 - func nopEncoder() scriptedEncoder { 28 28 - return scriptedEncoder(func(*millproto.Message) error { return nil }) 29 29 - } 30 30 - 31 31 - func TestHashToken(t *testing.T) { 32 32 - const raw = "super-secret-executor-token" 33 33 - 34 34 - if HashToken(raw) != HashToken(raw) { 35 35 - t.Fatal("HashToken is not deterministic; the same token would stop authenticating") 36 36 - } 37 37 - if HashToken("token-a") == HashToken("token-b") { 38 38 - t.Fatal("HashToken collided two distinct tokens") 39 39 - } 40 40 - if HashToken(raw) == raw { 41 41 - t.Fatal("HashToken returned the raw token; a hash leak would expose a usable credential") 42 42 - } 43 43 - } 44 44 - 45 45 - func TestGenerateTokenDistinct(t *testing.T) { 46 46 - const n = 100 47 47 - seen := make(map[string]struct{}, n) 48 48 - for i := range n { 49 49 - tok, err := GenerateToken() 50 50 - if err != nil { 51 51 - t.Fatalf("GenerateToken: %v", err) 52 52 - } 53 53 - if tok == "" { 54 54 - t.Fatalf("GenerateToken returned an empty token on call %d", i) 55 55 - } 56 56 - if _, dup := seen[tok]; dup { 57 57 - t.Fatalf("GenerateToken repeated a token after %d calls: %q", i, tok) 58 58 - } 59 59 - seen[tok] = struct{}{} 60 60 - } 61 61 - } 62 62 - 63 63 - func TestAttachSessionRejectsSecondLiveSession(t *testing.T) { 64 64 - l := discardLogger() 65 65 - m := New(l, Config{ReconnectGrace: time.Minute}) 66 66 - 67 67 - sessionOf := func(node string) *millSession { 68 68 - m.mu.Lock() 69 69 - defer m.mu.Unlock() 70 70 - return m.sessions[node] 71 71 - } 72 72 - 73 73 - sess1 := newSession("node-1", "inc-1", nil, nopEncoder(), l) 74 74 - if _, ok := m.attachSession(sess1); !ok { 75 75 - t.Fatal("first attach of a node was rejected; want accept") 76 76 - } 77 77 - if sessionOf("node-1") != sess1 { 78 78 - t.Fatal("first session was not registered as the live session") 79 79 - } 80 80 - 81 81 - sess2 := newSession("node-1", "inc-2", nil, nopEncoder(), l) 82 82 - if _, ok := m.attachSession(sess2); ok { 83 83 - t.Fatal("second live attach for an already-live node was accepted; a valid token hijacked the executor") 84 84 - } 85 85 - if sessionOf("node-1") != sess1 { 86 86 - t.Fatal("rejected newcomer evicted the incumbent session") 87 87 - } 88 88 - 89 89 - m.detachSession(sess1) 90 90 - sess3 := newSession("node-1", "inc-3", nil, nopEncoder(), l) 91 91 - if _, ok := m.attachSession(sess3); !ok { 92 92 - t.Fatal("attach during the incumbent's reconnect grace was rejected; want adopt") 93 93 - } 94 94 - if sessionOf("node-1") != sess3 { 95 95 - t.Fatal("adopted session was not installed as the live session") 96 96 - } 97 97 - } 98 98 - 99 99 - func TestOnAttemptResultIgnoresForeignLease(t *testing.T) { 100 100 - ctx := context.Background() 101 101 - l := discardLogger() 102 102 - bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 103 103 - if err != nil { 104 104 - t.Fatalf("db.Make: %v", err) 105 105 - } 106 106 - t.Cleanup(func() { bdb.Close() }) 107 107 - n := notifier.New() 108 108 - 109 109 - m := New(l, Config{ReconnectGrace: time.Minute}) 110 110 - m.Attach(bdb, &n) 111 111 - 112 112 - foreign := newLease("lease-foreign", "node-a", "inc-a", "dummy") 113 113 - m.mu.Lock() 114 114 - m.leases[foreign.id] = foreign 115 115 - m.mu.Unlock() 116 116 - 117 117 - sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l) 118 118 - m.attachSession(sessB) 119 119 - _ = m.onEventBatch(sessB, &millv1.EventBatch{ 120 120 - Epoch: sessB.epoch, 121 121 - Events: []*millv1.Event{ 122 122 - { 123 123 - Seqno: 1, 124 124 - LeaseId: foreign.id, 125 125 - Payload: &millv1.Event_AttemptResult{ 126 126 - AttemptResult: &millv1.AttemptResult{ 127 127 - Status: millv1.TerminalStatus_SUCCESS, 128 128 - }, 129 129 - }, 130 130 - }, 131 131 - }, 132 132 - }) 133 133 - 134 134 - if _, ok := pollTerminal(foreign); ok { 135 135 - t.Fatal("attempt-result on a foreign lease delivered a terminal; an executor forged another node's job result") 136 136 - } 137 137 - if foreign.getState() == leaseDone { 138 138 - t.Fatal("attempt-result on a foreign lease sealed the lease") 139 139 - } 140 140 - } 141 141 - 142 142 - func TestOnAttemptResultIgnoresAbsentLease(t *testing.T) { 143 143 - ctx := context.Background() 144 144 - l := discardLogger() 145 145 - bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 146 146 - if err != nil { 147 147 - t.Fatalf("db.Make: %v", err) 148 148 - } 149 149 - t.Cleanup(func() { bdb.Close() }) 150 150 - n := notifier.New() 151 151 - 152 152 - m := New(l, Config{ReconnectGrace: time.Minute}) 153 153 - m.Attach(bdb, &n) 154 154 - 155 155 - // bystander lease the reporting node owns, proving an absent-lease stream 156 156 - // does not spill onto another lease 157 157 - bystander := newLease("lease-bystander", "node-b", "inc-b", "dummy") 158 158 - m.mu.Lock() 159 159 - m.leases[bystander.id] = bystander 160 160 - m.mu.Unlock() 161 161 - 162 162 - sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l) 163 163 - m.attachSession(sessB) 164 164 - 165 165 - _ = m.onEventBatch(sessB, &millv1.EventBatch{ 166 166 - Epoch: sessB.epoch, 167 167 - Events: []*millv1.Event{ 168 168 - { 169 169 - Seqno: 1, 170 170 - LeaseId: "lease-nonexistent", 171 171 - Payload: &millv1.Event_AttemptResult{ 172 172 - AttemptResult: &millv1.AttemptResult{ 173 173 - Status: millv1.TerminalStatus_SUCCESS, 174 174 - }, 175 175 - }, 176 176 - }, 177 177 - }, 178 178 - }) 179 179 - 180 180 - if _, ok := pollTerminal(bystander); ok { 181 181 - t.Fatal("attempt-result for an absent lease delivered a terminal to a bystander lease") 182 182 - } 183 183 - if bystander.getState() == leaseDone { 184 184 - t.Fatal("attempt-result for an absent lease sealed a bystander lease") 185 185 - } 186 186 - } 187 187 - 188 188 - // owned-lease path proves ignore tests above are not passing merely because 189 189 - // delivery is broken. correctly owned terminal is delivered 190 190 - func TestOnAttemptResultDeliversOwnedLease(t *testing.T) { 191 191 - ctx := context.Background() 192 192 - l := discardLogger() 193 193 - bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 194 194 - if err != nil { 195 195 - t.Fatalf("db.Make: %v", err) 196 196 - } 197 197 - t.Cleanup(func() { bdb.Close() }) 198 198 - n := notifier.New() 199 199 - 200 200 - m := New(l, Config{ReconnectGrace: time.Minute}) 201 201 - m.Attach(bdb, &n) 202 202 - 203 203 - owned := newLease("lease-owned", "node-b", "inc-b", "dummy") 204 204 - m.mu.Lock() 205 205 - m.leases[owned.id] = owned 206 206 - m.mu.Unlock() 207 207 - 208 208 - sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l) 209 209 - m.attachSession(sessB) 210 210 - _ = m.onEventBatch(sessB, &millv1.EventBatch{ 211 211 - Epoch: sessB.epoch, 212 212 - Events: []*millv1.Event{ 213 213 - { 214 214 - Seqno: 1, 215 215 - LeaseId: owned.id, 216 216 - Payload: &millv1.Event_AttemptResult{ 217 217 - AttemptResult: &millv1.AttemptResult{ 218 218 - Status: millv1.TerminalStatus_SUCCESS, 219 219 - }, 220 220 - }, 221 221 - }, 222 222 - }, 223 223 - }) 224 224 - 225 225 - res, ok := pollTerminal(owned) 226 226 - if !ok { 227 227 - t.Fatal("attempt-result on an owned lease was not delivered") 228 228 - } 229 229 - if got := res.GetStatus(); got != millv1.TerminalStatus_SUCCESS { 230 230 - t.Fatalf("delivered terminal status = %v, want %v", got, millv1.TerminalStatus_SUCCESS) 231 231 - } 232 232 - if owned.getState() != leaseDone { 233 233 - t.Fatal("owned lease was not sealed after its terminal was delivered") 234 234 - } 235 235 - } 236 236 - 237 237 - func TestOnStatusEventOwnership(t *testing.T) { 238 238 - ctx := context.Background() 239 239 - l := discardLogger() 240 240 - 241 241 - bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 242 242 - if err != nil { 243 243 - t.Fatalf("db.Make: %v", err) 244 244 - } 245 245 - t.Cleanup(func() { bdb.Close() }) 246 246 - n := notifier.New() 247 247 - 248 248 - m := New(l, Config{ReconnectGrace: time.Minute}) 249 249 - m.Attach(bdb, &n) 250 250 - 251 251 - foreign := newLease("lease-foreign", "node-x", "inc-x", "dummy") 252 252 - foreign.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "foreign"}, Name: "build"} 253 253 - owned := newLease("lease-owned", "node-z", "inc-z", "dummy") 254 254 - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "owned"}, Name: "build"} 255 255 - m.mu.Lock() 256 256 - m.leases[foreign.id] = foreign 257 257 - m.leases[owned.id] = owned 258 258 - m.mu.Unlock() 259 259 - 260 260 - sessY := newSession("node-y", "inc-y", nil, nopEncoder(), l) 261 261 - m.attachSession(sessY) 262 262 - sessZ := newSession("node-z", "inc-z", nil, nopEncoder(), l) 263 263 - m.attachSession(sessZ) 264 264 - 265 265 - _ = m.onEventBatch(sessY, &millv1.EventBatch{ 266 266 - Epoch: sessY.epoch, 267 267 - Events: []*millv1.Event{ 268 268 - { 269 269 - Seqno: 1, 270 270 - LeaseId: foreign.id, 271 271 - Payload: &millv1.Event_StatusEvent{ 272 272 - StatusEvent: &millv1.StatusEvent{ 273 273 - Status: millv1.NonterminalStatus_RUNNING, 274 274 - }, 275 275 - }, 276 276 - }, 277 277 - }, 278 278 - }) 279 279 - if _, err := bdb.GetStatus(foreign.wid); err == nil { 280 280 - t.Fatal("status stream for a foreign lease authored a status row; an executor forged another pipeline's status") 281 281 - } 282 282 - 283 283 - _ = m.onEventBatch(sessZ, &millv1.EventBatch{ 284 284 - Epoch: sessZ.epoch, 285 285 - Events: []*millv1.Event{ 286 286 - { 287 287 - Seqno: 1, 288 288 - LeaseId: owned.id, 289 289 - Payload: &millv1.Event_StatusEvent{ 290 290 - StatusEvent: &millv1.StatusEvent{ 291 291 - Status: millv1.NonterminalStatus_RUNNING, 292 292 - }, 293 293 - }, 294 294 - }, 295 295 - }, 296 296 - }) 297 297 - st, err := bdb.GetStatus(owned.wid) 298 298 - if err != nil { 299 299 - t.Fatalf("owned status stream did not author a status row: %v", err) 300 300 - } 301 301 - if st.Status != "running" { 302 302 - t.Fatalf("owned status = %q, want %q", st.Status, "running") 303 303 - } 304 304 - } 305 305 - 306 306 - func setupTestServer(t *testing.T, authorizedLabels []string) (*Mill, *db.DB, *httptest.Server, string) { 307 307 - ctx := context.Background() 308 308 - bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) 309 309 - if err != nil { 310 310 - t.Fatalf("db.Make: %v", err) 311 311 - } 312 312 - t.Cleanup(func() { bdb.Close() }) 313 313 - n := notifier.New() 314 314 - 315 315 - m := New(discardLogger(), Config{ 316 316 - ReconnectGrace: time.Minute, 317 317 - }) 318 318 - m.Attach(bdb, &n) 319 319 - 320 320 - const secret = "test-secret" 321 321 - if err := bdb.AddExecutorToken("dev-node", HashToken(secret), nil, authorizedLabels); err != nil { 322 322 - t.Fatalf("AddExecutorToken: %v", err) 323 323 - } 324 324 - 325 325 - server := httptest.NewServer(http.HandlerFunc(m.HandleExecutorConn)) 326 326 - t.Cleanup(server.Close) 327 327 - 328 328 - return m, bdb, server, secret 329 329 - } 330 330 - 331 331 - func TestAuthLabelEscalation(t *testing.T) { 332 332 - _, _, server, secret := setupTestServer(t, []string{"linux", "amd64"}) 333 333 - 334 334 - wsUrl := "ws" + strings.TrimPrefix(server.URL, "http") 335 335 - 336 336 - { 337 337 - header := http.Header{} 338 338 - header.Set("Authorization", "Bearer bad-token") 339 339 - _, resp, err := websocket.DefaultDialer.Dial(wsUrl, header) 340 340 - if err == nil { 341 341 - t.Fatal("expected connection with invalid token to fail") 342 342 - } 343 343 - if resp != nil && resp.StatusCode != http.StatusUnauthorized { 344 344 - t.Fatalf("expected 401 Unauthorized, got %d", resp.StatusCode) 345 345 - } 346 346 - } 347 347 - 348 348 - { 349 349 - header := http.Header{} 350 350 - header.Set("Authorization", "Bearer "+secret) 351 351 - conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header) 352 352 - if err != nil { 353 353 - t.Fatalf("dial failed: %v", err) 354 354 - } 355 355 - defer conn.Close() 356 356 - 357 357 - stream := millproto.NewWSStream(conn) 358 358 - enc := millproto.NewEncoder(stream) 359 359 - dec := millproto.NewDecoder(stream) 360 360 - 361 361 - hello := &millproto.Message{Hello: &millv1.Hello{ 362 362 - ProtocolVersion: millproto.ProtocolVersion, 363 363 - Arch: "amd64", 364 364 - Labels: []string{"linux", "gpu"}, 365 365 - Epoch: "inc-1", 366 366 - }} 367 367 - if err := enc.Encode(hello); err != nil { 368 368 - t.Fatalf("encode hello: %v", err) 369 369 - } 370 370 - 371 371 - _, err = dec.Decode() 372 372 - if err == nil { 373 373 - t.Fatal("expected server to close connection for unauthorized label, but got a message") 374 374 - } 375 375 - } 376 376 - 377 377 - { 378 378 - header := http.Header{} 379 379 - header.Set("Authorization", "Bearer "+secret) 380 380 - conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header) 381 381 - if err != nil { 382 382 - t.Fatalf("dial failed: %v", err) 383 383 - } 384 384 - defer conn.Close() 385 385 - 386 386 - stream := millproto.NewWSStream(conn) 387 387 - enc := millproto.NewEncoder(stream) 388 388 - dec := millproto.NewDecoder(stream) 389 389 - 390 390 - hello := &millproto.Message{Hello: &millv1.Hello{ 391 391 - ProtocolVersion: millproto.ProtocolVersion, 392 392 - Arch: "amd64", 393 393 - Labels: []string{"linux"}, 394 394 - Epoch: "inc-1", 395 395 - }} 396 396 - if err := enc.Encode(hello); err != nil { 397 397 - t.Fatalf("encode hello: %v", err) 398 398 - } 399 399 - 400 400 - msg, err := dec.Decode() 401 401 - if err != nil { 402 402 - t.Fatalf("expected resume message, got error: %v", err) 403 403 - } 404 404 - res := msg.GetResume() 405 405 - if res == nil { 406 406 - t.Fatal("expected Resume message, got nil") 407 407 - } 408 408 - if res.GetEpoch() != "inc-1" { 409 409 - t.Fatalf("expected epoch inc-1, got %q", res.GetEpoch()) 410 410 - } 411 411 - } 412 412 - } 413 413 - 414 414 - func TestHandshakeTimeoutAndConcurrency(t *testing.T) { 415 415 - _, _, server, secret := setupTestServer(t, []string{"linux"}) 416 416 - wsUrl := "ws" + strings.TrimPrefix(server.URL, "http") 417 417 - 418 418 - // executor that never sends Hello is dropped after the 5s pre-hello deadline 419 419 - { 420 420 - header := http.Header{} 421 421 - header.Set("Authorization", "Bearer "+secret) 422 422 - conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header) 423 423 - if err != nil { 424 424 - t.Fatalf("dial failed: %v", err) 425 425 - } 426 426 - defer conn.Close() 427 427 - 428 428 - time.Sleep(6 * time.Second) 429 429 - 430 430 - stream := millproto.NewWSStream(conn) 431 431 - enc := millproto.NewEncoder(stream) 432 432 - hello := &millproto.Message{Hello: &millv1.Hello{ 433 433 - ProtocolVersion: millproto.ProtocolVersion, 434 434 - Arch: "amd64", 435 435 - Labels: []string{"linux"}, 436 436 - Epoch: "inc-1", 437 437 - }} 438 438 - err = enc.Encode(hello) 439 439 - dec := millproto.NewDecoder(stream) 440 440 - _, readErr := dec.Decode() 441 441 - if readErr == nil { 442 442 - t.Fatal("expected server to have closed connection due to handshake timeout") 443 443 - } 444 444 - } 445 445 - 446 446 - // second live session for one identity is rejected with 409 before the ws upgrade 447 447 - { 448 448 - header := http.Header{} 449 449 - header.Set("Authorization", "Bearer "+secret) 450 450 - 451 451 - conn1, _, err := websocket.DefaultDialer.Dial(wsUrl, header) 452 452 - if err != nil { 453 453 - t.Fatalf("dial 1 failed: %v", err) 454 454 - } 455 455 - defer conn1.Close() 456 456 - 457 457 - stream1 := millproto.NewWSStream(conn1) 458 458 - enc1 := millproto.NewEncoder(stream1) 459 459 - dec1 := millproto.NewDecoder(stream1) 460 460 - hello1 := &millproto.Message{Hello: &millv1.Hello{ 461 461 - ProtocolVersion: millproto.ProtocolVersion, 462 462 - Arch: "amd64", 463 463 - Labels: []string{"linux"}, 464 464 - Epoch: "inc-1", 465 465 - }} 466 466 - if err := enc1.Encode(hello1); err != nil { 467 467 - t.Fatalf("encode hello 1: %v", err) 468 468 - } 469 469 - _, err = dec1.Decode() 470 470 - if err != nil { 471 471 - t.Fatalf("first connection handshake failed: %v", err) 472 472 - } 473 473 - 474 474 - _, resp, err := websocket.DefaultDialer.Dial(wsUrl, header) 475 475 - if err == nil { 476 476 - t.Fatal("expected second connection for same live identity to be rejected") 477 477 - } 478 478 - if resp != nil && resp.StatusCode != http.StatusConflict { 479 479 - t.Fatalf("expected 409 Conflict for duplicate session, got %d", resp.StatusCode) 480 480 - } 481 481 - } 482 482 - }

-85

spindle/mill/engine.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "log/slog" 6 6 - "time" 7 7 - 8 8 - "tangled.org/core/api/tangled" 9 9 - "tangled.org/core/spindle/engine" 10 10 - "tangled.org/core/spindle/models" 11 11 - "tangled.org/core/spindle/secrets" 12 12 - ) 13 13 - 14 14 - // raw pipeline/workflow carried forward, executor runs the real InitWorkflow 15 15 - type millWorkflowState struct { 16 16 - RawWorkflow tangled.Pipeline_Workflow 17 17 - RawPipeline tangled.Pipeline 18 18 - Lease *RemoteLease 19 19 - } 20 20 - 21 21 - // stand-in for a real engine, registered under the real names 22 22 - // ("microvm", "nixery"), all sharing one Mill 23 23 - type Engine struct { 24 24 - name string 25 25 - mill *Mill 26 26 - l *slog.Logger 27 27 - } 28 28 - 29 29 - func (e *Engine) AuthorsRemoteStatus() {} 30 30 - 31 31 - func NewEngine(name string, mill *Mill) *Engine { 32 32 - return &Engine{name: name, mill: mill, l: mill.l.With("engine", "mill:"+name)} 33 33 - } 34 34 - 35 35 - // synthetic one-step workflow so processPipeline injects TANGLED_* env 36 36 - // and marks pending normally. the real InitWorkflow runs exactly once, on 37 37 - // the executor inside ReserveSeat, and commit reuses that workflow 38 38 - func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { 39 39 - return &models.Workflow{ 40 40 - Name: twf.Name, 41 41 - Environment: map[string]string{}, 42 42 - Steps: []models.Step{remoteStep{}}, 43 43 - Data: &millWorkflowState{ 44 44 - RawWorkflow: twf, 45 45 - RawPipeline: tpl, 46 46 - }, 47 47 - }, nil 48 48 - } 49 49 - 50 50 - // no-op logger for the synthetic workflow. the executor's lines stream 51 51 - // into this wid's log directly, a local logger would just write competing 52 52 - // lines 53 53 - func (e *Engine) WorkflowLogger(wid models.WorkflowId) models.WorkflowLogger { 54 54 - return models.NullLogger{} 55 55 - } 56 56 - 57 57 - // the placement seam, blocks on remote placement which the user sees as 58 58 - // "pending". only StartWorkflows calls this, always Wait 59 59 - func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) { 60 60 - return e.mill.place(ctx, e.name, wid, wf) 61 61 - } 62 62 - 63 63 - // no-op. real setup happens on the executor 64 64 - func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error { 65 65 - e.l.Info("remote job placed, awaiting commit", "wid", wid) 66 66 - return nil 67 67 - } 68 68 - 69 69 - // hands over the secrets and blocks on the terminal result streamed over the 70 70 - // session 71 71 - func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, unlocked []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { 72 72 - return e.mill.commitAndWait(ctx, w, unlocked) 73 73 - } 74 74 - 75 75 - // deliberately generous, the executor enforces the real timeout. the mill 76 76 - // only caps a hung or silent executor, true death is caught by reconnect grace 77 77 - func (e *Engine) WorkflowTimeout() time.Duration { 78 78 - return e.mill.cfg.JobTimeout 79 79 - } 80 80 - 81 81 - // cancels a still-running attempt. no-op if already terminal 82 82 - func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { 83 83 - e.mill.destroy(wid) 84 84 - return nil 85 85 - }

-119

spindle/mill/executor/capability_test.go

Reviewed

··· 1 1 - package executor 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "encoding/json" 6 6 - "errors" 7 7 - "io" 8 8 - "log/slog" 9 9 - "testing" 10 10 - 11 11 - "tangled.org/core/api/tangled" 12 12 - millv1 "tangled.org/core/spindle/mill/proto/gen" 13 13 - "tangled.org/core/spindle/models" 14 14 - ) 15 15 - 16 16 - type placementEngine struct { 17 17 - *fakeEngine 18 18 - validationErr error 19 19 - validated bool 20 20 - } 21 21 - 22 22 - func (e *placementEngine) ValidateWorkflowPlacement(*models.Workflow) error { 23 23 - e.validated = true 24 24 - return e.validationErr 25 25 - } 26 26 - 27 27 - func TestHandleReserveRejectsMissingTriggerMetadata(t *testing.T) { 28 28 - enc := newCaptureEncoder() 29 29 - e := &Executor{ 30 30 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 31 31 - enc: enc, 32 32 - seats: 1, 33 33 - engines: map[string]models.Engine{"microvm": &fakeEngine{}}, 34 34 - active: make(map[string]*reservation), 35 35 - } 36 36 - twf, err := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) 37 37 - if err != nil { 38 38 - t.Fatal(err) 39 39 - } 40 40 - tpl, err := json.Marshal(tangled.Pipeline{}) 41 41 - if err != nil { 42 42 - t.Fatal(err) 43 43 - } 44 44 - 45 45 - // engines deref TriggerMetadata unconditionally. this must be a reject, 46 46 - // not a panic that takes the whole executor down 47 47 - e.handleReserve(context.Background(), &millv1.ReserveSeat{ 48 48 - LeaseId: "lease-1", 49 49 - TargetEngine: "microvm", 50 50 - RawWorkflowJson: string(twf), 51 51 - RawPipelineJson: string(tpl), 52 52 - Knot: "k", 53 53 - Rkey: "r", 54 54 - }) 55 55 - 56 56 - result := (<-enc.messages).GetReserveResult() 57 57 - if result == nil { 58 58 - t.Fatal("handleReserve() did not send ReserveResult") 59 59 - } 60 60 - if result.GetAccepted() { 61 61 - t.Fatal("handleReserve() accepted a pipeline without trigger metadata") 62 62 - } 63 63 - if result.GetRejectClass() != millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE { 64 64 - t.Fatalf("reject class = %v, want incompatible", result.GetRejectClass()) 65 65 - } 66 66 - if len(e.active) != 0 { 67 67 - t.Fatalf("active reservations = %d, want 0", len(e.active)) 68 68 - } 69 69 - } 70 70 - 71 71 - func TestHandleReserveValidatesPlacementBeforeAcquiringSlot(t *testing.T) { 72 72 - enc := newCaptureEncoder() 73 73 - validationErr := errors.New("image architecture is not native") 74 74 - eng := &placementEngine{fakeEngine: &fakeEngine{}, validationErr: validationErr} 75 75 - e := &Executor{ 76 76 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 77 77 - enc: enc, 78 78 - seats: 1, 79 79 - engines: map[string]models.Engine{"microvm": eng}, 80 80 - active: make(map[string]*reservation), 81 81 - } 82 82 - twf, err := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) 83 83 - if err != nil { 84 84 - t.Fatal(err) 85 85 - } 86 86 - tpl, err := json.Marshal(tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) 87 87 - if err != nil { 88 88 - t.Fatal(err) 89 89 - } 90 90 - 91 91 - e.handleReserve(context.Background(), &millv1.ReserveSeat{ 92 92 - LeaseId: "lease-1", 93 93 - TargetEngine: "microvm", 94 94 - RawWorkflowJson: string(twf), 95 95 - RawPipelineJson: string(tpl), 96 96 - Knot: "k", 97 97 - Rkey: "r", 98 98 - }) 99 99 - 100 100 - result := (<-enc.messages).GetReserveResult() 101 101 - if result == nil { 102 102 - t.Fatal("handleReserve() did not send ReserveResult") 103 103 - } 104 104 - if result.GetAccepted() { 105 105 - t.Fatal("handleReserve() accepted placement validation failure") 106 106 - } 107 107 - if result.GetRejectClass() != millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE { 108 108 - t.Fatalf("reject class = %v, want incompatible", result.GetRejectClass()) 109 109 - } 110 110 - if !eng.validated { 111 111 - t.Fatal("placement validator was not called") 112 112 - } 113 113 - if eng.acquireCalled { 114 114 - t.Fatal("slot acquisition ran after placement validation failed") 115 115 - } 116 116 - if len(e.active) != 0 { 117 117 - t.Fatalf("active reservations = %d, want 0", len(e.active)) 118 118 - } 119 119 - }

-613

spindle/mill/executor/executor.go

Reviewed

··· 1 1 - package executor 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "encoding/json" 6 6 - "errors" 7 7 - "fmt" 8 8 - "maps" 9 9 - "net/http" 10 10 - "runtime" 11 11 - "sync" 12 12 - "time" 13 13 - 14 14 - "github.com/bluesky-social/indigo/atproto/syntax" 15 15 - "log/slog" 16 16 - "strings" 17 17 - 18 18 - "tangled.org/core/api/tangled" 19 19 - "tangled.org/core/netutil" 20 20 - "tangled.org/core/notifier" 21 21 - "tangled.org/core/spindle/artifactstore" 22 22 - "tangled.org/core/spindle/config" 23 23 - "tangled.org/core/spindle/db" 24 24 - "tangled.org/core/spindle/engine" 25 25 - millproto "tangled.org/core/spindle/mill/proto" 26 26 - millv1 "tangled.org/core/spindle/mill/proto/gen" 27 27 - "tangled.org/core/spindle/models" 28 28 - ) 29 29 - 30 30 - const ( 31 31 - dialBackoffMin = 1 * time.Second 32 32 - dialBackoffMax = 30 * time.Second 33 33 - snapshotEvery = 15 * time.Second 34 34 - defaultSeats = 4 35 35 - ) 36 36 - 37 37 - type Executor struct { 38 38 - millURL string 39 39 - token string 40 40 - nodeID string 41 41 - seats int 42 42 - labels []string 43 43 - 44 44 - engines map[string]models.Engine 45 45 - db *db.DB 46 46 - n *notifier.Notifier 47 47 - cfg *config.Config 48 48 - l *slog.Logger 49 49 - writer artifactstore.Writer 50 50 - 51 51 - epoch string 52 52 - outboxBytes int64 53 53 - maxOutboxBytes int64 // 10 MiB default outbox cap 54 54 - 55 55 - eventMu sync.Mutex 56 56 - sendMu sync.Mutex 57 57 - flushMu sync.Mutex 58 58 - 59 59 - sentSeqno uint64 60 60 - 61 61 - connMu sync.Mutex 62 62 - enc messageEncoder 63 63 - sessionCancel context.CancelFunc 64 64 - 65 65 - mu sync.Mutex 66 66 - active map[string]*reservation 67 67 - draining bool 68 68 - 69 69 - snapshotMu sync.Mutex 70 70 - nextSeqno uint64 71 71 - 72 72 - lifecycleCtx context.Context 73 73 - jobsWG sync.WaitGroup 74 74 - } 75 75 - 76 76 - type reservation struct { 77 77 - leaseID string 78 78 - wid models.WorkflowId 79 79 - realEngine models.Engine 80 80 - slot engine.WorkflowSlot 81 81 - wf *models.Workflow 82 82 - repoDid syntax.DID 83 83 - vault *memVault 84 84 - 85 85 - committed bool 86 86 - cancelled bool 87 87 - cancel context.CancelFunc 88 88 - ttlTimer *time.Timer 89 89 - stopTail func() 90 90 - } 91 91 - 92 92 - type messageEncoder interface { 93 93 - Encode(*millproto.Message) error 94 94 - } 95 95 - 96 96 - func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *notifier.Notifier, l *slog.Logger, writers ...artifactstore.Writer) (*Executor, error) { 97 97 - seats := defaultSeats 98 98 - millURL := "" 99 99 - token := "" 100 100 - nodeID := "" 101 101 - var labels []string 102 102 - if cfg != nil { 103 103 - if cfg.Mill.Seats > 0 { 104 104 - seats = cfg.Mill.Seats 105 105 - } 106 106 - labels = normalizeLabels(cfg.Mill.Labels) 107 107 - millURL = cfg.Mill.URL 108 108 - token = cfg.Mill.SharedSecret 109 109 - nodeID = cfg.Server.Hostname 110 110 - } 111 111 - if d == nil || n == nil { 112 112 - return nil, fmt.Errorf("executor requires a database and notifier") 113 113 - } 114 114 - var writer artifactstore.Writer 115 115 - if len(writers) > 0 { 116 116 - writer = writers[0] 117 117 - } 118 118 - e := &Executor{ 119 119 - millURL: millURL, 120 120 - token: token, 121 121 - nodeID: nodeID, 122 122 - seats: seats, 123 123 - labels: labels, 124 124 - engines: engines, 125 125 - db: d, 126 126 - n: n, 127 127 - cfg: cfg, 128 128 - l: l.With("component", "mill.executor"), 129 129 - writer: writer, 130 130 - active: make(map[string]*reservation), 131 131 - maxOutboxBytes: 10 * 1024 * 1024, 132 132 - } 133 133 - if err := e.initOutbox(); err != nil { 134 134 - return nil, fmt.Errorf("initialize executor outbox: %w", err) 135 135 - } 136 136 - return e, nil 137 137 - } 138 138 - 139 139 - func (e *Executor) Connect(ctx context.Context) { 140 140 - e.lifecycleCtx = ctx 141 141 - sub := e.n.Subscribe() 142 142 - cursor, err := e.db.EventHighWater() 143 143 - if err != nil { 144 144 - e.n.Unsubscribe(sub) 145 145 - e.l.Error("establish event cursor failed", "err", err) 146 146 - return 147 147 - } 148 148 - e.drainEvents(&cursor) 149 149 - observerCtx, stopObserver := context.WithCancel(ctx) 150 150 - observerDone := make(chan struct{}) 151 151 - go func() { 152 152 - defer close(observerDone) 153 153 - e.observeLoop(observerCtx, sub, cursor) 154 154 - }() 155 155 - defer e.n.Unsubscribe(sub) 156 156 - 157 157 - backoff := dialBackoffMin 158 158 - for { 159 159 - if ctx.Err() != nil { 160 160 - break 161 161 - } 162 162 - err := e.runSession(ctx) 163 163 - if ctx.Err() != nil { 164 164 - break 165 165 - } 166 166 - e.l.Warn("mill session ended; reconnecting", "err", err, "backoff", backoff) 167 167 - select { 168 168 - case <-ctx.Done(): 169 169 - break 170 170 - case <-time.After(backoff): 171 171 - } 172 172 - backoff = min(backoff*2, dialBackoffMax) 173 173 - } 174 174 - 175 175 - e.jobsWG.Wait() 176 176 - stopObserver() 177 177 - <-observerDone 178 178 - e.drainEvents(&cursor) 179 179 - } 180 180 - 181 181 - func (e *Executor) runSession(ctx context.Context) error { 182 182 - dev := e.cfg == nil || e.cfg.Server.Dev 183 183 - if _, err := netutil.EnforceWSSURL(e.millURL, dev); err != nil { 184 184 - return fmt.Errorf("mill url: %w", err) 185 185 - } 186 186 - header := http.Header{} 187 187 - if e.token != "" { 188 188 - header.Set("Authorization", "Bearer "+e.token) 189 189 - } 190 190 - conn, _, err := netutil.SSRFWebsocketDialer(dev).DialContext(ctx, e.millURL, header) 191 191 - if err != nil { 192 192 - return fmt.Errorf("dial mill: %w", err) 193 193 - } 194 194 - defer conn.Close() 195 195 - 196 196 - sessionCtx, cancelSession := context.WithCancel(ctx) 197 197 - defer cancelSession() 198 198 - stopClose := context.AfterFunc(sessionCtx, func() { _ = conn.Close() }) 199 199 - defer stopClose() 200 200 - 201 201 - stream := millproto.NewWSStream(conn) 202 202 - enc := millproto.NewEncoder(stream) 203 203 - dec := millproto.NewDecoder(stream) 204 204 - 205 205 - hello := &millproto.Message{Hello: &millv1.Hello{ 206 206 - ProtocolVersion: millproto.ProtocolVersion, 207 207 - Arch: runtime.GOARCH, 208 208 - Labels: e.labels, 209 209 - Epoch: e.epoch, 210 210 - }} 211 211 - if err := enc.Encode(hello); err != nil { 212 212 - return fmt.Errorf("send hello: %w", err) 213 213 - } 214 214 - 215 215 - resumeMsg, err := dec.Decode() 216 216 - if err != nil { 217 217 - return fmt.Errorf("read resume: %w", err) 218 218 - } 219 219 - resume := resumeMsg.GetResume() 220 220 - if resume == nil { 221 221 - return fmt.Errorf("expected resume, got something else") 222 222 - } 223 223 - if resume.GetEpoch() != e.epoch { 224 224 - return fmt.Errorf("resume epoch mismatch: got %q, want %q", resume.GetEpoch(), e.epoch) 225 225 - } 226 226 - 227 227 - e.connMu.Lock() 228 228 - e.sessionCancel = cancelSession 229 229 - e.enc = enc 230 230 - e.connMu.Unlock() 231 231 - defer func() { 232 232 - e.connMu.Lock() 233 233 - e.enc = nil 234 234 - e.sessionCancel = nil 235 235 - e.connMu.Unlock() 236 236 - }() 237 237 - 238 238 - readErr := make(chan error, 1) 239 239 - go func() { 240 240 - for { 241 241 - msg, err := dec.Decode() 242 242 - if err != nil { 243 243 - readErr <- fmt.Errorf("read: %w", err) 244 244 - return 245 245 - } 246 246 - e.dispatch(sessionCtx, msg) 247 247 - } 248 248 - }() 249 249 - 250 250 - if err := e.replay(resume.GetAckSeqno()); err != nil { 251 251 - cancelSession() 252 252 - <-readErr 253 253 - return fmt.Errorf("replay failed: %w", err) 254 254 - } 255 255 - e.pushSnapshot() 256 256 - e.l.Info("connected to mill", "node", e.nodeID, "resumeFrom", resume.GetAckSeqno()) 257 257 - 258 258 - go e.snapshotLoop(sessionCtx, enc) 259 259 - return <-readErr 260 260 - } 261 261 - func (e *Executor) send(msg *millproto.Message) { 262 262 - e.connMu.Lock() 263 263 - enc := e.enc 264 264 - cancel := e.sessionCancel 265 265 - e.connMu.Unlock() 266 266 - if enc != nil { 267 267 - e.sendMu.Lock() 268 268 - err := enc.Encode(msg) 269 269 - e.sendMu.Unlock() 270 270 - if err != nil { 271 271 - e.l.Error("send failed, ending session", "err", err) 272 272 - if cancel != nil { 273 273 - cancel() 274 274 - } 275 275 - } 276 276 - } 277 277 - } 278 278 - 279 279 - func (e *Executor) dispatch(ctx context.Context, msg *millproto.Message) { 280 280 - switch { 281 281 - case msg.GetReserveSeat() != nil: 282 282 - e.handleReserve(ctx, msg.GetReserveSeat()) 283 283 - case msg.GetCommitLease() != nil: 284 284 - e.handleCommit(ctx, msg.GetCommitLease()) 285 285 - case msg.GetReleaseLease() != nil: 286 286 - e.handleRelease(msg.GetReleaseLease().GetLeaseId()) 287 287 - case msg.GetCancelAttempt() != nil: 288 288 - e.handleCancel(msg.GetCancelAttempt().GetLeaseId()) 289 289 - case msg.GetAck() != nil: 290 290 - e.handleAck(msg.GetAck()) 291 291 - default: 292 292 - e.l.Warn("unhandled incoming message", "type", fmt.Sprintf("%T", msg)) 293 293 - } 294 294 - } 295 295 - 296 296 - func (e *Executor) sendReject(leaseID string, reason string, class millv1.RejectClass) { 297 297 - e.send(&millproto.Message{ReserveResult: &millv1.ReserveResult{ 298 298 - LeaseId: leaseID, 299 299 - Accepted: false, 300 300 - RejectReason: reason, 301 301 - RejectClass: class, 302 302 - }}) 303 303 - } 304 304 - 305 305 - func (e *Executor) sendCommitted(leaseID string) { 306 306 - e.send(&millproto.Message{Committed: &millv1.Committed{LeaseId: leaseID}}) 307 307 - } 308 308 - 309 309 - func (e *Executor) sendCancelAck(leaseID string) { 310 310 - e.send(&millproto.Message{CancelAck: &millv1.CancelAck{LeaseId: leaseID}}) 311 311 - } 312 312 - 313 313 - func (e *Executor) releaseReservation(cleanup func()) { 314 314 - if cleanup != nil { 315 315 - cleanup() 316 316 - } 317 317 - e.pushSnapshot() 318 318 - } 319 319 - 320 320 - func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { 321 321 - reject := func(reason string, class millv1.RejectClass) { 322 322 - e.sendReject(rs.GetLeaseId(), reason, class) 323 323 - } 324 324 - 325 325 - e.mu.Lock() 326 326 - draining := e.draining 327 327 - e.mu.Unlock() 328 328 - if draining { 329 329 - reject("draining", millv1.RejectClass_REJECT_CLASS_TRANSIENT) 330 330 - return 331 331 - } 332 332 - 333 333 - realEngine, ok := e.engines[rs.GetTargetEngine()] 334 334 - if !ok { 335 335 - reject("unknown engine "+rs.GetTargetEngine(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 336 336 - return 337 337 - } 338 338 - slotter, ok := realEngine.(engine.WorkflowSlotter) 339 339 - if !ok { 340 340 - reject("engine does not support workflow slots", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 341 341 - return 342 342 - } 343 343 - 344 344 - var twf tangled.Pipeline_Workflow 345 345 - if err := json.Unmarshal([]byte(rs.GetRawWorkflowJson()), &twf); err != nil { 346 346 - reject("bad workflow json", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 347 347 - return 348 348 - } 349 349 - var tpl tangled.Pipeline 350 350 - if err := json.Unmarshal([]byte(rs.GetRawPipelineJson()), &tpl); err != nil { 351 351 - reject("bad pipeline json", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 352 352 - return 353 353 - } 354 354 - if tpl.TriggerMetadata == nil { 355 355 - reject("pipeline missing trigger metadata", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 356 356 - return 357 357 - } 358 358 - 359 359 - pipelineId := models.PipelineId{Knot: rs.GetKnot(), Rkey: rs.GetRkey()} 360 360 - wid := models.WorkflowId{PipelineId: pipelineId, Name: twf.Name} 361 361 - 362 362 - wf, err := realEngine.InitWorkflow(twf, tpl) 363 363 - if err != nil { 364 364 - reject("init workflow: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 365 365 - return 366 366 - } 367 367 - if validator, ok := realEngine.(engine.WorkflowPlacementValidator); ok { 368 368 - if err := validator.ValidateWorkflowPlacement(wf); err != nil { 369 369 - reject("validate workflow placement: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) 370 370 - return 371 371 - } 372 372 - } 373 373 - if wf.Environment == nil { 374 374 - wf.Environment = make(map[string]string) 375 375 - } 376 376 - maps.Copy(wf.Environment, models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId)) 377 377 - 378 378 - slot, err := slotter.AcquireWorkflowSlot(ctx, wid, wf, engine.NoWait) 379 379 - if err != nil { 380 380 - class := millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE 381 381 - if errors.Is(err, engine.ErrNoWorkflowSlots) { 382 382 - class = millv1.RejectClass_REJECT_CLASS_TRANSIENT 383 383 - } 384 384 - reject(err.Error(), class) 385 385 - return 386 386 - } 387 387 - 388 388 - var repoDid syntax.DID 389 389 - if tpl.TriggerMetadata != nil && tpl.TriggerMetadata.Repo != nil && tpl.TriggerMetadata.Repo.RepoDid != nil { 390 390 - repoDid, _ = syntax.ParseDID(*tpl.TriggerMetadata.Repo.RepoDid) 391 391 - } 392 392 - 393 393 - res := &reservation{ 394 394 - leaseID: rs.GetLeaseId(), 395 395 - wid: wid, 396 396 - realEngine: realEngine, 397 397 - slot: slot, 398 398 - wf: wf, 399 399 - repoDid: repoDid, 400 400 - } 401 401 - 402 402 - e.snapshotMu.Lock() 403 403 - e.mu.Lock() 404 404 - e.active[res.leaseID] = res 405 405 - res.ttlTimer = time.AfterFunc(ttlDuration(rs.GetTtlSeconds()), func() { e.expireReservation(res.leaseID) }) 406 406 - e.mu.Unlock() 407 407 - 408 408 - e.send(&millproto.Message{ReserveResult: &millv1.ReserveResult{ 409 409 - LeaseId: rs.GetLeaseId(), 410 410 - Accepted: true, 411 411 - }}) 412 412 - e.pushSnapshotLocked() 413 413 - e.snapshotMu.Unlock() 414 414 - } 415 415 - 416 416 - func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) { 417 417 - e.mu.Lock() 418 418 - res := e.active[cl.GetLeaseId()] 419 419 - if res == nil { 420 420 - e.mu.Unlock() 421 421 - e.sendReject(cl.GetLeaseId(), "reservation missing or expired", millv1.RejectClass_REJECT_CLASS_TRANSIENT) 422 422 - return 423 423 - } 424 424 - if res.committed { 425 425 - e.mu.Unlock() 426 426 - e.sendCommitted(cl.GetLeaseId()) 427 427 - return 428 428 - } 429 429 - res.committed = true 430 430 - if res.ttlTimer != nil { 431 431 - res.ttlTimer.Stop() 432 432 - } 433 433 - 434 434 - jobCtx, cancel := context.WithCancel(e.lifecycleCtx) 435 435 - res.cancel = cancel 436 436 - e.mu.Unlock() 437 437 - 438 438 - vault := newMemVault(cl.GetSecrets()) 439 439 - re := newReservedEngine(res.realEngine, res.slot) 440 440 - pipeline := &models.Pipeline{ 441 441 - RepoDid: res.repoDid, 442 442 - Workflows: map[models.Engine][]models.Workflow{re: {*res.wf}}, 443 443 - TrustedSource: true, 444 444 - } 445 445 - 446 446 - e.startTail(res) 447 447 - 448 448 - e.jobsWG.Add(1) 449 449 - go func() { 450 450 - defer e.jobsWG.Done() 451 451 - engine.StartWorkflows(e.l, vault, e.cfg, nil, e.db, e.n, jobCtx, pipeline, res.wid.PipelineId) 452 452 - }() 453 453 - 454 454 - e.sendCommitted(cl.GetLeaseId()) 455 455 - } 456 456 - 457 457 - func (e *Executor) handleRelease(leaseID string) { 458 458 - cleanup, ok := e.takeUncommittedReservation(leaseID, true) 459 459 - if !ok { 460 460 - return 461 461 - } 462 462 - e.releaseReservation(cleanup) 463 463 - } 464 464 - 465 465 - func (e *Executor) handleCancel(leaseID string) { 466 466 - e.mu.Lock() 467 467 - res := e.active[leaseID] 468 468 - if res == nil { 469 469 - e.mu.Unlock() 470 470 - if err := e.appendTerminal(leaseID, string(models.StatusKindCancelled), nil); err != nil { 471 471 - e.l.Error("persist cancelled reservation terminal", "lease", leaseID, "err", err) 472 472 - return 473 473 - } 474 474 - e.sendCancelAck(leaseID) 475 475 - return 476 476 - } 477 477 - res.cancelled = true 478 478 - cancel := res.cancel 479 479 - committed := res.committed 480 480 - var cleanup func() 481 481 - if !committed { 482 482 - cleanup = e.removeReservationLocked(res, true) 483 483 - } 484 484 - e.mu.Unlock() 485 485 - 486 486 - if !committed { 487 487 - if err := e.appendTerminal(leaseID, string(models.StatusKindCancelled), nil); err != nil { 488 488 - e.l.Error("persist cancelled reservation terminal", "lease", leaseID, "err", err) 489 489 - e.releaseReservation(cleanup) 490 490 - return 491 491 - } 492 492 - e.sendCancelAck(leaseID) 493 493 - e.releaseReservation(cleanup) 494 494 - return 495 495 - } 496 496 - 497 497 - e.sendCancelAck(leaseID) 498 498 - if cancel != nil { 499 499 - cancel() 500 500 - } 501 501 - } 502 502 - 503 503 - func (e *Executor) expireReservation(leaseID string) { 504 504 - cleanup, ok := e.takeUncommittedReservation(leaseID, true) 505 505 - if !ok { 506 506 - return 507 507 - } 508 508 - e.releaseReservation(cleanup) 509 509 - } 510 510 - 511 511 - func (e *Executor) takeUncommittedReservation(leaseID string, releaseSlot bool) (func(), bool) { 512 512 - e.mu.Lock() 513 513 - defer e.mu.Unlock() 514 514 - res := e.active[leaseID] 515 515 - if res == nil || res.committed { 516 516 - return nil, false 517 517 - } 518 518 - if res.ttlTimer != nil { 519 519 - res.ttlTimer.Stop() 520 520 - } 521 521 - return e.removeReservationLocked(res, releaseSlot), true 522 522 - } 523 523 - 524 524 - func (e *Executor) removeReservationLocked(res *reservation, releaseSlot bool) func() { 525 525 - delete(e.active, res.leaseID) 526 526 - slot := res.slot 527 527 - return func() { 528 528 - if releaseSlot && slot != nil { 529 529 - slot.Release() 530 530 - } 531 531 - } 532 532 - } 533 533 - 534 534 - func (e *Executor) snapshotLoop(ctx context.Context, enc *millproto.Encoder) { 535 535 - ticker := time.NewTicker(snapshotEvery) 536 536 - defer ticker.Stop() 537 537 - 538 538 - for { 539 539 - select { 540 540 - case <-ctx.Done(): 541 541 - return 542 542 - case <-ticker.C: 543 543 - e.pushSnapshot() 544 544 - } 545 545 - } 546 546 - } 547 547 - 548 548 - func (e *Executor) pushSnapshot() { 549 549 - e.snapshotMu.Lock() 550 550 - defer e.snapshotMu.Unlock() 551 551 - e.pushSnapshotLocked() 552 552 - } 553 553 - 554 554 - func (e *Executor) pushSnapshotLocked() { 555 555 - e.mu.Lock() 556 556 - activeLeases := make([]string, 0, len(e.active)) 557 557 - for leaseID := range e.active { 558 558 - activeLeases = append(activeLeases, leaseID) 559 559 - } 560 560 - e.mu.Unlock() 561 561 - 562 562 - avail := make(map[string]*millv1.EngineAvailability) 563 563 - for name, eng := range e.engines { 564 564 - a := &millv1.EngineAvailability{Available: true} 565 565 - if getter, ok := eng.(interface{ Load() map[string]float64 }); ok { 566 566 - a.Load = getter.Load() 567 567 - } 568 568 - avail[name] = a 569 569 - } 570 570 - 571 571 - e.nextSeqno++ 572 572 - snap := &millproto.Message{ 573 573 - NodeSnapshot: &millv1.NodeSnapshot{ 574 574 - Seqno: e.nextSeqno, 575 575 - Engines: avail, 576 576 - ActiveLeaseIds: activeLeases, 577 577 - }, 578 578 - } 579 579 - e.send(snap) 580 580 - } 581 581 - 582 582 - func (e *Executor) Drain() { 583 583 - e.mu.Lock() 584 584 - e.draining = true 585 585 - e.mu.Unlock() 586 586 - e.pushSnapshot() 587 587 - } 588 588 - 589 589 - func ttlDuration(secs uint32) time.Duration { 590 590 - if secs == 0 { 591 591 - return defaultReservationTTL 592 592 - } 593 593 - return time.Duration(secs) * time.Second 594 594 - } 595 595 - 596 596 - const defaultReservationTTL = 60 * time.Second 597 597 - 598 598 - func normalizeLabels(labels []string) []string { 599 599 - seen := make(map[string]struct{}, len(labels)) 600 600 - out := make([]string, 0, len(labels)) 601 601 - for _, label := range labels { 602 602 - label = strings.TrimSpace(label) 603 603 - if label == "" { 604 604 - continue 605 605 - } 606 606 - if _, ok := seen[label]; ok { 607 607 - continue 608 608 - } 609 609 - seen[label] = struct{}{} 610 610 - out = append(out, label) 611 611 - } 612 612 - return out 613 613 - }

-251

spindle/mill/executor/observe.go

Reviewed

··· 1 1 - package executor 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "crypto/sha256" 6 6 - "encoding/hex" 7 7 - "fmt" 8 8 - "io" 9 9 - "os" 10 10 - "strings" 11 11 - "sync" 12 12 - "time" 13 13 - 14 14 - "github.com/hpcloud/tail" 15 15 - "tangled.org/core/api/tangled" 16 16 - millproto "tangled.org/core/spindle/mill/proto" 17 17 - millv1 "tangled.org/core/spindle/mill/proto/gen" 18 18 - "tangled.org/core/spindle/models" 19 19 - "tangled.org/core/spindle/secrets" 20 20 - ) 21 21 - 22 22 - func (e *Executor) observeLoop(ctx context.Context, sub <-chan struct{}, cursor int64) { 23 23 - ticker := time.NewTicker(5 * time.Second) 24 24 - defer ticker.Stop() 25 25 - 26 26 - for { 27 27 - select { 28 28 - case <-ctx.Done(): 29 29 - return 30 30 - case <-sub: 31 31 - case <-ticker.C: 32 32 - } 33 33 - e.drainEvents(&cursor) 34 34 - } 35 35 - } 36 36 - 37 37 - func (e *Executor) drainEvents(cursor *int64) { 38 38 - events, err := e.db.GetEvents(*cursor, 128) 39 39 - if err != nil { 40 40 - e.l.Error("drain status events failed", "err", err) 41 41 - return 42 42 - } 43 43 - for _, ev := range events { 44 44 - if ev.Created > *cursor { 45 45 - *cursor = ev.Created 46 46 - } 47 47 - st, ok := parseStatus(ev.EventJson) 48 48 - if !ok { 49 49 - continue 50 50 - } 51 51 - if err := e.onStatusRow(st); err != nil { 52 52 - e.l.Error("process status row failed", "err", err) 53 53 - } 54 54 - } 55 55 - } 56 56 - 57 57 - func (e *Executor) onStatusRow(st *tangled.PipelineStatus) error { 58 58 - res := e.reservationFor(st.Pipeline, st.Workflow) 59 59 - if res == nil { 60 60 - return nil 61 61 - } 62 62 - 63 63 - if models.StatusKind(st.Status).IsFinish() { 64 64 - return e.finishJob(res, st) 65 65 - } 66 66 - 67 67 - return e.appendStatus(res.leaseID, st) 68 68 - } 69 69 - 70 70 - func (e *Executor) finishJob(res *reservation, st *tangled.PipelineStatus) error { 71 71 - e.mu.Lock() 72 72 - if e.active[res.leaseID] != res { 73 73 - e.mu.Unlock() 74 74 - return nil 75 75 - } 76 76 - cancelled := res.cancelled 77 77 - e.mu.Unlock() 78 78 - 79 79 - // 1. Finalize log tail first so all log lines precede terminal event 80 80 - if res.stopTail != nil { 81 81 - res.stopTail() 82 82 - } 83 83 - 84 84 - terminalStatus := st.Status 85 85 - if cancelled { 86 86 - terminalStatus = string(models.StatusKindCancelled) 87 87 - } 88 88 - var logDir string 89 89 - if e.cfg != nil { 90 90 - logDir = e.cfg.Server.LogDir 91 91 - } 92 92 - logPath := models.LogFilePath(logDir, res.wid) 93 93 - 94 94 - var errStr string 95 95 - if st != nil && st.Error != nil { 96 96 - errStr = *st.Error 97 97 - } 98 98 - var exitCode int64 99 99 - if st != nil && st.ExitCode != nil { 100 100 - exitCode = *st.ExitCode 101 101 - } 102 102 - 103 103 - // Calculate SHA256 of log file 104 104 - hash := "" 105 105 - if f, err := os.Open(logPath); err == nil { 106 106 - h := sha256.New() 107 107 - if _, err := io.Copy(h, f); err == nil { 108 108 - hash = "sha256:" + hex.EncodeToString(h.Sum(nil)) 109 109 - } 110 110 - _ = f.Close() 111 111 - } 112 112 - 113 113 - // refs are opaque keys interpreted by the configured artifact store 114 114 - ref := "logs/" + res.leaseID + ".log" 115 115 - 116 116 - // 2. Persist restart-retryable pending artifact state 117 117 - if e.db != nil { 118 118 - _ = e.db.SavePendingArtifact(res.leaseID, res.wid.Name, terminalStatus, errStr, exitCode, ref, hash) 119 119 - } 120 120 - 121 121 - // 3. Upload artifact with non-cancelled cleanup context 122 122 - cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(context.Background()), 2*time.Minute) 123 123 - defer cancel() 124 124 - 125 125 - if e.writer != nil { 126 126 - if f, err := os.Open(logPath); err == nil { 127 127 - defer f.Close() 128 128 - if uploadErr := e.writer.Put(cleanupCtx, ref, f); uploadErr != nil { 129 129 - e.l.Error("artifact upload failed", "lease", res.leaseID, "err", uploadErr) 130 130 - return fmt.Errorf("artifact upload: %w", uploadErr) 131 131 - } 132 132 - } 133 133 - } 134 134 - 135 135 - // 4. Append terminal event (with LogArtifact) to outbox 136 136 - if err := e.appendTerminalWithArtifact(res.leaseID, terminalStatus, st, ref, hash); err != nil { 137 137 - return err 138 138 - } 139 139 - 140 140 - // 5. Remove pending artifact state after terminal outbox append succeeds 141 141 - if e.db != nil { 142 142 - _ = e.db.RemovePendingArtifact(res.leaseID) 143 143 - } 144 144 - e.mu.Lock() 145 145 - cleanup := e.removeReservationLocked(res, false) 146 146 - e.mu.Unlock() 147 147 - 148 148 - cleanup() 149 149 - 150 150 - e.pushSnapshot() 151 151 - return nil 152 152 - } 153 153 - 154 154 - func (e *Executor) reservationFor(pipelineAturi, workflow string) *reservation { 155 155 - e.mu.Lock() 156 156 - defer e.mu.Unlock() 157 157 - for _, res := range e.active { 158 158 - if string(res.wid.PipelineId.AtUri()) == pipelineAturi && res.wid.Name == workflow { 159 159 - return res 160 160 - } 161 161 - } 162 162 - return nil 163 163 - } 164 164 - 165 165 - func (e *Executor) maskSecrets(res *reservation, text string) string { 166 166 - if res == nil || res.vault == nil { 167 167 - return text 168 168 - } 169 169 - for _, s := range res.vault.secrets { 170 170 - if s.Value != "" { 171 171 - text = strings.ReplaceAll(text, s.Value, "***") 172 172 - } 173 173 - } 174 174 - return text 175 175 - } 176 176 - 177 177 - func (e *Executor) SendLiveLog(leaseID string, raw []byte) error { 178 178 - e.connMu.Lock() 179 179 - enc := e.enc 180 180 - e.connMu.Unlock() 181 181 - if enc == nil { 182 182 - return nil 183 183 - } 184 184 - return enc.Encode(&millproto.Message{ 185 185 - LiveLog: &millv1.LiveLog{ 186 186 - LeaseId: leaseID, 187 187 - RawJson: raw, 188 188 - }, 189 189 - }) 190 190 - } 191 191 - 192 192 - func (e *Executor) startTail(res *reservation) { 193 193 - path := models.LogFilePath(e.cfg.Server.LogDir, res.wid) 194 194 - t, err := tail.TailFile(path, tail.Config{ 195 195 - Follow: true, 196 196 - ReOpen: true, 197 197 - MustExist: false, 198 198 - Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}, 199 199 - Logger: tail.DiscardingLogger, 200 200 - }) 201 201 - if err != nil { 202 202 - e.l.Error("tail log file failed", "wid", res.wid, "err", err) 203 203 - return 204 204 - } 205 205 - 206 206 - done := make(chan struct{}) 207 207 - go func() { 208 208 - defer close(done) 209 209 - for line := range t.Lines { 210 210 - if line == nil || line.Err != nil { 211 211 - continue 212 212 - } 213 213 - masked := e.maskSecrets(res, line.Text) 214 214 - _ = e.SendLiveLog(res.leaseID, []byte(masked+"\n")) 215 215 - } 216 216 - }() 217 217 - 218 218 - var once sync.Once 219 219 - res.stopTail = func() { 220 220 - once.Do(func() { 221 221 - _ = t.StopAtEOF() 222 222 - <-done 223 223 - }) 224 224 - } 225 225 - } 226 226 - 227 227 - type memVault struct { 228 228 - secrets []secrets.UnlockedSecret 229 229 - } 230 230 - 231 231 - func newMemVault(pb []*millv1.Secret) *memVault { 232 232 - v := &memVault{secrets: make([]secrets.UnlockedSecret, 0, len(pb))} 233 233 - for _, s := range pb { 234 234 - v.secrets = append(v.secrets, secrets.UnlockedSecret{ 235 235 - Key: s.Key, 236 236 - Value: s.Value, 237 237 - }) 238 238 - } 239 239 - return v 240 240 - } 241 241 - 242 242 - func (v *memVault) GetSecretsUnlocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.UnlockedSecret, error) { 243 243 - return v.secrets, nil 244 244 - } 245 245 - func (v *memVault) GetSecretsLocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.LockedSecret, error) { 246 246 - return nil, nil 247 247 - } 248 248 - func (v *memVault) AddSecret(ctx context.Context, s secrets.UnlockedSecret) error { return nil } 249 249 - func (v *memVault) RemoveSecret(ctx context.Context, s secrets.Secret[any]) error { return nil } 250 250 - 251 251 - var _ secrets.Manager = (*memVault)(nil)

-344

spindle/mill/executor/outbox.go

Reviewed

··· 1 1 - package executor 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "crypto/rand" 6 6 - "encoding/hex" 7 7 - "encoding/json" 8 8 - "fmt" 9 9 - "os" 10 10 - "time" 11 11 - 12 12 - "google.golang.org/protobuf/proto" 13 13 - "tangled.org/core/api/tangled" 14 14 - "tangled.org/core/spindle/db" 15 15 - millproto "tangled.org/core/spindle/mill/proto" 16 16 - millv1 "tangled.org/core/spindle/mill/proto/gen" 17 17 - "tangled.org/core/spindle/models" 18 18 - ) 19 19 - 20 20 - const ( 21 21 - maxBatchBytes = 4 * 1024 * 1024 22 22 - maxBatchEvents = 128 23 23 - ) 24 24 - 25 25 - func generateEpoch() string { 26 26 - var b [8]byte 27 27 - _, _ = rand.Read(b[:]) 28 28 - return hex.EncodeToString(b[:]) 29 29 - } 30 30 - 31 31 - func (e *Executor) initOutbox() error { 32 32 - e.eventMu.Lock() 33 33 - defer e.eventMu.Unlock() 34 34 - 35 35 - epoch, _, err := e.db.GetOutboxState() 36 36 - if err != nil { 37 37 - return err 38 38 - } 39 39 - if epoch != "" { 40 40 - e.epoch = epoch 41 41 - } else { 42 42 - e.epoch = generateEpoch() 43 43 - if err := e.db.SetOutboxEpoch(e.epoch); err != nil { 44 44 - return fmt.Errorf("set outbox epoch: %w", err) 45 45 - } 46 46 - } 47 47 - 48 48 - rows, err := e.db.ListOutboxRows() 49 49 - if err != nil { 50 50 - return fmt.Errorf("list outbox rows: %w", err) 51 51 - } 52 52 - 53 53 - for _, r := range rows { 54 54 - e.outboxBytes += r.ByteSize 55 55 - } 56 56 - _ = e.recoverPendingArtifacts() 57 57 - return nil 58 58 - } 59 59 - 60 60 - func (e *Executor) appendAndSend(leaseID string, payload any, control bool) error { 61 61 - entry := &millv1.Event{LeaseId: leaseID} 62 62 - switch payload := payload.(type) { 63 63 - case *millv1.Event_StatusEvent: 64 64 - entry.Payload = payload 65 65 - case *millv1.Event_AttemptResult: 66 66 - entry.Payload = payload 67 67 - default: 68 68 - return fmt.Errorf("unsupported stream payload %T", payload) 69 69 - } 70 70 - isTerminal := false 71 71 - if _, ok := payload.(*millv1.Event_AttemptResult); ok { 72 72 - isTerminal = true 73 73 - } 74 74 - entry.Seqno = ^uint64(0) 75 75 - wireSize := proto.Size(&millproto.Message{EventBatch: &millv1.EventBatch{ 76 76 - Epoch: e.epoch, 77 77 - Events: []*millv1.Event{entry}, 78 78 - }}) 79 79 - entry.Seqno = 0 80 80 - if wireSize > maxBatchBytes { 81 81 - return fmt.Errorf("control stream entry exceeds wire limit: %d > %d", wireSize, maxBatchBytes) 82 82 - } 83 83 - 84 84 - encoded, err := proto.Marshal(entry) 85 85 - if err != nil { 86 86 - return fmt.Errorf("marshal stream entry: %w", err) 87 87 - } 88 88 - 89 89 - e.eventMu.Lock() 90 90 - if control && !isTerminal && e.maxOutboxBytes > 0 && e.outboxBytes+int64(len(encoded)) > e.maxOutboxBytes { 91 91 - e.l.Warn("outbox reserve exhausted; dropping nonterminal status", "cap", e.maxOutboxBytes) 92 92 - e.eventMu.Unlock() 93 93 - return nil 94 94 - } 95 95 - if _, err := e.db.AppendOutboxRow(encoded, control); err != nil { 96 96 - e.eventMu.Unlock() 97 97 - return fmt.Errorf("append outbox row: %w", err) 98 98 - } 99 99 - e.outboxBytes += int64(len(encoded)) 100 100 - e.eventMu.Unlock() 101 101 - 102 102 - e.sendPending() 103 103 - return nil 104 104 - } 105 105 - 106 106 - func (e *Executor) appendStatus(leaseID string, st *tangled.PipelineStatus) error { 107 107 - if st.Status != string(models.StatusKindRunning) { 108 108 - return fmt.Errorf("unsupported nonterminal status %q", st.Status) 109 109 - } 110 110 - exit, errStr := parseStatusExitAndError(st) 111 111 - payload := &millv1.Event_StatusEvent{StatusEvent: &millv1.StatusEvent{ 112 112 - Status: millv1.NonterminalStatus_RUNNING, 113 113 - Error: errStr, 114 114 - ExitCode: exit, 115 115 - }} 116 116 - return e.appendAndSend(leaseID, payload, true) 117 117 - } 118 118 - 119 119 - func (e *Executor) appendTerminal(leaseID, status string, st *tangled.PipelineStatus) error { 120 120 - return e.appendTerminalWithArtifact(leaseID, status, st, "", "") 121 121 - } 122 122 - 123 123 - func (e *Executor) appendTerminalWithArtifact(leaseID, status string, st *tangled.PipelineStatus, ref, hash string) error { 124 124 - var terminalStatus millv1.TerminalStatus 125 125 - switch status { 126 126 - case string(models.StatusKindSuccess): 127 127 - terminalStatus = millv1.TerminalStatus_SUCCESS 128 128 - case string(models.StatusKindFailed): 129 129 - terminalStatus = millv1.TerminalStatus_FAILED 130 130 - case string(models.StatusKindTimeout): 131 131 - terminalStatus = millv1.TerminalStatus_TIMEOUT 132 132 - case string(models.StatusKindCancelled): 133 133 - terminalStatus = millv1.TerminalStatus_CANCELLED 134 134 - default: 135 135 - return fmt.Errorf("unsupported terminal status %q", status) 136 136 - } 137 137 - 138 138 - exit, errStr := parseStatusExitAndError(st) 139 139 - var logArtifact *millv1.LogArtifact 140 140 - if ref != "" { 141 141 - logArtifact = &millv1.LogArtifact{ 142 142 - Ref: ref, 143 143 - Hash: hash, 144 144 - } 145 145 - } 146 146 - payload := &millv1.Event_AttemptResult{AttemptResult: &millv1.AttemptResult{ 147 147 - Status: terminalStatus, 148 148 - Error: errStr, 149 149 - ExitCode: exit, 150 150 - LogArtifact: logArtifact, 151 151 - }} 152 152 - return e.appendAndSend(leaseID, payload, true) 153 153 - } 154 154 - 155 155 - func (e *Executor) recoverPendingArtifacts() error { 156 156 - if e.db == nil { 157 157 - return nil 158 158 - } 159 159 - pending, err := e.db.ListPendingArtifacts() 160 160 - if err != nil || len(pending) == 0 { 161 161 - return err 162 162 - } 163 163 - for _, p := range pending { 164 164 - if p.Ref != "" { 165 165 - if e.writer == nil { 166 166 - continue 167 167 - } 168 168 - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) 169 169 - var logDir string 170 170 - if e.cfg != nil { 171 171 - logDir = e.cfg.Server.LogDir 172 172 - } 173 173 - logPath := models.LogFilePath(logDir, models.WorkflowId{Name: p.Workflow}) 174 174 - f, openErr := os.Open(logPath) 175 175 - if openErr != nil { 176 176 - cancel() 177 177 - continue 178 178 - } 179 179 - uploadErr := e.writer.Put(ctx, p.Ref, f) 180 180 - _ = f.Close() 181 181 - cancel() 182 182 - if uploadErr != nil { 183 183 - continue 184 184 - } 185 185 - } 186 186 - 187 187 - st := &tangled.PipelineStatus{ 188 188 - Status: p.Status, 189 189 - Error: &p.Error, 190 190 - ExitCode: &p.ExitCode, 191 191 - } 192 192 - if err := e.appendTerminalWithArtifact(p.LeaseID, p.Status, st, p.Ref, p.Hash); err == nil { 193 193 - _ = e.db.RemovePendingArtifact(p.LeaseID) 194 194 - } 195 195 - } 196 196 - return nil 197 197 - } 198 198 - 199 199 - func truncateEventString(value string) string { 200 200 - if len(value) > 65536 { 201 201 - return value[:65536] 202 202 - } 203 203 - return value 204 204 - } 205 205 - 206 206 - func (e *Executor) sendPending() { 207 207 - e.connMu.Lock() 208 208 - enc := e.enc 209 209 - e.connMu.Unlock() 210 210 - 211 211 - if enc == nil { 212 212 - return 213 213 - } 214 214 - 215 215 - e.flushMu.Lock() 216 216 - defer e.flushMu.Unlock() 217 217 - 218 218 - if err := e.sendPendingLocked(enc); err != nil { 219 219 - e.l.Error("send pending events failed", "err", err) 220 220 - } 221 221 - } 222 222 - 223 223 - func (e *Executor) sendPendingLocked(enc messageEncoder) error { 224 224 - for { 225 225 - rows, err := e.db.ListOutboxRowsAfter(e.sentSeqno, maxBatchEvents) 226 226 - if err != nil { 227 227 - return fmt.Errorf("list outbox rows after %d: %w", e.sentSeqno, err) 228 228 - } 229 229 - if len(rows) == 0 { 230 230 - return nil 231 231 - } 232 232 - if err := e.sendRows(enc, rows); err != nil { 233 233 - return err 234 234 - } 235 235 - } 236 236 - } 237 237 - 238 238 - func (e *Executor) sendRows(enc messageEncoder, rows []db.OutboxRow) error { 239 239 - events := make([]*millv1.Event, 0, len(rows)) 240 240 - var lastSeqno uint64 241 241 - 242 242 - for _, r := range rows { 243 243 - ev, err := decodeEvent(r.Payload, r.Seqno) 244 244 - if err != nil { 245 245 - return fmt.Errorf("decode event at seqno %d: %w", r.Seqno, err) 246 246 - } 247 247 - events = append(events, ev) 248 248 - lastSeqno = r.Seqno 249 249 - } 250 250 - 251 251 - batch := &millv1.EventBatch{ 252 252 - Epoch: e.epoch, 253 253 - Events: events, 254 254 - } 255 255 - 256 256 - e.sendMu.Lock() 257 257 - defer e.sendMu.Unlock() 258 258 - 259 259 - if err := enc.Encode(&millproto.Message{EventBatch: batch}); err != nil { 260 260 - return fmt.Errorf("encode event batch (seqno %d..%d): %w", rows[0].Seqno, lastSeqno, err) 261 261 - } 262 262 - e.sentSeqno = lastSeqno 263 263 - return nil 264 264 - } 265 265 - 266 266 - func (e *Executor) replay(ackSeqno uint64) error { 267 267 - e.connMu.Lock() 268 268 - enc := e.enc 269 269 - e.connMu.Unlock() 270 270 - if enc == nil { 271 271 - return nil 272 272 - } 273 273 - 274 274 - e.flushMu.Lock() 275 275 - defer e.flushMu.Unlock() 276 276 - 277 277 - e.sendMu.Lock() 278 278 - e.sentSeqno = ackSeqno 279 279 - e.sendMu.Unlock() 280 280 - 281 281 - return e.sendPendingLocked(enc) 282 282 - } 283 283 - 284 284 - func (e *Executor) handleAck(ack *millv1.Ack) { 285 285 - if ack == nil || ack.GetEpoch() != e.epoch { 286 286 - return 287 287 - } 288 288 - 289 289 - upTo := ack.GetUpToSeqno() 290 290 - if upTo == 0 { 291 291 - return 292 292 - } 293 293 - 294 294 - if err := e.deleteOutboxPrefix(upTo); err != nil { 295 295 - e.l.Error("delete outbox prefix failed", "upTo", upTo, "err", err) 296 296 - } 297 297 - } 298 298 - 299 299 - func (e *Executor) subtractOutboxBytes(deleted db.OutboxDeletion) { 300 300 - e.outboxBytes = max(0, e.outboxBytes-deleted.Bytes) 301 301 - } 302 302 - 303 303 - func parseStatus(raw json.RawMessage) (*tangled.PipelineStatus, bool) { 304 304 - var st tangled.PipelineStatus 305 305 - if err := json.Unmarshal(raw, &st); err != nil { 306 306 - return nil, false 307 307 - } 308 308 - return &st, true 309 309 - } 310 310 - 311 311 - func (e *Executor) deleteOutboxPrefix(upTo uint64) error { 312 312 - e.eventMu.Lock() 313 313 - defer e.eventMu.Unlock() 314 314 - 315 315 - deleted, err := e.db.DeleteOutboxPrefix(upTo) 316 316 - if err == nil { 317 317 - e.subtractOutboxBytes(deleted) 318 318 - } 319 319 - return err 320 320 - } 321 321 - 322 322 - func parseStatusExitAndError(st *tangled.PipelineStatus) (int64, string) { 323 323 - if st == nil { 324 324 - return 0, "" 325 325 - } 326 326 - var exit int64 327 327 - if st.ExitCode != nil { 328 328 - exit = *st.ExitCode 329 329 - } 330 330 - var errStr string 331 331 - if st.Error != nil { 332 332 - errStr = truncateEventString(*st.Error) 333 333 - } 334 334 - return exit, errStr 335 335 - } 336 336 - 337 337 - func decodeEvent(payload []byte, seqno uint64) (*millv1.Event, error) { 338 338 - var ev millv1.Event 339 339 - if err := proto.Unmarshal(payload, &ev); err != nil { 340 340 - return nil, err 341 341 - } 342 342 - ev.Seqno = seqno 343 343 - return &ev, nil 344 344 - }

-37

spindle/mill/executor/reserved.go

Reviewed

··· 1 1 - package executor 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "fmt" 6 6 - "sync" 7 7 - 8 8 - "tangled.org/core/spindle/engine" 9 9 - "tangled.org/core/spindle/models" 10 10 - ) 11 11 - 12 12 - // wraps a real engine so StartWorkflows gets the slot ReserveSeat already 13 13 - // acquired, not a second one. everything else delegates, the execution 14 14 - // path runs exactly like standalone 15 15 - type reservedEngine struct { 16 16 - models.Engine 17 17 - slot engine.WorkflowSlot 18 18 - once sync.Once 19 19 - } 20 20 - 21 21 - func newReservedEngine(inner models.Engine, slot engine.WorkflowSlot) models.Engine { 22 22 - return &reservedEngine{Engine: inner, slot: slot} 23 23 - } 24 24 - 25 25 - // hands back the pre-acquired slot exactly once, a second acquire would 26 26 - // double-count it 27 27 - func (e *reservedEngine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) { 28 28 - var slot engine.WorkflowSlot 29 29 - e.once.Do(func() { 30 30 - slot = e.slot 31 31 - e.slot = nil 32 32 - }) 33 33 - if slot == nil { 34 34 - return nil, fmt.Errorf("reserved slot already consumed") 35 35 - } 36 36 - return slot, nil 37 37 - }

-546

spindle/mill/executor/reserved_test.go

Reviewed

··· 1 1 - package executor 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "encoding/json" 6 6 - "github.com/bluesky-social/indigo/atproto/syntax" 7 7 - "github.com/gorilla/websocket" 8 8 - "google.golang.org/protobuf/proto" 9 9 - "io" 10 10 - "log/slog" 11 11 - "net/http" 12 12 - "net/http/httptest" 13 13 - "path/filepath" 14 14 - "strings" 15 15 - "testing" 16 16 - "time" 17 17 - 18 18 - "tangled.org/core/api/tangled" 19 19 - "tangled.org/core/notifier" 20 20 - "tangled.org/core/spindle/config" 21 21 - "tangled.org/core/spindle/db" 22 22 - "tangled.org/core/spindle/engine" 23 23 - millproto "tangled.org/core/spindle/mill/proto" 24 24 - millv1 "tangled.org/core/spindle/mill/proto/gen" 25 25 - "tangled.org/core/spindle/models" 26 26 - "tangled.org/core/spindle/secrets" 27 27 - ) 28 28 - 29 29 - type captureEncoder struct { 30 30 - messages chan *millproto.Message 31 31 - } 32 32 - 33 33 - func newCaptureEncoder() *captureEncoder { 34 34 - return &captureEncoder{messages: make(chan *millproto.Message, 4)} 35 35 - } 36 36 - 37 37 - func (e *captureEncoder) Encode(msg *millproto.Message) error { 38 38 - e.messages <- msg 39 39 - return nil 40 40 - } 41 41 - 42 42 - type fakeSlot struct{ released int } 43 43 - 44 44 - func (s *fakeSlot) Release() { s.released++ } 45 45 - 46 46 - type fakeEngine struct { 47 47 - setupCalled bool 48 48 - runCalled bool 49 49 - destroyCalled bool 50 50 - acquireCalled bool 51 51 - secrets chan []secrets.UnlockedSecret 52 52 - done chan struct{} 53 53 - } 54 54 - 55 55 - func (e *fakeEngine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { 56 56 - return &models.Workflow{Name: twf.Name}, nil 57 57 - } 58 58 - func (e *fakeEngine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, l models.WorkflowLogger) error { 59 59 - e.setupCalled = true 60 60 - return nil 61 61 - } 62 62 - func (e *fakeEngine) WorkflowTimeout() time.Duration { return 7 * time.Minute } 63 63 - func (e *fakeEngine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { 64 64 - e.destroyCalled = true 65 65 - return nil 66 66 - } 67 67 - func (e *fakeEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, s []secrets.UnlockedSecret, l models.WorkflowLogger) error { 68 68 - e.runCalled = true 69 69 - if e.secrets != nil { 70 70 - e.secrets <- s 71 71 - } 72 72 - if e.done != nil { 73 73 - close(e.done) 74 74 - } 75 75 - return nil 76 76 - } 77 77 - func (e *fakeEngine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode engine.AcquireMode) (engine.WorkflowSlot, error) { 78 78 - e.acquireCalled = true 79 79 - return &fakeSlot{}, nil 80 80 - } 81 81 - 82 82 - type fakeStep struct{} 83 83 - 84 84 - func (fakeStep) Name() string { return "test" } 85 85 - func (fakeStep) Command() string { return "true" } 86 86 - func (fakeStep) Kind() models.StepKind { return models.StepKindUser } 87 87 - 88 88 - func TestNewFailsWhenOutboxCannotInitialize(t *testing.T) { 89 89 - d := testDB(t) 90 90 - if err := d.Close(); err != nil { 91 91 - t.Fatal(err) 92 92 - } 93 93 - n := notifier.New() 94 94 - cfg := &config.Config{} 95 95 - if _, err := New(cfg, nil, d, &n, slog.New(slog.NewTextHandler(io.Discard, nil))); err == nil { 96 96 - t.Fatal("New succeeded with an unavailable outbox database") 97 97 - } 98 98 - } 99 99 - 100 100 - func TestReservedEngineHandsBackHeldSlotOnce(t *testing.T) { 101 101 - inner := &fakeEngine{} 102 102 - slot := &fakeSlot{} 103 103 - re := newReservedEngine(inner, slot) 104 104 - 105 105 - got, err := re.(engine.WorkflowSlotter).AcquireWorkflowSlot(context.Background(), models.WorkflowId{}, nil, engine.Wait) 106 106 - if err != nil { 107 107 - t.Fatal(err) 108 108 - } 109 109 - if got != engine.WorkflowSlot(slot) { 110 110 - t.Fatal("AcquireWorkflowSlot() did not return the held slot") 111 111 - } 112 112 - if inner.acquireCalled { 113 113 - t.Fatal("wrapper must not call the inner engine's AcquireWorkflowSlot") 114 114 - } 115 115 - 116 116 - if _, err := re.(engine.WorkflowSlotter).AcquireWorkflowSlot(context.Background(), models.WorkflowId{}, nil, engine.Wait); err == nil { 117 117 - t.Fatal("second AcquireWorkflowSlot() should error") 118 118 - } 119 119 - } 120 120 - 121 121 - func TestHandleCommitIsIdempotent(t *testing.T) { 122 122 - enc := newCaptureEncoder() 123 123 - e := testExecutor(t) 124 124 - e.enc = enc 125 125 - e.active["lease-1"] = &reservation{leaseID: "lease-1", committed: true} 126 126 - 127 127 - e.handleCommit(context.Background(), &millv1.CommitLease{LeaseId: "lease-1"}) 128 128 - msg := <-enc.messages 129 129 - if got := msg.GetCommitted().GetLeaseId(); got != "lease-1" { 130 130 - t.Fatalf("Committed lease = %q, want lease-1", got) 131 131 - } 132 132 - } 133 133 - 134 134 - func TestHandleCommitRejectsMissingReservation(t *testing.T) { 135 135 - enc := newCaptureEncoder() 136 136 - e := testExecutor(t) 137 137 - e.enc = enc 138 138 - 139 139 - e.handleCommit(context.Background(), &millv1.CommitLease{LeaseId: "expired"}) 140 140 - result := (<-enc.messages).GetReserveResult() 141 141 - if result == nil { 142 142 - t.Fatal("missing reservation commit did not receive a ReserveResult") 143 143 - } 144 144 - if result.GetLeaseId() != "expired" || result.GetAccepted() { 145 145 - t.Fatalf("ReserveResult = %+v, want correlated rejection", result) 146 146 - } 147 147 - } 148 148 - 149 149 - func TestHandleCancelFinalizesExpiredReservation(t *testing.T) { 150 150 - enc := newCaptureEncoder() 151 151 - e := testExecutor(t) 152 152 - e.enc = enc 153 153 - 154 154 - e.handleCancel("lease-expired") 155 155 - var ack *millv1.CancelAck 156 156 - for ack == nil { 157 157 - select { 158 158 - case msg := <-enc.messages: 159 159 - ack = msg.GetCancelAck() 160 160 - case <-time.After(time.Second): 161 161 - t.Fatal("cancel acknowledgement timed out") 162 162 - } 163 163 - } 164 164 - if ack.GetLeaseId() != "lease-expired" { 165 165 - t.Fatalf("CancelAck = %+v, want lease-expired", ack) 166 166 - } 167 167 - rows, err := e.db.ListOutboxRows() 168 168 - if err != nil { 169 169 - t.Fatal(err) 170 170 - } 171 171 - if len(rows) != 1 { 172 172 - t.Fatalf("cancel terminal outbox rows = %d, want 1", len(rows)) 173 173 - } 174 174 - var entry millv1.Event 175 175 - if err := proto.Unmarshal(rows[0].Payload, &entry); err != nil { 176 176 - t.Fatal(err) 177 177 - } 178 178 - if got := entry.GetAttemptResult().GetStatus(); got != millv1.TerminalStatus_CANCELLED { 179 179 - t.Fatalf("cancel terminal = %v, want CANCELLED", got) 180 180 - } 181 181 - } 182 182 - 183 183 - func TestHandleCommitPreservesPreauthorizedSecrets(t *testing.T) { 184 184 - d := testDB(t) 185 185 - n := notifier.New() 186 186 - enc := newCaptureEncoder() 187 187 - e := &Executor{ 188 188 - cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 189 189 - db: d, 190 190 - n: &n, 191 191 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 192 192 - active: make(map[string]*reservation), 193 193 - maxOutboxBytes: 10 * 1024 * 1024, 194 194 - enc: enc, 195 195 - } 196 196 - if err := e.initOutbox(); err != nil { 197 197 - t.Fatal(err) 198 198 - } 199 199 - e.lifecycleCtx = context.Background() 200 200 - 201 201 - inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})} 202 202 - slot := &fakeSlot{} 203 203 - repoDid, err := syntax.ParseDID("did:web:example.com") 204 204 - if err != nil { 205 205 - t.Fatal(err) 206 206 - } 207 207 - res := &reservation{ 208 208 - leaseID: "lease-1", 209 209 - wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, 210 210 - realEngine: inner, 211 211 - slot: slot, 212 212 - wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, 213 213 - repoDid: repoDid, 214 214 - } 215 215 - e.active[res.leaseID] = res 216 216 - 217 217 - e.handleCommit(context.Background(), &millv1.CommitLease{ 218 218 - LeaseId: res.leaseID, 219 219 - Secrets: []*millv1.Secret{{Key: "TOKEN", Value: "secret-value"}}, 220 220 - }) 221 221 - if got := (<-enc.messages).GetCommitted().GetLeaseId(); got != res.leaseID { 222 222 - t.Fatalf("Committed lease = %q, want %q", got, res.leaseID) 223 223 - } 224 224 - select { 225 225 - case got := <-inner.secrets: 226 226 - if len(got) != 1 || got[0].Key != "TOKEN" || got[0].Value != "secret-value" { 227 227 - t.Fatalf("RunStep secrets = %+v", got) 228 228 - } 229 229 - case <-time.After(2 * time.Second): 230 230 - t.Fatal("RunStep did not receive CommitLease secrets") 231 231 - } 232 232 - select { 233 233 - case <-inner.done: 234 234 - case <-time.After(2 * time.Second): 235 235 - t.Fatal("workflow did not finish") 236 236 - } 237 237 - if res.stopTail != nil { 238 238 - res.stopTail() 239 239 - } 240 240 - e.jobsWG.Wait() 241 241 - } 242 242 - 243 243 - func TestRunSessionCancellationClosesStalledWebsocket(t *testing.T) { 244 244 - connected := make(chan struct{}) 245 245 - release := make(chan struct{}) 246 246 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 247 247 - conn, err := websocket.Upgrade(w, r, nil, 1024, 1024) 248 248 - if err != nil { 249 249 - return 250 250 - } 251 251 - defer conn.Close() 252 252 - close(connected) 253 253 - <-release 254 254 - })) 255 255 - t.Cleanup(func() { 256 256 - close(release) 257 257 - srv.Close() 258 258 - }) 259 259 - 260 260 - e := testSessionExecutor(t, "ws"+strings.TrimPrefix(srv.URL, "http")) 261 261 - ctx, cancel := context.WithCancel(context.Background()) 262 262 - done := make(chan error, 1) 263 263 - go func() { done <- e.runSession(ctx) }() 264 264 - <-connected 265 265 - cancel() 266 266 - 267 267 - select { 268 268 - case <-done: 269 269 - case <-time.After(2 * time.Second): 270 270 - t.Fatal("runSession did not return after context cancellation") 271 271 - } 272 272 - } 273 273 - 274 274 - func testSessionExecutor(t *testing.T, url string) *Executor { 275 275 - d := testDB(t) 276 276 - e := &Executor{ 277 277 - millURL: url, 278 278 - seats: 1, 279 279 - engines: make(map[string]models.Engine), 280 280 - db: d, 281 281 - cfg: &config.Config{Server: config.Server{Dev: true}}, 282 282 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 283 283 - active: make(map[string]*reservation), 284 284 - maxOutboxBytes: 10 * 1024 * 1024, 285 285 - } 286 286 - if err := e.initOutbox(); err != nil { 287 287 - t.Fatal(err) 288 288 - } 289 289 - return e 290 290 - } 291 291 - 292 292 - func testExecutor(t *testing.T) *Executor { 293 293 - d := testDB(t) 294 294 - e := &Executor{ 295 295 - db: d, 296 296 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 297 297 - active: make(map[string]*reservation), 298 298 - maxOutboxBytes: 10 * 1024 * 1024, 299 299 - } 300 300 - if err := e.initOutbox(); err != nil { 301 301 - t.Fatal(err) 302 302 - } 303 303 - return e 304 304 - } 305 305 - 306 306 - func testDB(t *testing.T) *db.DB { 307 307 - d, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "spindle.db")) 308 308 - if err != nil { 309 309 - t.Fatal(err) 310 310 - } 311 311 - return d 312 312 - } 313 313 - 314 314 - func TestFinishJobReportsCancelledReservationAsCancelled(t *testing.T) { 315 315 - d := testDB(t) 316 316 - res := &reservation{ 317 317 - leaseID: "lease-1", 318 318 - wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, 319 319 - cancelled: true, 320 320 - } 321 321 - e := &Executor{ 322 322 - db: d, 323 323 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 324 324 - active: map[string]*reservation{res.leaseID: res}, 325 325 - maxOutboxBytes: 10 * 1024 * 1024, 326 326 - } 327 327 - if err := e.initOutbox(); err != nil { 328 328 - t.Fatal(err) 329 329 - } 330 330 - 331 331 - e.finishJob(res, &tangled.PipelineStatus{ 332 332 - Pipeline: string(res.wid.PipelineId.AtUri()), 333 333 - Workflow: res.wid.Name, 334 334 - Status: string(models.StatusKindFailed), 335 335 - }) 336 336 - 337 337 - rows, err := d.ListOutboxRows() 338 338 - if err != nil { 339 339 - t.Fatal(err) 340 340 - } 341 341 - if len(rows) != 1 { 342 342 - t.Fatalf("outbox rows = %d, want 1", len(rows)) 343 343 - } 344 344 - 345 345 - var entry millv1.Event 346 346 - if err := proto.Unmarshal(rows[0].Payload, &entry); err != nil { 347 347 - t.Fatal(err) 348 348 - } 349 349 - got := entry.GetAttemptResult().GetStatus() 350 350 - if got != millv1.TerminalStatus_CANCELLED { 351 351 - t.Fatalf("terminal status = %v, want CANCELLED", got) 352 352 - } 353 353 - } 354 354 - 355 355 - func TestReplayRejectsMalformedOutboxRow(t *testing.T) { 356 356 - d := testDB(t) 357 357 - e := &Executor{ 358 358 - db: d, 359 359 - enc: newCaptureEncoder(), 360 360 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 361 361 - } 362 362 - if err := e.initOutbox(); err != nil { 363 363 - t.Fatal(err) 364 364 - } 365 365 - if _, err := d.AppendOutboxRow([]byte("not protobuf"), true); err != nil { 366 366 - t.Fatal(err) 367 367 - } 368 368 - if err := e.replay(0); err == nil { 369 369 - t.Fatal("replay accepted a malformed row and would leave a permanent seqno gap") 370 370 - } 371 371 - } 372 372 - 373 373 - func TestSocketCancellationIndependence(t *testing.T) { 374 374 - d := testDB(t) 375 375 - n := notifier.New() 376 376 - e := &Executor{ 377 377 - db: d, 378 378 - n: &n, 379 379 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 380 380 - active: make(map[string]*reservation), 381 381 - maxOutboxBytes: 10 * 1024 * 1024, 382 382 - cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 383 383 - } 384 384 - if err := e.initOutbox(); err != nil { 385 385 - t.Fatal(err) 386 386 - } 387 387 - 388 388 - lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background()) 389 389 - defer cancelLifecycle() 390 390 - e.lifecycleCtx = lifecycleCtx 391 391 - 392 392 - inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})} 393 393 - slot := &fakeSlot{} 394 394 - repoDid, err := syntax.ParseDID("did:web:example.com") 395 395 - if err != nil { 396 396 - t.Fatal(err) 397 397 - } 398 398 - res := &reservation{ 399 399 - leaseID: "lease-1", 400 400 - wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, 401 401 - realEngine: inner, 402 402 - slot: slot, 403 403 - wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, 404 404 - repoDid: repoDid, 405 405 - } 406 406 - e.active[res.leaseID] = res 407 407 - 408 408 - sessionCtx, cancelSession := context.WithCancel(lifecycleCtx) 409 409 - 410 410 - e.handleCommit(sessionCtx, &millv1.CommitLease{ 411 411 - LeaseId: "lease-1", 412 412 - }) 413 413 - 414 414 - cancelSession() 415 415 - 416 416 - // session disconnect must not cancel the running job 417 417 - select { 418 418 - case <-inner.done: 419 419 - case <-time.After(2 * time.Second): 420 420 - t.Fatal("workflow did not complete even though websocket session was cancelled") 421 421 - } 422 422 - 423 423 - e.jobsWG.Wait() 424 424 - } 425 425 - 426 426 - func TestMonotonicSnapshots(t *testing.T) { 427 427 - enc := newCaptureEncoder() 428 428 - e := &Executor{ 429 429 - enc: enc, 430 430 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 431 431 - active: make(map[string]*reservation), 432 432 - maxOutboxBytes: 10 * 1024 * 1024, 433 433 - } 434 434 - 435 435 - e.pushSnapshot() 436 436 - msg1 := <-enc.messages 437 437 - seq1 := msg1.GetNodeSnapshot().GetSeqno() 438 438 - if seq1 != 1 { 439 439 - t.Fatalf("first seq = %d, want 1", seq1) 440 440 - } 441 441 - 442 442 - e.pushSnapshot() 443 443 - msg2 := <-enc.messages 444 444 - seq2 := msg2.GetNodeSnapshot().GetSeqno() 445 445 - if seq2 != 2 { 446 446 - t.Fatalf("second seq = %d, want 2", seq2) 447 447 - } 448 448 - } 449 449 - 450 450 - func TestTimerRace(t *testing.T) { 451 451 - d := testDB(t) 452 452 - e := &Executor{ 453 453 - db: d, 454 454 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 455 455 - active: make(map[string]*reservation), 456 456 - maxOutboxBytes: 10 * 1024 * 1024, 457 457 - } 458 458 - if err := e.initOutbox(); err != nil { 459 459 - t.Fatal(err) 460 460 - } 461 461 - 462 462 - twf, _ := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) 463 463 - tpl, _ := json.Marshal(tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) 464 464 - 465 465 - inner := &fakeEngine{} 466 466 - e.engines = map[string]models.Engine{"microvm": inner} 467 467 - 468 468 - e.handleReserve(context.Background(), &millv1.ReserveSeat{ 469 469 - LeaseId: "lease-1", 470 470 - TargetEngine: "microvm", 471 471 - RawWorkflowJson: string(twf), 472 472 - RawPipelineJson: string(tpl), 473 473 - TtlSeconds: 1, 474 474 - }) 475 475 - 476 476 - e.mu.Lock() 477 477 - res := e.active["lease-1"] 478 478 - e.mu.Unlock() 479 479 - 480 480 - if res == nil { 481 481 - t.Fatal("reservation was not added") 482 482 - } 483 483 - 484 484 - deadline := time.Now().Add(5 * time.Second) 485 485 - for { 486 486 - e.mu.Lock() 487 487 - activeLen := len(e.active) 488 488 - e.mu.Unlock() 489 489 - if activeLen == 0 { 490 490 - break 491 491 - } 492 492 - if time.Now().After(deadline) { 493 493 - t.Fatal("reservation was leaked and never expired") 494 494 - } 495 495 - time.Sleep(10 * time.Millisecond) 496 496 - } 497 497 - } 498 498 - 499 499 - func TestStructuredShutdown(t *testing.T) { 500 500 - d := testDB(t) 501 501 - n := notifier.New() 502 502 - e := &Executor{ 503 503 - db: d, 504 504 - n: &n, 505 505 - l: slog.New(slog.NewTextHandler(io.Discard, nil)), 506 506 - active: make(map[string]*reservation), 507 507 - maxOutboxBytes: 10 * 1024 * 1024, 508 508 - cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, 509 509 - } 510 510 - if err := e.initOutbox(); err != nil { 511 511 - t.Fatal(err) 512 512 - } 513 513 - 514 514 - ctx, cancel := context.WithCancel(context.Background()) 515 515 - e.lifecycleCtx = ctx 516 516 - 517 517 - inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})} 518 518 - slot := &fakeSlot{} 519 519 - repoDid, err := syntax.ParseDID("did:web:example.com") 520 520 - if err != nil { 521 521 - t.Fatal(err) 522 522 - } 523 523 - res := &reservation{ 524 524 - leaseID: "lease-1", 525 525 - wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, 526 526 - realEngine: inner, 527 527 - slot: slot, 528 528 - wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, 529 529 - repoDid: repoDid, 530 530 - } 531 531 - e.active[res.leaseID] = res 532 532 - 533 533 - e.handleCommit(ctx, &millv1.CommitLease{ 534 534 - LeaseId: "lease-1", 535 535 - }) 536 536 - 537 537 - cancel() 538 538 - 539 539 - e.jobsWG.Wait() 540 540 - 541 541 - select { 542 542 - case <-inner.done: 543 543 - default: 544 544 - t.Fatal("shutdown returned but running job did not finish") 545 545 - } 546 546 - }

-183

spindle/mill/handler.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "github.com/gorilla/websocket" 5 5 - "io" 6 6 - "net/http" 7 7 - "slices" 8 8 - "strings" 9 9 - "sync" 10 10 - "time" 11 11 - 12 12 - millproto "tangled.org/core/spindle/mill/proto" 13 13 - millv1 "tangled.org/core/spindle/mill/proto/gen" 14 14 - ) 15 15 - 16 16 - var ( 17 17 - handshakeSem = make(chan struct{}, 16) 18 18 - handshakeMu sync.Mutex 19 19 - inFlightHandshakes = make(map[string]struct{}) 20 20 - ) 21 21 - 22 22 - type livenessReader struct { 23 23 - r io.Reader 24 24 - conn *websocket.Conn 25 25 - readTimeout time.Duration 26 26 - } 27 27 - 28 28 - func (lr *livenessReader) Read(p []byte) (int, error) { 29 29 - if err := lr.conn.SetReadDeadline(time.Now().Add(lr.readTimeout)); err != nil { 30 30 - return 0, err 31 31 - } 32 32 - return lr.r.Read(p) 33 33 - } 34 34 - 35 35 - var upgrader = websocket.Upgrader{ 36 36 - ReadBufferSize: 1024, 37 37 - WriteBufferSize: 1024, 38 38 - } 39 39 - 40 40 - // auth before upgrade, a bad token never opens a socket 41 41 - func (m *Mill) HandleExecutorConn(w http.ResponseWriter, r *http.Request) { 42 42 - name, authorizedLabels, ok := m.authenticate(r) 43 43 - if !ok { 44 44 - http.Error(w, "unauthorized", http.StatusUnauthorized) 45 45 - return 46 46 - } 47 47 - 48 48 - select { 49 49 - case handshakeSem <- struct{}{}: 50 50 - case <-r.Context().Done(): 51 51 - return 52 52 - } 53 53 - handshakeSlotHeld := true 54 54 - defer func() { 55 55 - if handshakeSlotHeld { 56 56 - <-handshakeSem 57 57 - } 58 58 - }() 59 59 - 60 60 - // enforces one in-flight handshake and one live session at a time per identity 61 61 - handshakeMu.Lock() 62 62 - if _, ok := inFlightHandshakes[name]; ok { 63 63 - handshakeMu.Unlock() 64 64 - http.Error(w, "handshake already in progress", http.StatusConflict) 65 65 - return 66 66 - } 67 67 - m.mu.Lock() 68 68 - old, exists := m.sessions[name] 69 69 - isLive := exists && old.live(m.cfg.ReconnectGrace) 70 70 - m.mu.Unlock() 71 71 - if isLive { 72 72 - handshakeMu.Unlock() 73 73 - http.Error(w, "session already active", http.StatusConflict) 74 74 - return 75 75 - } 76 76 - inFlightHandshakes[name] = struct{}{} 77 77 - handshakeMu.Unlock() 78 78 - identityHandshakeHeld := true 79 79 - 80 80 - defer func() { 81 81 - if identityHandshakeHeld { 82 82 - handshakeMu.Lock() 83 83 - delete(inFlightHandshakes, name) 84 84 - handshakeMu.Unlock() 85 85 - } 86 86 - }() 87 87 - 88 88 - conn, err := upgrader.Upgrade(w, r, nil) 89 89 - if err != nil { 90 90 - m.l.Error("fleet ws upgrade failed", "err", err) 91 91 - return 92 92 - } 93 93 - defer conn.Close() 94 94 - 95 95 - if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { 96 96 - m.l.Error("failed to set pre-hello read deadline", "err", err) 97 97 - return 98 98 - } 99 99 - 100 100 - stream := millproto.NewWSStream(conn) 101 101 - enc := millproto.NewEncoder(stream) 102 102 - dec := millproto.NewDecoder(stream) 103 103 - 104 104 - hello, err := dec.Decode() 105 105 - if err != nil { 106 106 - m.l.Error("fleet read hello failed", "err", err) 107 107 - return 108 108 - } 109 109 - h := hello.GetHello() 110 110 - if h == nil { 111 111 - m.l.Error("fleet first frame was not hello") 112 112 - return 113 113 - } 114 114 - if h.GetProtocolVersion() != millproto.ProtocolVersion { 115 115 - m.l.Error("fleet protocol version mismatch", "got", h.GetProtocolVersion(), "want", millproto.ProtocolVersion) 116 116 - return 117 117 - } 118 118 - if h.GetEpoch() == "" { 119 119 - m.l.Error("fleet hello missing epoch") 120 120 - return 121 121 - } 122 122 - 123 123 - for _, l := range h.GetLabels() { 124 124 - if !slices.Contains(authorizedLabels, l) { 125 125 - m.l.Error("executor requested unauthorized label", "label", l, "authorized", authorizedLabels) 126 126 - return 127 127 - } 128 128 - } 129 129 - 130 130 - sess := newSession(name, h.GetEpoch(), authorizedLabels, enc, m.l) 131 131 - sess.closeTransport = conn.Close 132 132 - sess.labels = h.GetLabels() 133 133 - 134 134 - resume, ok := m.attachSession(sess) 135 135 - if !ok { 136 136 - m.l.Warn("rejecting duplicate live executor session", "node", name) 137 137 - return 138 138 - } 139 139 - m.l.Info("executor connected", "node", sess.nodeID, "arch", h.GetArch(), "labels", h.GetLabels(), "resume", resume) 140 140 - 141 141 - if err := sess.send(&millproto.Message{Resume: &millv1.Resume{Epoch: h.GetEpoch(), AckSeqno: resume}}); err != nil { 142 142 - m.l.Error("fleet send resume failed", "err", err) 143 143 - m.detachSession(sess) 144 144 - return 145 145 - } 146 146 - handshakeSlotHeld = false 147 147 - <-handshakeSem 148 148 - handshakeMu.Lock() 149 149 - delete(inFlightHandshakes, name) 150 150 - handshakeMu.Unlock() 151 151 - identityHandshakeHeld = false 152 152 - m.sessionReady(sess) 153 153 - 154 154 - readTimeout := m.cfg.ReconnectGrace 155 155 - if readTimeout <= 0 { 156 156 - readTimeout = 45 * time.Second 157 157 - } 158 158 - liveDec := millproto.NewDecoder(&livenessReader{r: stream, conn: conn, readTimeout: readTimeout}) 159 159 - 160 160 - if err := sess.readLoop(m, liveDec); err != nil { 161 161 - m.l.Debug("session read ended", "node", sess.nodeID, "err", err) 162 162 - } 163 163 - m.detachSession(sess) 164 164 - } 165 165 - 166 166 - // identity comes from the token hash. unknown or missing token fails closed 167 167 - func (m *Mill) authenticate(r *http.Request) (string, []string, bool) { 168 168 - const prefix = "Bearer " 169 169 - h := r.Header.Get("Authorization") 170 170 - if !strings.HasPrefix(h, prefix) { 171 171 - return "", nil, false 172 172 - } 173 173 - token := strings.TrimPrefix(h, prefix) 174 174 - if token == "" || m.db == nil { 175 175 - return "", nil, false 176 176 - } 177 177 - name, labels, ok, err := m.db.ResolveExecutorToken(HashToken(token)) 178 178 - if err != nil { 179 179 - m.l.Error("executor token lookup failed", "err", err) 180 180 - return "", nil, false 181 181 - } 182 182 - return name, labels, ok 183 183 - }

-345

spindle/mill/integration_test.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "encoding/json" 6 6 - "io" 7 7 - "log/slog" 8 8 - "net/http" 9 9 - "net/http/httptest" 10 10 - "os" 11 11 - "path/filepath" 12 12 - "strings" 13 13 - "testing" 14 14 - "time" 15 15 - 16 16 - "tangled.org/core/api/tangled" 17 17 - "tangled.org/core/notifier" 18 18 - "tangled.org/core/spindle/config" 19 19 - "tangled.org/core/spindle/db" 20 20 - "tangled.org/core/spindle/engines/dummy" 21 21 - "tangled.org/core/spindle/mill/executor" 22 22 - "tangled.org/core/spindle/models" 23 23 - ) 24 24 - 25 25 - func TestEndToEndDummyJob(t *testing.T) { 26 26 - ctx, cancel := context.WithCancel(context.Background()) 27 27 - defer cancel() 28 28 - l := slog.New(slog.NewTextHandler(io.Discard, nil)) 29 29 - 30 30 - millDir := t.TempDir() 31 31 - bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db")) 32 32 - if err != nil { 33 33 - t.Fatalf("mill db: %v", err) 34 34 - } 35 35 - bn := notifier.New() 36 36 - mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second}) 37 37 - mill.Attach(bdb, &bn) 38 38 - if err := bdb.AddExecutorToken("exec-1", HashToken("test-token"), nil, nil); err != nil { 39 39 - t.Fatalf("register executor token: %v", err) 40 40 - } 41 41 - 42 42 - srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn)) 43 43 - defer srv.Close() 44 44 - wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") 45 45 - 46 46 - execDir := t.TempDir() 47 47 - edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db")) 48 48 - if err != nil { 49 49 - t.Fatalf("exec db: %v", err) 50 50 - } 51 51 - en := notifier.New() 52 52 - cfg := &config.Config{} 53 53 - cfg.Server.Dev = true 54 54 - cfg.Server.LogDir = execDir 55 55 - cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") 56 56 - cfg.Server.Hostname = "exec-1" 57 57 - cfg.Mill.URL = wsURL 58 58 - cfg.Mill.Seats = 2 59 59 - cfg.Mill.SharedSecret = "test-token" 60 60 - 61 61 - dummyEng := dummy.New(l) 62 62 - dummyEng.StepDelay = 50 * time.Millisecond 63 63 - engines := map[string]models.Engine{"dummy": dummyEng} 64 64 - exec, err := executor.New(cfg, engines, edb, &en, l) 65 65 - if err != nil { 66 66 - t.Fatalf("executor.New: %v", err) 67 67 - } 68 68 - go exec.Connect(ctx) 69 69 - 70 70 - be := NewEngine("dummy", mill) 71 71 - twf := tangled.Pipeline_Workflow{ 72 72 - Name: "build", 73 73 - Raw: "steps:\n - name: hello\n command: echo hi\n", 74 74 - } 75 75 - wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) 76 76 - if err != nil { 77 77 - t.Fatalf("InitWorkflow: %v", err) 78 78 - } 79 79 - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} 80 80 - 81 81 - placeCtx, placeCancel := context.WithTimeout(ctx, 10*time.Second) 82 82 - defer placeCancel() 83 83 - 84 84 - slot, err := mill.place(placeCtx, "dummy", wid, wf) 85 85 - if err != nil { 86 86 - t.Fatalf("place: %v", err) 87 87 - } 88 88 - defer slot.Release() 89 89 - 90 90 - logPath := models.LogFilePath(millDir, wid) 91 91 - ch := bn.Subscribe() 92 92 - defer bn.Unsubscribe(ch) 93 93 - 94 94 - sawLogContent := make(chan bool, 1) 95 95 - go func() { 96 96 - ticker := time.NewTicker(20 * time.Millisecond) 97 97 - defer ticker.Stop() 98 98 - timeout := time.After(10 * time.Second) 99 99 - for { 100 100 - select { 101 101 - case <-ch: 102 102 - case <-ticker.C: 103 103 - case <-timeout: 104 104 - sawLogContent <- false 105 105 - return 106 106 - } 107 107 - data, err := os.ReadFile(logPath) 108 108 - if err == nil && strings.Contains(string(data), "echo hi") { 109 109 - sawLogContent <- true 110 110 - return 111 111 - } 112 112 - } 113 113 - }() 114 114 - 115 115 - if err := mill.commitAndWait(placeCtx, wf, nil); err != nil { 116 116 - t.Fatalf("commitAndWait: %v, want success", err) 117 117 - } 118 118 - 119 119 - if !<-sawLogContent { 120 120 - t.Fatal("mill live log file never received expected content while job was running") 121 121 - } 122 122 - 123 123 - if !waitForFileRemoval(t, logPath) { 124 124 - t.Fatalf("mill live log file was not removed after terminal artifact was recorded") 125 125 - } 126 126 - 127 127 - if !waitForStatus(t, bdb, wid, "running") { 128 128 - events, _ := bdb.GetEvents(0, 1000) 129 129 - t.Logf("mill events after completion: %+v", events) 130 130 - t.Fatal("mill never saw streamed running status") 131 131 - } 132 132 - } 133 133 - 134 134 - func TestExecutorConfiguredLabelsAreStoredOnSession(t *testing.T) { 135 135 - ctx, cancel := context.WithCancel(context.Background()) 136 136 - defer cancel() 137 137 - l := slog.New(slog.NewTextHandler(io.Discard, nil)) 138 138 - 139 139 - millDir := t.TempDir() 140 140 - bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db")) 141 141 - if err != nil { 142 142 - t.Fatalf("mill db: %v", err) 143 143 - } 144 144 - bn := notifier.New() 145 145 - mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second}) 146 146 - mill.Attach(bdb, &bn) 147 147 - if err := bdb.AddExecutorToken("exec-labels", HashToken("test-token"), nil, []string{"linux", "arm64", "gpu"}); err != nil { 148 148 - t.Fatalf("register executor token: %v", err) 149 149 - } 150 150 - 151 151 - srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn)) 152 152 - defer srv.Close() 153 153 - wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") 154 154 - 155 155 - execDir := t.TempDir() 156 156 - edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db")) 157 157 - if err != nil { 158 158 - t.Fatalf("exec db: %v", err) 159 159 - } 160 160 - en := notifier.New() 161 161 - cfg := &config.Config{} 162 162 - cfg.Server.Dev = true 163 163 - cfg.Server.LogDir = execDir 164 164 - cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") 165 165 - cfg.Server.Hostname = "exec-labels" 166 166 - cfg.Mill.URL = wsURL 167 167 - cfg.Mill.Seats = 2 168 168 - cfg.Mill.SharedSecret = "test-token" 169 169 - cfg.Mill.Labels = []string{"linux", "arm64", "gpu"} 170 170 - 171 171 - engines := map[string]models.Engine{"dummy": dummy.New(l)} 172 172 - exec, err := executor.New(cfg, engines, edb, &en, l) 173 173 - if err != nil { 174 174 - t.Fatalf("executor.New: %v", err) 175 175 - } 176 176 - go exec.Connect(ctx) 177 177 - 178 178 - if !waitForSessionLabels(t, mill, "exec-labels", []string{"linux", "arm64", "gpu"}) { 179 179 - t.Fatal("mill session never stored executor labels from hello") 180 180 - } 181 181 - } 182 182 - 183 183 - func TestEndToEndDummyJobUsesRequiredLabelsAcrossExecutors(t *testing.T) { 184 184 - ctx, cancel := context.WithCancel(context.Background()) 185 185 - defer cancel() 186 186 - l := slog.New(slog.NewTextHandler(io.Discard, nil)) 187 187 - 188 188 - millDir := t.TempDir() 189 189 - bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db")) 190 190 - if err != nil { 191 191 - t.Fatalf("mill db: %v", err) 192 192 - } 193 193 - bn := notifier.New() 194 194 - mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second}) 195 195 - mill.Attach(bdb, &bn) 196 196 - if err := bdb.AddExecutorToken("exec-x86", HashToken("token-x86"), nil, []string{"linux/amd64", "kvm"}); err != nil { 197 197 - t.Fatalf("register x86 executor token: %v", err) 198 198 - } 199 199 - if err := bdb.AddExecutorToken("exec-arm", HashToken("token-arm"), nil, []string{"linux/arm64", "kvm"}); err != nil { 200 200 - t.Fatalf("register arm executor token: %v", err) 201 201 - } 202 202 - 203 203 - srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn)) 204 204 - defer srv.Close() 205 205 - wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") 206 206 - 207 207 - startExecutor := func(name, token string, labels []string) { 208 208 - t.Helper() 209 209 - execDir := t.TempDir() 210 210 - edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db")) 211 211 - if err != nil { 212 212 - t.Fatalf("%s exec db: %v", name, err) 213 213 - } 214 214 - en := notifier.New() 215 215 - cfg := &config.Config{} 216 216 - cfg.Server.Dev = true 217 217 - cfg.Server.LogDir = execDir 218 218 - cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") 219 219 - cfg.Server.Hostname = name 220 220 - cfg.Mill.URL = wsURL 221 221 - cfg.Mill.Seats = 1 222 222 - cfg.Mill.SharedSecret = token 223 223 - cfg.Mill.Labels = labels 224 224 - 225 225 - engines := map[string]models.Engine{"dummy": dummy.New(l)} 226 226 - exec, err := executor.New(cfg, engines, edb, &en, l) 227 227 - if err != nil { 228 228 - t.Fatalf("executor.New: %v", err) 229 229 - } 230 230 - go exec.Connect(ctx) 231 231 - } 232 232 - startExecutor("exec-x86", "token-x86", []string{"linux/amd64", "kvm"}) 233 233 - startExecutor("exec-arm", "token-arm", []string{"linux/arm64", "kvm"}) 234 234 - 235 235 - if !waitForSessionLabels(t, mill, "exec-x86", []string{"linux/amd64", "kvm"}) { 236 236 - t.Fatal("x86 executor did not connect with labels") 237 237 - } 238 238 - if !waitForSessionLabels(t, mill, "exec-arm", []string{"linux/arm64", "kvm"}) { 239 239 - t.Fatal("arm executor did not connect with labels") 240 240 - } 241 241 - 242 242 - be := NewEngine("dummy", mill) 243 243 - twf := tangled.Pipeline_Workflow{ 244 244 - Name: "build-arm", 245 245 - RunsOn: []string{"linux/arm64"}, 246 246 - Raw: "steps:\n - name: hello\n command: echo hi\n", 247 247 - } 248 248 - wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) 249 249 - if err != nil { 250 250 - t.Fatalf("InitWorkflow: %v", err) 251 251 - } 252 252 - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey-arm"}, Name: "build-arm"} 253 253 - 254 254 - placeCtx, placeCancel := context.WithTimeout(ctx, 10*time.Second) 255 255 - defer placeCancel() 256 256 - 257 257 - slot, err := mill.place(placeCtx, "dummy", wid, wf) 258 258 - if err != nil { 259 259 - t.Fatalf("place: %v", err) 260 260 - } 261 261 - defer slot.Release() 262 262 - 263 263 - lease := wf.Data.(*millWorkflowState).Lease 264 264 - if lease == nil { 265 265 - t.Fatal("place did not attach lease to workflow state") 266 266 - } 267 267 - if lease.nodeID != "exec-arm" { 268 268 - t.Fatalf("placed on %q, want exec-arm", lease.nodeID) 269 269 - } 270 270 - 271 271 - if err := mill.commitAndWait(placeCtx, wf, nil); err != nil { 272 272 - t.Fatalf("commitAndWait: %v, want success", err) 273 273 - } 274 274 - } 275 275 - 276 276 - func waitForSessionLabels(t *testing.T, m *Mill, nodeID string, want []string) bool { 277 277 - t.Helper() 278 278 - deadline := time.Now().Add(3 * time.Second) 279 279 - for time.Now().Before(deadline) { 280 280 - m.mu.Lock() 281 281 - sess := m.sessions[nodeID] 282 282 - var got []string 283 283 - if sess != nil { 284 284 - got = append([]string(nil), sess.labels...) 285 285 - } 286 286 - m.mu.Unlock() 287 287 - if sameStringMultiset(got, want) { 288 288 - return true 289 289 - } 290 290 - time.Sleep(10 * time.Millisecond) 291 291 - } 292 292 - return false 293 293 - } 294 294 - 295 295 - func waitForStatus(t *testing.T, d *db.DB, wid models.WorkflowId, want string) bool { 296 296 - t.Helper() 297 297 - deadline := time.Now().Add(3 * time.Second) 298 298 - aturi := string(wid.PipelineId.AtUri()) 299 299 - for time.Now().Before(deadline) { 300 300 - evs, err := d.GetEvents(0, 1000) 301 301 - if err != nil { 302 302 - t.Fatalf("GetEvents: %v", err) 303 303 - } 304 304 - for _, ev := range evs { 305 305 - if ev.Nsid != tangled.PipelineStatusNSID { 306 306 - continue 307 307 - } 308 308 - var st tangled.PipelineStatus 309 309 - if err := json.Unmarshal(ev.EventJson, &st); err != nil { 310 310 - continue 311 311 - } 312 312 - if st.Pipeline == aturi && st.Workflow == wid.Name && st.Status == want { 313 313 - return true 314 314 - } 315 315 - } 316 316 - time.Sleep(50 * time.Millisecond) 317 317 - } 318 318 - return false 319 319 - } 320 320 - 321 321 - func waitForLogFileContent(t *testing.T, path, want string) bool { 322 322 - t.Helper() 323 323 - deadline := time.Now().Add(5 * time.Second) 324 324 - for time.Now().Before(deadline) { 325 325 - data, err := os.ReadFile(path) 326 326 - if err == nil && strings.Contains(string(data), want) { 327 327 - return true 328 328 - } 329 329 - time.Sleep(20 * time.Millisecond) 330 330 - } 331 331 - return false 332 332 - } 333 333 - 334 334 - func waitForFileRemoval(t *testing.T, path string) bool { 335 335 - t.Helper() 336 336 - deadline := time.Now().Add(5 * time.Second) 337 337 - for time.Now().Before(deadline) { 338 338 - _, err := os.Stat(path) 339 339 - if os.IsNotExist(err) { 340 340 - return true 341 341 - } 342 342 - time.Sleep(20 * time.Millisecond) 343 343 - } 344 344 - return false 345 345 - }

-186

spindle/mill/lease.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "sync" 5 5 - 6 6 - "tangled.org/core/spindle/models" 7 7 - 8 8 - millv1 "tangled.org/core/spindle/mill/proto/gen" 9 9 - ) 10 10 - 11 11 - // the mill's view of a remote attempt 12 12 - type leaseState int32 13 13 - 14 14 - const ( 15 15 - // won a bid, executor is holding a seat, not yet committed 16 16 - leaseReserved leaseState = iota 17 17 - // CommitLease sent. the executor may already be running, but the mill 18 18 - // may not have seen Committed yet 19 19 - leaseCommitting 20 20 - // CommitLease sent and acked. job is running on the executor 21 21 - leaseRunning 22 22 - // terminal result arrived or we gave up, no further action 23 23 - leaseDone 24 24 - ) 25 25 - 26 26 - type cancelAction int 27 27 - 28 28 - const ( 29 29 - cancelNoop cancelAction = iota 30 30 - cancelLocal 31 31 - cancelRemote 32 32 - ) 33 33 - 34 34 - // mill-side fencing token for one placed job 35 35 - type RemoteLease struct { 36 36 - id string 37 37 - nodeID string 38 38 - epoch string 39 39 - engine string 40 40 - wid models.WorkflowId // job this lease carries, set once placed 41 41 - // restored after a mill restart. no RunStep waits on it, so terminals 42 42 - // and death are authored directly. set before publication, never mutated 43 43 - orphaned bool 44 44 - claimed bool 45 45 - cancelAcked bool 46 46 - cleanedUp bool 47 47 - cleanupRetry bool 48 48 - 49 49 - mu sync.Mutex 50 50 - state leaseState 51 51 - cancel bool 52 52 - reason string 53 53 - released bool 54 54 - terminal chan *millv1.AttemptResult // buffered(1), RunStep waits here 55 55 - 56 56 - finishMu sync.Mutex 57 57 - } 58 58 - 59 59 - func newLease(id, nodeID, epoch, engine string) *RemoteLease { 60 60 - return &RemoteLease{ 61 61 - id: id, 62 62 - nodeID: nodeID, 63 63 - epoch: epoch, 64 64 - engine: engine, 65 65 - state: leaseReserved, 66 66 - terminal: make(chan *millv1.AttemptResult, 1), 67 67 - } 68 68 - } 69 69 - 70 70 - func (l *RemoteLease) setState(s leaseState) { 71 71 - l.mu.Lock() 72 72 - l.state = s 73 73 - l.mu.Unlock() 74 74 - } 75 75 - 76 76 - func (l *RemoteLease) markCommitting() bool { 77 77 - l.mu.Lock() 78 78 - defer l.mu.Unlock() 79 79 - if l.state == leaseDone { 80 80 - return false 81 81 - } 82 82 - if l.state == leaseReserved { 83 83 - l.state = leaseCommitting 84 84 - } 85 85 - return true 86 86 - } 87 87 - 88 88 - func (l *RemoteLease) markRunning() { 89 89 - l.mu.Lock() 90 90 - defer l.mu.Unlock() 91 91 - if l.state != leaseDone { 92 92 - l.state = leaseRunning 93 93 - } 94 94 - } 95 95 - 96 96 - func (l *RemoteLease) getState() leaseState { 97 97 - l.mu.Lock() 98 98 - defer l.mu.Unlock() 99 99 - return l.state 100 100 - } 101 101 - 102 102 - // only one caller gets to mark it done 103 103 - func (l *RemoteLease) markDone() bool { 104 104 - l.mu.Lock() 105 105 - defer l.mu.Unlock() 106 106 - if l.state == leaseDone { 107 107 - return false 108 108 - } 109 109 - l.state = leaseDone 110 110 - return true 111 111 - } 112 112 - 113 113 - func (l *RemoteLease) requestCancel(reason string) cancelAction { 114 114 - l.mu.Lock() 115 115 - defer l.mu.Unlock() 116 116 - if l.state == leaseDone { 117 117 - return cancelNoop 118 118 - } 119 119 - l.cancel = true 120 120 - l.reason = reason 121 121 - if l.state == leaseReserved { 122 122 - return cancelLocal 123 123 - } 124 124 - return cancelRemote 125 125 - } 126 126 - 127 127 - func (l *RemoteLease) cancelRequested() (bool, string) { 128 128 - l.mu.Lock() 129 129 - defer l.mu.Unlock() 130 130 - return l.cancel, l.reason 131 131 - } 132 132 - 133 133 - func (l *RemoteLease) releaseState() (leaseState, bool) { 134 134 - l.mu.Lock() 135 135 - defer l.mu.Unlock() 136 136 - l.released = true 137 137 - return l.state, l.cancel 138 138 - } 139 139 - 140 140 - func (l *RemoteLease) cleanupReady() bool { 141 141 - l.mu.Lock() 142 142 - defer l.mu.Unlock() 143 143 - return l.released && l.state == leaseDone 144 144 - } 145 145 - 146 146 - func (l *RemoteLease) deliverCancelled(reason string) { 147 147 - l.deliverTerminal(&millv1.AttemptResult{ 148 148 - Status: millv1.TerminalStatus_CANCELLED, 149 149 - Error: reason, 150 150 - }) 151 151 - } 152 152 - 153 153 - // hands the terminal to a waiting RunStep without blocking. duplicates 154 154 - // (eg. reconnect replays) just drop, the channel holds one and the lease 155 155 - // is already done 156 156 - func (l *RemoteLease) deliverTerminal(res *millv1.AttemptResult) { 157 157 - if !l.markDone() { 158 158 - return 159 159 - } 160 160 - select { 161 161 - case l.terminal <- res: 162 162 - default: 163 163 - } 164 164 - } 165 165 - 166 166 - // mill's synthetic single step. real steps run on the executor and the 167 167 - // mill never mirrors them 168 168 - type remoteStep struct{} 169 169 - 170 170 - func (remoteStep) Name() string { return "remote execution" } 171 171 - func (remoteStep) Command() string { return "" } 172 172 - func (remoteStep) Kind() models.StepKind { return models.StepKindSystem } 173 173 - 174 174 - // what AcquireWorkflowSlot returns. Release unwinds placement 175 175 - type millSlot struct { 176 176 - fleet *Mill 177 177 - lease *RemoteLease 178 178 - once sync.Once 179 179 - } 180 180 - 181 181 - func (s *millSlot) Release() { 182 182 - if s == nil { 183 183 - return 184 184 - } 185 185 - s.once.Do(func() { s.fleet.releaseSlot(s) }) 186 186 - }

-1119

spindle/mill/mill.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "encoding/json" 6 6 - "errors" 7 7 - "fmt" 8 8 - "log/slog" 9 9 - "os" 10 10 - "path/filepath" 11 11 - "slices" 12 12 - "strings" 13 13 - "sync" 14 14 - "time" 15 15 - 16 16 - "tangled.org/core/notifier" 17 17 - "tangled.org/core/spindle/db" 18 18 - "tangled.org/core/spindle/engine" 19 19 - "tangled.org/core/spindle/models" 20 20 - "tangled.org/core/spindle/secrets" 21 21 - "tangled.org/core/tid" 22 22 - 23 23 - millproto "tangled.org/core/spindle/mill/proto" 24 24 - millv1 "tangled.org/core/spindle/mill/proto/gen" 25 25 - ) 26 26 - 27 27 - const ( 28 28 - defaultReconnectGrace = 45 * time.Second 29 29 - defaultJobTimeout = 24 * time.Hour 30 30 - defaultBidTimeout = 5 * time.Second 31 31 - defaultTopK = 3 32 32 - defaultMaxPending = 100 33 33 - ) 34 34 - 35 35 - type Config struct { 36 36 - // mill appends live-tailed executor lines here so logview can follow running remote jobs. 37 37 - LogDir string 38 38 - MaxPending int 39 39 - ReconnectGrace time.Duration 40 40 - JobTimeout time.Duration 41 41 - BidTimeout time.Duration 42 42 - TopK int 43 43 - CancelTimeout time.Duration 44 44 - } 45 45 - 46 46 - type Mill struct { 47 47 - l *slog.Logger 48 48 - cfg Config 49 49 - 50 50 - db *db.DB 51 51 - n *notifier.Notifier 52 52 - 53 53 - mu sync.Mutex 54 54 - sessions map[string]*millSession 55 55 - leases map[string]*RemoteLease 56 56 - reservations map[string]*RemoteLease 57 57 - nodeSeqno map[string]uint64 58 58 - pending int 59 59 - changeCh chan struct{} // closed and replaced to wake placement waiters 60 60 - 61 61 - leaseSeq uint64 62 62 - } 63 63 - 64 64 - func New(l *slog.Logger, cfg Config) *Mill { 65 65 - if cfg.ReconnectGrace <= 0 { 66 66 - cfg.ReconnectGrace = defaultReconnectGrace 67 67 - } 68 68 - if cfg.JobTimeout <= 0 { 69 69 - cfg.JobTimeout = defaultJobTimeout 70 70 - } 71 71 - if cfg.BidTimeout <= 0 { 72 72 - cfg.BidTimeout = defaultBidTimeout 73 73 - } 74 74 - if cfg.TopK <= 0 { 75 75 - cfg.TopK = defaultTopK 76 76 - } 77 77 - if cfg.MaxPending <= 0 { 78 78 - cfg.MaxPending = defaultMaxPending 79 79 - } 80 80 - if cfg.CancelTimeout <= 0 { 81 81 - cfg.CancelTimeout = 10 * time.Second 82 82 - } 83 83 - return &Mill{ 84 84 - l: l, 85 85 - cfg: cfg, 86 86 - sessions: make(map[string]*millSession), 87 87 - leases: make(map[string]*RemoteLease), 88 88 - reservations: make(map[string]*RemoteLease), 89 89 - nodeSeqno: make(map[string]uint64), 90 90 - changeCh: make(chan struct{}), 91 91 - } 92 92 - } 93 93 - 94 94 - func (m *Mill) Attach(d *db.DB, n *notifier.Notifier) { 95 95 - m.mu.Lock() 96 96 - m.db = d 97 97 - m.n = n 98 98 - m.mu.Unlock() 99 99 - } 100 100 - 101 101 - func (m *Mill) nextLeaseID() string { 102 102 - m.mu.Lock() 103 103 - m.leaseSeq++ 104 104 - seq := m.leaseSeq 105 105 - m.mu.Unlock() 106 106 - return fmt.Sprintf("%s-%d", tid.TID(), seq) 107 107 - } 108 108 - 109 109 - func (m *Mill) dropReservation(id string) { 110 110 - m.mu.Lock() 111 111 - delete(m.reservations, id) 112 112 - m.mu.Unlock() 113 113 - } 114 114 - 115 115 - func (m *Mill) notifyChange() { 116 116 - m.mu.Lock() 117 117 - m.notifyChangeLocked() 118 118 - m.mu.Unlock() 119 119 - } 120 120 - 121 121 - func (m *Mill) notifyChangeLocked() { 122 122 - close(m.changeCh) 123 123 - m.changeCh = make(chan struct{}) 124 124 - } 125 125 - 126 126 - func (m *Mill) currentChangeCh() <-chan struct{} { 127 127 - m.mu.Lock() 128 128 - defer m.mu.Unlock() 129 129 - return m.changeCh 130 130 - } 131 131 - 132 132 - func (m *Mill) attachSession(sess *millSession) (uint64, bool) { 133 133 - m.mu.Lock() 134 134 - defer m.mu.Unlock() 135 135 - 136 136 - if old := m.sessions[sess.nodeID]; old != nil { 137 137 - if old.live(m.cfg.ReconnectGrace) { 138 138 - // a second live session for the same identity is a hijack 139 139 - // attempt, reject it 140 140 - return 0, false 141 141 - } 142 142 - if old.graceTimer != nil { 143 143 - old.graceTimer.Stop() 144 144 - } 145 145 - old.close() 146 146 - if old.disconnected { 147 147 - m.l.Info("executor reconnected", "node", sess.nodeID) 148 148 - } else { 149 149 - m.l.Warn("replacing silent executor session", "node", sess.nodeID) 150 150 - } 151 151 - } 152 152 - m.sessions[sess.nodeID] = sess 153 153 - // wakes commit retries waiting out reconnect grace 154 154 - m.notifyChangeLocked() 155 155 - return m.nodeSeqno[sess.nodeID+"/"+sess.epoch], true 156 156 - } 157 157 - 158 158 - func (m *Mill) touchSession(sess *millSession) bool { 159 159 - m.mu.Lock() 160 160 - defer m.mu.Unlock() 161 161 - if m.sessions[sess.nodeID] != sess || sess.disconnected { 162 162 - return false 163 163 - } 164 164 - sess.lastSeen = time.Now() 165 165 - return true 166 166 - } 167 167 - 168 168 - func (m *Mill) detachSession(sess *millSession) { 169 169 - m.mu.Lock() 170 170 - if m.sessions[sess.nodeID] != sess { 171 171 - // already replaced by a reconnect 172 172 - m.mu.Unlock() 173 173 - sess.close() 174 174 - return 175 175 - } 176 176 - sess.disconnected = true 177 177 - sess.graceTimer = time.AfterFunc(m.cfg.ReconnectGrace, func() { m.failLeasesAfterGrace(sess) }) 178 178 - m.mu.Unlock() 179 179 - 180 180 - sess.close() 181 181 - m.l.Warn("executor session lost; entering reconnect grace", "node", sess.nodeID, "grace", m.cfg.ReconnectGrace) 182 182 - m.notifyChange() 183 183 - } 184 184 - 185 185 - func (m *Mill) sessionReady(sess *millSession) { 186 186 - for _, lease := range m.cancelledLeasesForNode(sess.nodeID) { 187 187 - _, reason := lease.cancelRequested() 188 188 - m.sendCancel(sess, lease, reason) 189 189 - } 190 190 - m.notifyChange() 191 191 - } 192 192 - 193 193 - func (m *Mill) cancelledLeasesForNode(nodeID string) []*RemoteLease { 194 194 - m.mu.Lock() 195 195 - var candidates []*RemoteLease 196 196 - for _, lease := range m.leases { 197 197 - if lease.nodeID == nodeID { 198 198 - candidates = append(candidates, lease) 199 199 - } 200 200 - } 201 201 - m.mu.Unlock() 202 202 - 203 203 - var leases []*RemoteLease 204 204 - for _, lease := range candidates { 205 205 - if cancelled, _ := lease.cancelRequested(); cancelled && lease.getState() != leaseDone { 206 206 - leases = append(leases, lease) 207 207 - } 208 208 - } 209 209 - return leases 210 210 - } 211 211 - 212 212 - // reconnect grace ran out without the executor coming back. fails every 213 213 - // lease the node held, and reschedules itself if any failure didn't stick 214 214 - func (m *Mill) failLeasesAfterGrace(sess *millSession) { 215 215 - m.mu.Lock() 216 216 - if m.sessions[sess.nodeID] != sess || !sess.disconnected { 217 217 - // reconnected in the meantime 218 218 - m.mu.Unlock() 219 219 - return 220 220 - } 221 221 - var dead []*RemoteLease 222 222 - for _, lease := range m.leases { 223 223 - if lease.nodeID == sess.nodeID { 224 224 - dead = append(dead, lease) 225 225 - } 226 226 - } 227 227 - m.mu.Unlock() 228 228 - 229 229 - m.l.Warn("executor declared dead; failing its in-flight jobs", "node", sess.nodeID, "jobs", len(dead)) 230 230 - deadReason := "executor lost" 231 231 - success := true 232 232 - for _, lease := range dead { 233 233 - switch { 234 234 - case lease.orphaned: 235 235 - // restored leases have no RunStep waiting, fail them straight 236 236 - // into the event stream 237 237 - if err := m.finishOrphan(lease, string(models.StatusKindFailed), &deadReason, nil); err != nil { 238 238 - m.l.Error("finish orphan after executor loss failed, will retry", "lease", lease.id, "err", err) 239 239 - success = false 240 240 - } 241 241 - default: 242 242 - // live leases have a blocked RunStep, keep a pending cancellation 243 243 - // as the terminal reason 244 244 - status := string(models.StatusKindFailed) 245 245 - reason := deadReason 246 246 - if cancelled, cancelReason := lease.cancelRequested(); cancelled { 247 247 - status = string(models.StatusKindCancelled) 248 248 - reason = cancelReason 249 249 - } 250 250 - if err := m.finishLiveLease(lease, status, reason); err != nil { 251 251 - m.l.Error("finish lease after executor loss failed, will retry", "lease", lease.id, "err", err) 252 252 - success = false 253 253 - } 254 254 - } 255 255 - } 256 256 - 257 257 - m.mu.Lock() 258 258 - if !success { 259 259 - // something didn't finish cleanly, try again shortly. finished 260 260 - // leases skip themselves on the next pass 261 261 - sess.graceTimer = time.AfterFunc(5*time.Second, func() { m.failLeasesAfterGrace(sess) }) 262 262 - m.mu.Unlock() 263 263 - return 264 264 - } 265 265 - 266 266 - // everything failed cleanly. forget the node and wake placement 267 267 - if m.sessions[sess.nodeID] == sess { 268 268 - delete(m.sessions, sess.nodeID) 269 269 - } 270 270 - m.mu.Unlock() 271 271 - m.notifyChange() 272 272 - m.sweepUnclaimedOrphans() 273 273 - } 274 274 - 275 275 - func (m *Mill) place(ctx context.Context, engineName string, wid models.WorkflowId, wf *models.Workflow) (engine.WorkflowSlot, error) { 276 276 - m.mu.Lock() 277 277 - if m.cfg.MaxPending > 0 && m.pending >= m.cfg.MaxPending { 278 278 - max := m.cfg.MaxPending 279 279 - cur := m.pending 280 280 - m.mu.Unlock() 281 281 - return nil, fmt.Errorf("%w: mill has %d pending jobs (max %d)", engine.ErrNoWorkflowSlots, cur, max) 282 282 - } 283 283 - m.pending++ 284 284 - m.mu.Unlock() 285 285 - defer func() { 286 286 - m.mu.Lock() 287 287 - m.pending-- 288 288 - m.mu.Unlock() 289 289 - }() 290 290 - 291 291 - for { 292 292 - if err := ctx.Err(); err != nil { 293 293 - return nil, err 294 294 - } 295 295 - 296 296 - // grab the channel before bidding. a change mid-bid closes it, so 297 297 - // the wait below re-bids right away 298 298 - ch := m.currentChangeCh() 299 299 - 300 300 - lease, err := m.bid(ctx, engineName, wid, wf) 301 301 - if err != nil { 302 302 - return nil, err 303 303 - } 304 304 - if lease != nil { 305 305 - lease.wid = wid 306 306 - if err := m.persistLease(lease, leaseRowReserved); err != nil { 307 307 - m.releaseRemote(lease) 308 308 - m.mu.Lock() 309 309 - delete(m.reservations, lease.id) 310 310 - m.mu.Unlock() 311 311 - return nil, fmt.Errorf("persist reserved mill lease: %w", err) 312 312 - } 313 313 - m.mu.Lock() 314 314 - delete(m.reservations, lease.id) 315 315 - m.leases[lease.id] = lease 316 316 - if st, ok := wf.Data.(*millWorkflowState); ok && st != nil { 317 317 - st.Lease = lease 318 318 - } 319 319 - m.mu.Unlock() 320 320 - return &millSlot{fleet: m, lease: lease}, nil 321 321 - } 322 322 - 323 323 - // no executor available. wait for a change or ctx 324 324 - select { 325 325 - case <-ctx.Done(): 326 326 - return nil, ctx.Err() 327 327 - case <-ch: 328 328 - } 329 329 - } 330 330 - } 331 331 - 332 332 - func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId, wf *models.Workflow) (*RemoteLease, error) { 333 333 - rawPipeline, rawWorkflow, err := marshalJob(wf) 334 334 - if err != nil { 335 335 - return nil, err 336 336 - } 337 337 - 338 338 - requiredLabels := requiredLabels(wf) 339 339 - candidates := m.rankCandidates(engineName, requiredLabels) 340 340 - if len(candidates) == 0 { 341 341 - return nil, nil 342 342 - } 343 343 - 344 344 - type bidResult struct { 345 345 - sess *millSession 346 346 - lease *RemoteLease 347 347 - rank int 348 348 - incompatible bool 349 349 - reason string 350 350 - } 351 351 - limit := m.cfg.TopK 352 352 - if limit <= 0 { 353 353 - limit = len(candidates) 354 354 - } 355 355 - if len(candidates) > limit { 356 356 - candidates = candidates[:limit] 357 357 - } 358 358 - results := make(chan bidResult, limit) 359 359 - ask := func(rank int, sess *millSession) { 360 360 - bidCtx, cancel := context.WithTimeout(ctx, m.cfg.BidTimeout) 361 361 - defer cancel() 362 362 - leaseID := m.nextLeaseID() 363 363 - lease := newLease(leaseID, sess.nodeID, sess.epoch, engineName) 364 364 - m.mu.Lock() 365 365 - m.reservations[leaseID] = lease 366 366 - m.mu.Unlock() 367 367 - msg := &millproto.Message{ReserveSeat: &millv1.ReserveSeat{ 368 368 - LeaseId: leaseID, 369 369 - TargetEngine: engineName, 370 370 - RawPipelineJson: rawPipeline, 371 371 - RawWorkflowJson: rawWorkflow, 372 372 - Knot: wid.Knot, 373 373 - Rkey: wid.Rkey, 374 374 - TtlSeconds: uint32(m.cfg.ReconnectGrace / time.Second), 375 375 - }} 376 376 - resp, err := sess.request(bidCtx, leaseID, msg) 377 377 - if err != nil { 378 378 - m.dropReservation(leaseID) 379 379 - results <- bidResult{} 380 380 - return 381 381 - } 382 382 - rr := resp.GetReserveResult() 383 383 - if rr == nil { 384 384 - m.dropReservation(leaseID) 385 385 - results <- bidResult{} 386 386 - return 387 387 - } 388 388 - if !rr.GetAccepted() { 389 389 - m.dropReservation(leaseID) 390 390 - if rr.GetRejectClass() == millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE { 391 391 - results <- bidResult{sess: sess, rank: rank, incompatible: true, reason: rr.GetRejectReason()} 392 392 - return 393 393 - } 394 394 - results <- bidResult{} 395 395 - return 396 396 - } 397 397 - results <- bidResult{sess: sess, lease: lease, rank: rank} 398 398 - } 399 399 - next := 0 400 400 - inFlight := 0 401 401 - for next < len(candidates) && inFlight < limit { 402 402 - inFlight++ 403 403 - go ask(next, candidates[next]) 404 404 - next++ 405 405 - } 406 406 - 407 407 - var winner *bidResult 408 408 - var losers []*RemoteLease 409 409 - var incompatible []string 410 410 - // any soft reject (transient or timeout) means the fleet was just 411 411 - // busy, so an all-incompatible outcome isn't a hard placement error 412 412 - softReject := false 413 413 - for inFlight > 0 { 414 414 - r := <-results 415 415 - inFlight-- 416 416 - // incompatible rejects get reported to the user, any other failure 417 417 - // just means the fleet is busy 418 418 - if r.incompatible { 419 419 - if r.reason != "" { 420 420 - incompatible = append(incompatible, r.reason) 421 421 - } 422 422 - } else if r.lease == nil { 423 423 - softReject = true 424 424 - } 425 425 - // a failed bid means ask the next candidate, unless someone won 426 426 - if r.lease == nil { 427 427 - for winner == nil && next < len(candidates) && inFlight < limit { 428 428 - inFlight++ 429 429 - go ask(next, candidates[next]) 430 430 - next++ 431 431 - } 432 432 - continue 433 433 - } 434 434 - // if this bid is worse than the winner it goes to the losers pile 435 435 - if winner != nil && r.rank >= winner.rank { 436 436 - losers = append(losers, r.lease) 437 437 - continue 438 438 - } 439 439 - // otherwise it's the new best and the old winner joins the losers 440 440 - if winner != nil { 441 441 - losers = append(losers, winner.lease) 442 442 - } 443 443 - winner = &r 444 444 - } 445 445 - 446 446 - // let the losers go so they free their seats right away 447 447 - for _, l := range losers { 448 448 - m.dropReservation(l.id) 449 449 - m.releaseRemote(l) 450 450 - } 451 451 - 452 452 - if winner == nil { 453 453 - if len(incompatible) > 0 && !softReject { 454 454 - return nil, fmt.Errorf("no compatible executor for %s: %s", engineName, strings.Join(incompatible, "; ")) 455 455 - } 456 456 - return nil, nil 457 457 - } 458 458 - return winner.lease, nil 459 459 - } 460 460 - 461 461 - // ranks nodes that are least busy first. if a resource is used a lot 462 462 - // then that node will lose to one that is more even across the board. 463 463 - func (m *Mill) rankCandidates(engineName string, requiredLabels []string) []*millSession { 464 464 - m.mu.Lock() 465 465 - defer m.mu.Unlock() 466 466 - 467 467 - type ranked struct { 468 468 - sess *millSession 469 469 - worst float64 470 470 - sum float64 471 471 - } 472 472 - var rs []ranked 473 473 - for _, s := range m.sessions { 474 474 - // only live, reporting sessions can take work 475 475 - if s.disconnected { 476 476 - continue 477 477 - } 478 478 - if s.snapshot == nil { 479 479 - continue 480 480 - } 481 481 - // the engine has to exist and have room right now 482 482 - ea, ok := s.snapshot.GetEngines()[engineName] 483 483 - if !ok || !ea.GetAvailable() { 484 484 - continue 485 485 - } 486 486 - // and satisfy the wf's label requirements 487 487 - if !hasLabels(s.labels, requiredLabels) { 488 488 - continue 489 489 - } 490 490 - worst, sum := loadScore(ea.GetLoad()) 491 491 - rs = append(rs, ranked{sess: s, worst: worst, sum: sum}) 492 492 - } 493 493 - slices.SortStableFunc(rs, func(a, b ranked) int { 494 494 - if a.worst < b.worst { 495 495 - return -1 496 496 - } 497 497 - if a.worst > b.worst { 498 498 - return 1 499 499 - } 500 500 - if a.sum < b.sum { 501 501 - return -1 502 502 - } 503 503 - if a.sum > b.sum { 504 504 - return 1 505 505 - } 506 506 - return 0 507 507 - }) 508 508 - 509 509 - out := make([]*millSession, len(rs)) 510 510 - for i := range rs { 511 511 - out[i] = rs[i].sess 512 512 - } 513 513 - return out 514 514 - } 515 515 - 516 516 - func loadScore(load map[string]float64) (worst, sum float64) { 517 517 - for _, v := range load { 518 518 - if v > worst { 519 519 - worst = v 520 520 - } 521 521 - sum += v 522 522 - } 523 523 - return worst, sum 524 524 - } 525 525 - 526 526 - func requiredLabels(wf *models.Workflow) []string { 527 527 - st, ok := wf.Data.(*millWorkflowState) 528 528 - if !ok || st == nil { 529 529 - return nil 530 530 - } 531 531 - return st.RawWorkflow.RunsOn 532 532 - } 533 533 - 534 534 - func hasLabels(labels []string, required []string) bool { 535 535 - for _, want := range required { 536 536 - if !slices.Contains(labels, want) { 537 537 - return false 538 538 - } 539 539 - } 540 540 - return true 541 541 - } 542 542 - 543 543 - func (m *Mill) commitAndWait(ctx context.Context, wf *models.Workflow, unlocked []secrets.UnlockedSecret) error { 544 544 - st, ok := wf.Data.(*millWorkflowState) 545 545 - if !ok || st == nil || st.Lease == nil { 546 546 - return fmt.Errorf("mill workflow state missing lease") 547 547 - } 548 548 - lease := st.Lease 549 549 - 550 550 - pbSecrets := make([]*millv1.Secret, len(unlocked)) 551 551 - for i, s := range unlocked { 552 552 - pbSecrets[i] = &millv1.Secret{Key: s.Key, Value: s.Value} 553 553 - } 554 554 - 555 555 - commit := &millproto.Message{CommitLease: &millv1.CommitLease{ 556 556 - LeaseId: lease.id, 557 557 - Secrets: pbSecrets, 558 558 - }} 559 559 - 560 560 - // commit retries ride reconnects, a reservation outlives one 561 561 - // disconnect. lost session or slow executor just means wait and retry, 562 562 - // only job timeout or a dead lease stops the loop 563 563 - for { 564 564 - if res, ok := pollTerminal(lease); ok { 565 565 - return terminalError(res.Status) 566 566 - } 567 567 - if !lease.markCommitting() { 568 568 - return engine.ErrWorkflowFailed 569 569 - } 570 570 - 571 571 - sess := m.sessionForNode(lease.nodeID) 572 572 - if sess == nil { 573 573 - if done, err := m.waitCommitRetry(ctx, lease); done || err != nil { 574 574 - return err 575 575 - } 576 576 - continue 577 577 - } 578 578 - 579 579 - reqCtx, cancel := context.WithTimeout(ctx, m.cfg.BidTimeout) 580 580 - resp, err := sess.request(reqCtx, lease.id, commit) 581 581 - cancel() 582 582 - if err != nil { 583 583 - switch { 584 584 - case errors.Is(err, errSessionClosed): 585 585 - // session died mid-request. wait out the grace, then retry on 586 586 - // the new one 587 587 - if done, err := m.waitCommitRetry(ctx, lease); done || err != nil { 588 588 - return err 589 589 - } 590 590 - continue 591 591 - case errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil: 592 592 - // executor didn't answer in time, but its seat is still held so 593 593 - // retrying is safe 594 594 - continue 595 595 - case errors.Is(err, context.DeadlineExceeded): 596 596 - // the job ctx itself ran out, a real timeout 597 597 - return engine.ErrTimedOut 598 598 - case errors.Is(err, context.Canceled): 599 599 - return err 600 600 - default: 601 601 - m.l.Warn("commit lease send failed; waiting for reconnect", "lease", lease.id, "node", lease.nodeID, "err", err) 602 602 - if done, err := m.waitCommitRetry(ctx, lease); done || err != nil { 603 603 - return err 604 604 - } 605 605 - continue 606 606 - } 607 607 - } 608 608 - if resp.GetCommitted() == nil { 609 609 - return engine.ErrWorkflowFailed 610 610 - } 611 611 - lease.markRunning() 612 612 - if err := m.persistLease(lease, leaseRowRunning); err != nil { 613 613 - m.l.Error("persist running mill lease", "lease", lease.id, "err", err) 614 614 - } 615 615 - if cancelled, reason := lease.cancelRequested(); cancelled { 616 616 - m.sendCancel(sess, lease, reason) 617 617 - } 618 618 - break 619 619 - } 620 620 - 621 621 - select { 622 622 - case res := <-lease.terminal: 623 623 - return terminalError(res.Status) 624 624 - case <-ctx.Done(): 625 625 - if ctx.Err() == context.DeadlineExceeded { 626 626 - return engine.ErrTimedOut 627 627 - } 628 628 - return ctx.Err() 629 629 - } 630 630 - } 631 631 - 632 632 - func (m *Mill) waitCommitRetry(ctx context.Context, lease *RemoteLease) (bool, error) { 633 633 - // grab the channel before checking for a live session again 634 634 - // a reconnect will still close the channel if it happens in between 635 635 - ch := m.currentChangeCh() 636 636 - if m.sessionForNode(lease.nodeID) != nil { 637 637 - return false, nil 638 638 - } 639 639 - select { 640 640 - case res := <-lease.terminal: 641 641 - return true, terminalError(res.Status) 642 642 - case <-ctx.Done(): 643 643 - if ctx.Err() == context.DeadlineExceeded { 644 644 - return true, engine.ErrTimedOut 645 645 - } 646 646 - return true, ctx.Err() 647 647 - case <-ch: 648 648 - return false, nil 649 649 - } 650 650 - } 651 651 - 652 652 - func terminalError(status millv1.TerminalStatus) error { 653 653 - switch status { 654 654 - case millv1.TerminalStatus_SUCCESS: 655 655 - return nil 656 656 - case millv1.TerminalStatus_TIMEOUT: 657 657 - return engine.ErrTimedOut 658 658 - case millv1.TerminalStatus_CANCELLED: 659 659 - return engine.ErrWorkflowCanceled 660 660 - default: 661 661 - return engine.ErrWorkflowFailed 662 662 - } 663 663 - } 664 664 - 665 665 - func pollTerminal(lease *RemoteLease) (*millv1.AttemptResult, bool) { 666 666 - select { 667 667 - case res := <-lease.terminal: 668 668 - return res, true 669 669 - default: 670 670 - return nil, false 671 671 - } 672 672 - } 673 673 - 674 674 - func (m *Mill) destroy(wid models.WorkflowId) { 675 675 - m.mu.Lock() 676 676 - var lease *RemoteLease 677 677 - for _, l := range m.leases { 678 678 - if l.wid == wid { 679 679 - lease = l 680 680 - break 681 681 - } 682 682 - } 683 683 - m.mu.Unlock() 684 684 - if lease == nil { 685 685 - return 686 686 - } 687 687 - reason := "workflow destroyed" 688 688 - switch lease.requestCancel(reason) { 689 689 - case cancelLocal: 690 690 - if sess := m.sessionForNode(lease.nodeID); sess != nil { 691 691 - _ = sess.send(&millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: lease.id}}) 692 692 - } 693 693 - lease.deliverCancelled(reason) 694 694 - case cancelRemote: 695 695 - if sess := m.sessionForNode(lease.nodeID); sess != nil { 696 696 - m.sendCancel(sess, lease, reason) 697 697 - } 698 698 - } 699 699 - } 700 700 - 701 701 - func (m *Mill) releaseSlot(s *millSlot) { 702 702 - lease := s.lease 703 703 - state, cancelled := lease.releaseState() 704 704 - if state == leaseReserved { 705 705 - m.releaseRemote(lease) 706 706 - } else if cancelled && state != leaseDone { 707 707 - return 708 708 - } 709 709 - if err := m.cleanupLease(lease); err != nil { 710 710 - m.l.Error("releaseSlot cleanupLease failed", "lease", lease.id, "err", err) 711 711 - } 712 712 - } 713 713 - func (m *Mill) cleanupLease(lease *RemoteLease) error { 714 714 - lease.finishMu.Lock() 715 715 - defer lease.finishMu.Unlock() 716 716 - return m.cleanupLeaseLocked(lease) 717 717 - } 718 718 - 719 719 - func (m *Mill) cleanupLeaseLocked(lease *RemoteLease) error { 720 720 - if lease.cleanedUp { 721 721 - return nil 722 722 - } 723 723 - if m.db != nil { 724 724 - if err := m.db.DeleteMillLease(lease.id); err != nil { 725 725 - m.scheduleCleanupLocked(lease) 726 726 - return err 727 727 - } 728 728 - } 729 729 - m.mu.Lock() 730 730 - delete(m.leases, lease.id) 731 731 - m.mu.Unlock() 732 732 - lease.cleanedUp = true 733 733 - m.notifyChange() 734 734 - return nil 735 735 - } 736 736 - 737 737 - func (m *Mill) scheduleCleanupLocked(lease *RemoteLease) { 738 738 - if lease.cleanedUp || lease.cleanupRetry { 739 739 - return 740 740 - } 741 741 - lease.cleanupRetry = true 742 742 - time.AfterFunc(5*time.Second, func() { 743 743 - lease.finishMu.Lock() 744 744 - lease.cleanupRetry = false 745 745 - err := m.cleanupLeaseLocked(lease) 746 746 - lease.finishMu.Unlock() 747 747 - if err != nil { 748 748 - m.l.Error("retry lease cleanup failed", "lease", lease.id, "err", err) 749 749 - } 750 750 - }) 751 751 - } 752 752 - 753 753 - func (m *Mill) releaseRemote(lease *RemoteLease) { 754 754 - lease.setState(leaseDone) 755 755 - if sess := m.sessionForNode(lease.nodeID); sess != nil { 756 756 - _ = sess.send(&millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: lease.id}}) 757 757 - } 758 758 - } 759 759 - 760 760 - func (m *Mill) sendCancel(sess *millSession, lease *RemoteLease, reason string) { 761 761 - if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{ 762 762 - LeaseId: lease.id, 763 763 - Reason: reason, 764 764 - }}); err != nil { 765 765 - return 766 766 - } 767 767 - time.AfterFunc(m.cfg.CancelTimeout, func() { m.checkCancelDeadline(lease) }) 768 768 - } 769 769 - 770 770 - func (m *Mill) sessionForNode(nodeID string) *millSession { 771 771 - m.mu.Lock() 772 772 - defer m.mu.Unlock() 773 773 - sess := m.sessions[nodeID] 774 774 - if sess == nil || sess.disconnected { 775 775 - return nil 776 776 - } 777 777 - return sess 778 778 - } 779 779 - 780 780 - func (m *Mill) onSnapshot(sess *millSession, snap *millv1.NodeSnapshot) error { 781 781 - m.mu.Lock() 782 782 - if sess.snapshot != nil && snap.Seqno <= sess.snapshot.Seqno { 783 783 - m.mu.Unlock() 784 784 - return fmt.Errorf("protocol error: snapshot seqno regression. Got %d, last seen %d", snap.Seqno, sess.snapshot.Seqno) 785 785 - } 786 786 - sess.snapshot = snap 787 787 - m.mu.Unlock() 788 788 - 789 789 - if err := m.reconcileLeases(sess, snap.GetActiveLeaseIds()); err != nil { 790 790 - return err 791 791 - } 792 792 - m.mu.Lock() 793 793 - for _, id := range snap.GetActiveLeaseIds() { 794 794 - if lease := m.leases[id]; lease != nil && lease.nodeID == sess.nodeID && lease.epoch == sess.epoch { 795 795 - lease.claimed = true 796 796 - } 797 797 - } 798 798 - m.mu.Unlock() 799 799 - m.notifyChange() 800 800 - return nil 801 801 - } 802 802 - 803 803 - func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { 804 804 - if batch == nil { 805 805 - return nil 806 806 - } 807 807 - 808 808 - if batch.Epoch != sess.epoch { 809 809 - return fmt.Errorf("protocol error: batch epoch %q does not match session %q", batch.Epoch, sess.epoch) 810 810 - } 811 811 - 812 812 - m.mu.Lock() 813 813 - currentKey := sess.nodeID + "/" + sess.epoch 814 814 - current := m.nodeSeqno[currentKey] 815 815 - m.mu.Unlock() 816 816 - 817 817 - expected := current + 1 818 818 - var newEntries []*millv1.Event 819 819 - for _, entry := range batch.Events { 820 820 - // reconnect replays old seqnos, drop those 821 821 - if entry.Seqno <= current { 822 822 - continue 823 823 - } 824 824 - // gap means executor lost rows. so dont apply a partial batch 825 825 - if entry.Seqno != expected { 826 826 - return fmt.Errorf("protocol error: gap in stream seqnos. Expected %d, got %d", expected, entry.Seqno) 827 827 - } 828 828 - newEntries = append(newEntries, entry) 829 829 - expected++ 830 830 - } 831 831 - 832 832 - // all replays, still ack so the executor can trim its outbox 833 833 - if len(newEntries) == 0 { 834 834 - return m.sendAck(sess, current) 835 835 - } 836 836 - 837 837 - type pendingTerminal struct { 838 838 - lease *RemoteLease 839 839 - ar *millv1.AttemptResult 840 840 - } 841 841 - var pendingTerminals []pendingTerminal 842 842 - var artifactLeases []*RemoteLease 843 843 - finishedInBatch := make(map[string]struct{}) 844 844 - 845 845 - var highestSeqno uint64 = current 846 846 - 847 847 - applyFunc := func(tx *db.EventBatchTx) error { 848 848 - for _, entry := range newEntries { 849 849 - m.mu.Lock() 850 850 - lease := m.leases[entry.LeaseId] 851 851 - m.mu.Unlock() 852 852 - 853 853 - // events for leases this node doesn't own are skipped but still 854 854 - // count as processed 855 855 - if lease == nil || lease.nodeID != sess.nodeID { 856 856 - highestSeqno = entry.Seqno 857 857 - continue 858 858 - } 859 859 - 860 860 - // events arriving for a different epoch are invalid (different session) 861 861 - if lease.epoch != "" && lease.epoch != sess.epoch { 862 862 - return fmt.Errorf("protocol error: lease %q epoch %q does not match session %q", lease.id, lease.epoch, sess.epoch) 863 863 - } 864 864 - 865 865 - lease.mu.Lock() 866 866 - state := lease.state 867 867 - lease.mu.Unlock() 868 868 - 869 869 - // done leases can replay terminals on reconnect, skip them 870 870 - if state == leaseDone { 871 871 - highestSeqno = entry.Seqno 872 872 - continue 873 873 - } 874 874 - 875 875 - if _, finished := finishedInBatch[lease.id]; finished { 876 876 - return fmt.Errorf("protocol error: stream entry follows terminal for lease %q", lease.id) 877 877 - } 878 878 - 879 879 - switch { 880 880 - case entry.GetStatusEvent() != nil: 881 881 - ev := entry.GetStatusEvent() 882 882 - statusStr := string(models.StatusKindRunning) 883 883 - var errMsg *string 884 884 - if e := ev.GetError(); e != "" { 885 885 - errMsg = &e 886 886 - } 887 887 - var exitCode *int64 888 888 - if c := ev.GetExitCode(); c != 0 { 889 889 - exitCode = &c 890 890 - } 891 891 - pipelineAtUri := string(lease.wid.PipelineId.AtUri()) 892 892 - if tx != nil { 893 893 - if err := tx.InsertStatusEvent(pipelineAtUri, lease.wid.Name, statusStr, errMsg, exitCode); err != nil { 894 894 - return err 895 895 - } 896 896 - } 897 897 - 898 898 - case entry.GetAttemptResult() != nil: 899 899 - ar := entry.GetAttemptResult() 900 900 - statusStr := "success" 901 901 - switch ar.Status { 902 902 - case millv1.TerminalStatus_SUCCESS: 903 903 - statusStr = "success" 904 904 - case millv1.TerminalStatus_FAILED: 905 905 - statusStr = "failed" 906 906 - case millv1.TerminalStatus_TIMEOUT: 907 907 - statusStr = "timeout" 908 908 - case millv1.TerminalStatus_CANCELLED: 909 909 - statusStr = "cancelled" 910 910 - default: 911 911 - return fmt.Errorf("protocol error: unsupported terminal status %v", ar.Status) 912 912 - } 913 913 - var errMsg *string 914 914 - if e := ar.GetError(); e != "" { 915 915 - errMsg = &e 916 916 - } 917 917 - var exitCode *int64 918 918 - if c := ar.GetExitCode(); c != 0 { 919 919 - exitCode = &c 920 920 - } 921 921 - finishedInBatch[lease.id] = struct{}{} 922 922 - pipelineAtUri := string(lease.wid.PipelineId.AtUri()) 923 923 - if tx != nil { 924 924 - if err := tx.InsertStatusEvent(pipelineAtUri, lease.wid.Name, statusStr, errMsg, exitCode); err != nil { 925 925 - return err 926 926 - } 927 927 - if err := tx.DeleteLease(lease.id); err != nil { 928 928 - return err 929 929 - } 930 930 - if a := ar.GetLogArtifact(); a != nil { 931 931 - if a.GetRef() == "" { 932 932 - return fmt.Errorf("protocol error: empty log artifact ref") 933 933 - } 934 934 - if !strings.HasPrefix(a.GetHash(), "sha256:") { 935 935 - return fmt.Errorf("protocol error: invalid log artifact hash %q", a.GetHash()) 936 936 - } 937 937 - if tx != nil { 938 938 - if err := tx.InsertArtifactRef(lease.id, lease.wid.Name, a.GetRef(), a.GetHash()); err != nil { 939 939 - return err 940 940 - } 941 941 - } 942 942 - artifactLeases = append(artifactLeases, lease) 943 943 - } 944 944 - } 945 945 - pendingTerminals = append(pendingTerminals, pendingTerminal{ 946 946 - lease: lease, 947 947 - ar: ar, 948 948 - }) 949 949 - } 950 950 - 951 951 - highestSeqno = entry.Seqno 952 952 - } 953 953 - 954 954 - if tx != nil { 955 955 - return tx.AdvanceCursor(sess.nodeID, sess.epoch, highestSeqno) 956 956 - } 957 957 - return nil 958 958 - } 959 959 - 960 960 - var err error 961 961 - if m.db != nil { 962 962 - err = m.db.ApplyEventBatch(m.n, applyFunc) 963 963 - } else { 964 964 - err = applyFunc(nil) 965 965 - } 966 966 - 967 967 - if err != nil { 968 968 - return err 969 969 - } 970 970 - 971 971 - if m.cfg.LogDir != "" { 972 972 - for _, lease := range artifactLeases { 973 973 - path := models.LogFilePath(m.cfg.LogDir, lease.wid) 974 974 - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { 975 975 - m.l.Warn("failed to remove live log file after artifact recorded", "path", path, "err", err) 976 976 - } 977 977 - } 978 978 - } 979 979 - 980 980 - m.mu.Lock() 981 981 - if highestSeqno > m.nodeSeqno[currentKey] { 982 982 - m.nodeSeqno[currentKey] = highestSeqno 983 983 - } 984 984 - m.mu.Unlock() 985 985 - 986 986 - for _, pt := range pendingTerminals { 987 987 - // orphans have no waiting RunStep, just mark and clean up 988 988 - if pt.lease.orphaned { 989 989 - pt.lease.markDone() 990 990 - _ = m.cleanupLease(pt.lease) 991 991 - continue 992 992 - } 993 993 - pt.lease.deliverTerminal(pt.ar) 994 994 - if pt.lease.cleanupReady() { 995 995 - _ = m.cleanupLease(pt.lease) 996 996 - } 997 997 - } 998 998 - 999 999 - return m.sendAck(sess, highestSeqno) 1000 1000 - } 1001 1001 - 1002 1002 - func (m *Mill) sendAck(sess *millSession, seqno uint64) error { 1003 1003 - msg := &millproto.Message{Ack: &millv1.Ack{ 1004 1004 - Epoch: sess.epoch, 1005 1005 - UpToSeqno: seqno, 1006 1006 - }} 1007 1007 - if err := sess.send(msg); err != nil { 1008 1008 - return fmt.Errorf("send ack message: %w", err) 1009 1009 - } 1010 1010 - return nil 1011 1011 - } 1012 1012 - func (m *Mill) onLiveLog(sess *millSession, ll *millv1.LiveLog) error { 1013 1013 - if ll == nil || ll.GetLeaseId() == "" { 1014 1014 - return nil 1015 1015 - } 1016 1016 - m.mu.Lock() 1017 1017 - lease := m.leases[ll.GetLeaseId()] 1018 1018 - m.mu.Unlock() 1019 1019 - if lease == nil || lease.nodeID != sess.nodeID { 1020 1020 - return nil 1021 1021 - } 1022 1022 - raw := ll.GetRawJson() 1023 1023 - if m.cfg.LogDir == "" || len(raw) == 0 { 1024 1024 - if m.n != nil { 1025 1025 - m.n.NotifyAll() 1026 1026 - } 1027 1027 - return nil 1028 1028 - } 1029 1029 - lease.mu.Lock() 1030 1030 - isDone := (lease.state == leaseDone) 1031 1031 - lease.mu.Unlock() 1032 1032 - if isDone { 1033 1033 - if m.n != nil { 1034 1034 - m.n.NotifyAll() 1035 1035 - } 1036 1036 - return nil 1037 1037 - } 1038 1038 - 1039 1039 - logPath := models.LogFilePath(m.cfg.LogDir, lease.wid) 1040 1040 - if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { 1041 1041 - m.l.Warn("failed to create log dir", "path", filepath.Dir(logPath), "err", err) 1042 1042 - if m.n != nil { 1043 1043 - m.n.NotifyAll() 1044 1044 - } 1045 1045 - return nil 1046 1046 - } 1047 1047 - f, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) 1048 1048 - if err != nil { 1049 1049 - m.l.Warn("failed to open log file", "path", logPath, "err", err) 1050 1050 - if m.n != nil { 1051 1051 - m.n.NotifyAll() 1052 1052 - } 1053 1053 - return nil 1054 1054 - } 1055 1055 - if _, err := f.Write(raw); err != nil { 1056 1056 - m.l.Warn("failed to write log file", "path", logPath, "err", err) 1057 1057 - } 1058 1058 - _ = f.Close() 1059 1059 - 1060 1060 - if m.n != nil { 1061 1061 - m.n.NotifyAll() 1062 1062 - } 1063 1063 - return nil 1064 1064 - } 1065 1065 - 1066 1066 - func (m *Mill) onCancelAck(sess *millSession, ca *millv1.CancelAck) { 1067 1067 - m.mu.Lock() 1068 1068 - lease := m.leases[ca.GetLeaseId()] 1069 1069 - m.mu.Unlock() 1070 1070 - if lease == nil || lease.nodeID != sess.nodeID { 1071 1071 - return 1072 1072 - } 1073 1073 - lease.mu.Lock() 1074 1074 - lease.cancelAcked = true 1075 1075 - lease.mu.Unlock() 1076 1076 - } 1077 1077 - 1078 1078 - func (m *Mill) checkCancelDeadline(lease *RemoteLease) { 1079 1079 - lease.mu.Lock() 1080 1080 - isDone := (lease.state == leaseDone) 1081 1081 - isAcked := lease.cancelAcked 1082 1082 - lease.mu.Unlock() 1083 1083 - 1084 1084 - if isDone && isAcked { 1085 1085 - return 1086 1086 - } 1087 1087 - 1088 1088 - m.l.Warn("node failed to comply with cancel request within deadline, quarantining", "node", lease.nodeID, "lease", lease.id, "done", isDone, "acked", isAcked) 1089 1089 - 1090 1090 - reason := fmt.Sprintf("cancel noncompliance for lease %s (done: %t, acked: %t)", lease.id, isDone, isAcked) 1091 1091 - if m.db != nil { 1092 1092 - if err := m.db.QuarantineExecutor(lease.nodeID, reason); err != nil { 1093 1093 - m.l.Error("failed to quarantine executor", "node", lease.nodeID, "err", err) 1094 1094 - } 1095 1095 - } 1096 1096 - 1097 1097 - m.mu.Lock() 1098 1098 - sess := m.sessions[lease.nodeID] 1099 1099 - m.mu.Unlock() 1100 1100 - if sess != nil { 1101 1101 - sess.close() 1102 1102 - } 1103 1103 - } 1104 1104 - 1105 1105 - func marshalJob(wf *models.Workflow) (pipeline string, workflow string, err error) { 1106 1106 - st, ok := wf.Data.(*millWorkflowState) 1107 1107 - if !ok || st == nil { 1108 1108 - return "", "", fmt.Errorf("mill workflow state missing") 1109 1109 - } 1110 1110 - p, err := json.Marshal(st.RawPipeline) 1111 1111 - if err != nil { 1112 1112 - return "", "", fmt.Errorf("marshal pipeline: %w", err) 1113 1113 - } 1114 1114 - w, err := json.Marshal(st.RawWorkflow) 1115 1115 - if err != nil { 1116 1116 - return "", "", fmt.Errorf("marshal workflow: %w", err) 1117 1117 - } 1118 1118 - return string(p), string(w), nil 1119 1119 - }

-873

spindle/mill/mill_test.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "errors" 6 6 - "io" 7 7 - "log/slog" 8 8 - "sync" 9 9 - "testing" 10 10 - "time" 11 11 - 12 12 - "tangled.org/core/api/tangled" 13 13 - "tangled.org/core/spindle/engine" 14 14 - "tangled.org/core/spindle/models" 15 15 - 16 16 - millproto "tangled.org/core/spindle/mill/proto" 17 17 - millv1 "tangled.org/core/spindle/mill/proto/gen" 18 18 - ) 19 19 - 20 20 - type scriptedEncoder func(*millproto.Message) error 21 21 - 22 22 - func (e scriptedEncoder) Encode(msg *millproto.Message) error { return e(msg) } 23 23 - 24 24 - func testWorkflow(name string) *models.Workflow { 25 25 - return &models.Workflow{ 26 26 - Name: name, 27 27 - Environment: map[string]string{}, 28 28 - Steps: []models.Step{remoteStep{}}, 29 29 - Data: &millWorkflowState{ 30 30 - RawWorkflow: tangled.Pipeline_Workflow{Name: name}, 31 31 - RawPipeline: tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}, 32 32 - }, 33 33 - } 34 34 - } 35 35 - 36 36 - func testWorkflowWithRunsOn(name string, runsOn []string) *models.Workflow { 37 37 - wf := testWorkflow(name) 38 38 - wf.Data.(*millWorkflowState).RawWorkflow.RunsOn = runsOn 39 39 - return wf 40 40 - } 41 41 - 42 42 - func addCandidateSession(t *testing.T, m *Mill, nodeID string, labels []string, load float64, enc messageEncoder) *millSession { 43 43 - t.Helper() 44 44 - if enc == nil { 45 45 - enc = scriptedEncoder(func(*millproto.Message) error { return nil }) 46 46 - } 47 47 - sess := newSession(nodeID, "inc-"+nodeID, labels, enc, slog.New(slog.NewTextHandler(io.Discard, nil))) 48 48 - sess.snapshot = &millv1.NodeSnapshot{ 49 49 - Seqno: 1, 50 50 - Engines: map[string]*millv1.EngineAvailability{ 51 51 - "dummy": {Available: load < 1.0, Load: map[string]float64{"slots": load}}, 52 52 - }, 53 53 - } 54 54 - m.mu.Lock() 55 55 - m.sessions[nodeID] = sess 56 56 - m.mu.Unlock() 57 57 - return sess 58 58 - } 59 59 - 60 60 - func assertRankedNodes(t *testing.T, got []*millSession, want []string) { 61 61 - t.Helper() 62 62 - if len(got) != len(want) { 63 63 - t.Fatalf("rankCandidates() returned %d candidates, want %d: got %v want %v", len(got), len(want), sessionIDs(got), want) 64 64 - } 65 65 - for i := range want { 66 66 - if got[i].nodeID != want[i] { 67 67 - t.Fatalf("rankCandidates()[%d] = %q, want %q; full order got %v want %v", i, got[i].nodeID, want[i], sessionIDs(got), want) 68 68 - } 69 69 - } 70 70 - } 71 71 - 72 72 - func sessionIDs(sessions []*millSession) []string { 73 73 - out := make([]string, len(sessions)) 74 74 - for i, sess := range sessions { 75 75 - out[i] = sess.nodeID 76 76 - } 77 77 - return out 78 78 - } 79 79 - 80 80 - func sameStringMultiset(a, b []string) bool { 81 81 - if len(a) != len(b) { 82 82 - return false 83 83 - } 84 84 - counts := make(map[string]int, len(a)) 85 85 - for _, s := range a { 86 86 - counts[s]++ 87 87 - } 88 88 - for _, s := range b { 89 89 - if counts[s] == 0 { 90 90 - return false 91 91 - } 92 92 - counts[s]-- 93 93 - } 94 94 - return true 95 95 - } 96 96 - 97 97 - type reserveReply struct { 98 98 - accepted bool 99 99 - rejectClass millv1.RejectClass 100 100 - reason string 101 101 - } 102 102 - 103 103 - func addReplyingCandidateSession(t *testing.T, m *Mill, nodeID string, labels []string, load float64, asked chan<- string, reply reserveReply) *millSession { 104 104 - t.Helper() 105 105 - var sess *millSession 106 106 - sess = addCandidateSession(t, m, nodeID, labels, load, scriptedEncoder(func(msg *millproto.Message) error { 107 107 - rs := msg.GetReserveSeat() 108 108 - if rs == nil { 109 109 - return nil 110 110 - } 111 111 - if asked != nil { 112 112 - asked <- nodeID 113 113 - } 114 114 - sess.deliver(rs.GetLeaseId(), &millproto.Message{ReserveResult: &millv1.ReserveResult{ 115 115 - LeaseId: rs.GetLeaseId(), 116 116 - Accepted: reply.accepted, 117 117 - RejectReason: reply.reason, 118 118 - RejectClass: reply.rejectClass, 119 119 - }}) 120 120 - return nil 121 121 - })) 122 122 - return sess 123 123 - } 124 124 - 125 125 - func drainAsked(ch <-chan string) []string { 126 126 - var out []string 127 127 - for { 128 128 - select { 129 129 - case nodeID := <-ch: 130 130 - out = append(out, nodeID) 131 131 - default: 132 132 - return out 133 133 - } 134 134 - } 135 135 - } 136 136 - 137 137 - func TestCommitRetriesAfterSessionCloseBeforeCommitted(t *testing.T) { 138 138 - l := slog.New(slog.NewTextHandler(io.Discard, nil)) 139 139 - m := New(l, Config{BidTimeout: 25 * time.Millisecond, ReconnectGrace: time.Second}) 140 140 - wf := testWorkflow("build") 141 141 - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 142 142 - lease := newLease("lease-1", "node-1", "inc-1", "dummy") 143 143 - lease.wid = wid 144 144 - wf.Data.(*millWorkflowState).Lease = lease 145 145 - 146 146 - m.mu.Lock() 147 147 - m.leases[lease.id] = lease 148 148 - m.mu.Unlock() 149 149 - 150 150 - var sess1 *millSession 151 151 - firstCommit := make(chan struct{}) 152 152 - sess1 = newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 153 153 - if msg.GetCommitLease() != nil { 154 154 - close(firstCommit) 155 155 - m.detachSession(sess1) 156 156 - } 157 157 - return nil 158 158 - }), l) 159 159 - m.attachSession(sess1) 160 160 - 161 161 - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) 162 162 - defer cancel() 163 163 - done := make(chan error, 1) 164 164 - go func() { done <- m.commitAndWait(ctx, wf, nil) }() 165 165 - 166 166 - select { 167 167 - case <-firstCommit: 168 168 - case <-ctx.Done(): 169 169 - t.Fatal("first commit was not sent") 170 170 - } 171 171 - 172 172 - var sess2 *millSession 173 173 - sess2 = newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 174 174 - if msg.GetCommitLease() == nil { 175 175 - return nil 176 176 - } 177 177 - leaseID := msg.GetCommitLease().GetLeaseId() 178 178 - sess2.deliver(leaseID, &millproto.Message{Committed: &millv1.Committed{LeaseId: leaseID}}) 179 179 - _ = m.onEventBatch(sess2, &millv1.EventBatch{ 180 180 - Epoch: sess2.epoch, 181 181 - Events: []*millv1.Event{ 182 182 - { 183 183 - Seqno: 1, 184 184 - LeaseId: leaseID, 185 185 - Payload: &millv1.Event_AttemptResult{ 186 186 - AttemptResult: &millv1.AttemptResult{ 187 187 - Status: millv1.TerminalStatus_SUCCESS, 188 188 - }, 189 189 - }, 190 190 - }, 191 191 - }, 192 192 - }) 193 193 - return nil 194 194 - }), l) 195 195 - m.attachSession(sess2) 196 196 - m.sessionReady(sess2) 197 197 - 198 198 - select { 199 199 - case err := <-done: 200 200 - if err != nil { 201 201 - t.Fatalf("commitAndWait() error = %v, want success after reconnect", err) 202 202 - } 203 203 - case <-ctx.Done(): 204 204 - t.Fatal("commitAndWait() did not finish after reconnect") 205 205 - } 206 206 - } 207 207 - 208 208 - func TestDestroyRunningLeaseDoesNotDropCancelledTerminal(t *testing.T) { 209 209 - l := slog.New(slog.NewTextHandler(io.Discard, nil)) 210 210 - m := New(l, Config{}) 211 211 - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 212 212 - lease := newLease("lease-1", "node-1", "inc-1", "dummy") 213 213 - lease.wid = wid 214 214 - lease.setState(leaseRunning) 215 215 - 216 216 - m.mu.Lock() 217 217 - m.leases[lease.id] = lease 218 218 - m.mu.Unlock() 219 219 - 220 220 - m.destroy(wid) 221 221 - if lease.getState() == leaseDone { 222 222 - t.Fatal("destroy sealed the lease before the terminal result") 223 223 - } 224 224 - 225 225 - lease.deliverTerminal(&millv1.AttemptResult{ 226 226 - Status: millv1.TerminalStatus_CANCELLED, 227 227 - }) 228 228 - res := <-lease.terminal 229 229 - if err := terminalError(res.Status); !errors.Is(err, engine.ErrWorkflowCanceled) { 230 230 - t.Fatalf("terminalError() = %v, want ErrCancelled", err) 231 231 - } 232 232 - } 233 233 - 234 234 - func TestPlaceBlocksWhenNoCapacity(t *testing.T) { 235 235 - l := slog.New(slog.NewTextHandler(io.Discard, nil)) 236 236 - m := New(l, Config{}) 237 237 - wf := testWorkflow("build") 238 238 - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 239 239 - 240 240 - // no executors at all: place must block until ctx expires (user sees pending) 241 241 - ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) 242 242 - defer cancel() 243 243 - 244 244 - _, err := m.place(ctx, "dummy", wid, wf) 245 245 - if err != context.DeadlineExceeded { 246 246 - t.Fatalf("place() error = %v, want DeadlineExceeded", err) 247 247 - } 248 248 - } 249 249 - 250 250 - func TestRankCandidatesFiltersRequiredLabelsWithANDSemantics(t *testing.T) { 251 251 - m := New(slog.New(slog.NewTextHandler(io.Discard, nil)), Config{}) 252 252 - addCandidateSession(t, m, "linux-high", []string{"linux"}, 0.0, nil) 253 253 - addCandidateSession(t, m, "linux-arm", []string{"linux", "arm64"}, 0.25, nil) 254 254 - addCandidateSession(t, m, "unlabeled", nil, 0.5, nil) 255 255 - addCandidateSession(t, m, "linux-arm-gpu", []string{"linux", "arm64", "gpu"}, 0.75, nil) 256 256 - addCandidateSession(t, m, "linux-arm-full", []string{"linux", "arm64"}, 1.0, nil) 257 257 - 258 258 - tests := []struct { 259 259 - name string 260 260 - requiredLabels []string 261 261 - want []string 262 262 - }{ 263 263 - { 264 264 - name: "no required labels keeps old capacity ranking", 265 265 - want: []string{"linux-high", "linux-arm", "unlabeled", "linux-arm-gpu"}, 266 266 - }, 267 267 - { 268 268 - name: "single required label includes every candidate carrying it", 269 269 - requiredLabels: []string{"linux"}, 270 270 - want: []string{"linux-high", "linux-arm", "linux-arm-gpu"}, 271 271 - }, 272 272 - { 273 273 - name: "all required labels must be present", 274 274 - requiredLabels: []string{"linux", "arm64"}, 275 275 - want: []string{"linux-arm", "linux-arm-gpu"}, 276 276 - }, 277 277 - { 278 278 - name: "one missing required label excludes the candidate", 279 279 - requiredLabels: []string{"linux", "arm64", "gpu"}, 280 280 - want: []string{"linux-arm-gpu"}, 281 281 - }, 282 282 - { 283 283 - name: "unknown required label leaves no candidate", 284 284 - requiredLabels: []string{"linux", "arm64", "metal"}, 285 285 - }, 286 286 - } 287 287 - 288 288 - for _, tt := range tests { 289 289 - t.Run(tt.name, func(t *testing.T) { 290 290 - assertRankedNodes(t, m.rankCandidates("dummy", tt.requiredLabels), tt.want) 291 291 - }) 292 292 - } 293 293 - } 294 294 - 295 295 - func TestPlaceWithMissingRequiredLabelsStaysPendingWithoutReserve(t *testing.T) { 296 296 - l := slog.New(slog.NewTextHandler(io.Discard, nil)) 297 297 - m := New(l, Config{BidTimeout: 10 * time.Millisecond}) 298 298 - reserveSent := make(chan struct{}, 1) 299 299 - addCandidateSession(t, m, "linux-only", []string{"linux"}, 0.75, scriptedEncoder(func(msg *millproto.Message) error { 300 300 - if msg.GetReserveSeat() != nil { 301 301 - select { 302 302 - case reserveSent <- struct{}{}: 303 303 - default: 304 304 - } 305 305 - } 306 306 - return nil 307 307 - })) 308 308 - wf := testWorkflowWithRunsOn("build", []string{"linux", "arm64"}) 309 309 - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 310 310 - 311 311 - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond) 312 312 - defer cancel() 313 313 - _, err := m.place(ctx, "dummy", wid, wf) 314 314 - if err != context.DeadlineExceeded { 315 315 - t.Fatalf("place() error = %v, want DeadlineExceeded while job remains pending", err) 316 316 - } 317 317 - select { 318 318 - case <-reserveSent: 319 319 - t.Fatal("place() sent ReserveSeat to executor missing a required label") 320 320 - default: 321 321 - } 322 322 - } 323 323 - 324 324 - func TestMaxPendingRejects(t *testing.T) { 325 325 - l := slog.New(slog.NewTextHandler(io.Discard, nil)) 326 326 - m := New(l, Config{MaxPending: 1}) 327 327 - 328 328 - m.mu.Lock() 329 329 - m.pending = 1 330 330 - m.mu.Unlock() 331 331 - 332 332 - wf2 := testWorkflow("b") 333 333 - _, err := m.place(context.Background(), "dummy", models.WorkflowId{Name: "b"}, wf2) 334 334 - if err == nil { 335 335 - t.Fatal("place() past maxPending should error") 336 336 - } 337 337 - } 338 338 - 339 339 - func TestCancelledRunningLeaseSurvivesReleaseForReconnectReplay(t *testing.T) { 340 340 - m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 341 341 - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 342 342 - lease := newLease("lease-1", "node-1", "inc-1", "dummy") 343 343 - lease.wid = wid 344 344 - lease.setState(leaseRunning) 345 345 - if err := m.persistLease(lease, leaseRowRunning); err != nil { 346 346 - t.Fatalf("persistLease: %v", err) 347 347 - } 348 348 - m.mu.Lock() 349 349 - m.leases[lease.id] = lease 350 350 - m.mu.Unlock() 351 351 - 352 352 - m.destroy(wid) 353 353 - slot := &millSlot{fleet: m, lease: lease} 354 354 - slot.Release() 355 355 - 356 356 - m.mu.Lock() 357 357 - _, retained := m.leases[lease.id] 358 358 - m.mu.Unlock() 359 359 - if !retained { 360 360 - t.Fatal("slot release removed a cancellation-requested running lease before its terminal result") 361 361 - } 362 362 - if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 1 { 363 363 - t.Fatalf("durable leases after slot release = %+v, err = %v; want retained lease", rows, err) 364 364 - } 365 365 - 366 366 - var sentMu sync.Mutex 367 367 - var sent []*millproto.Message 368 368 - sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 369 369 - sentMu.Lock() 370 370 - sent = append(sent, msg) 371 371 - sentMu.Unlock() 372 372 - return nil 373 373 - }), discardLogger()) 374 374 - if _, ok := m.attachSession(sess); !ok { 375 375 - t.Fatal("attachSession rejected reconnect") 376 376 - } 377 377 - m.sessionReady(sess) 378 378 - sentMu.Lock() 379 379 - var replayed bool 380 380 - for _, msg := range sent { 381 381 - if cancel := msg.GetCancelAttempt(); cancel != nil && cancel.GetLeaseId() == lease.id { 382 382 - replayed = true 383 383 - } 384 384 - } 385 385 - sentMu.Unlock() 386 386 - if !replayed { 387 387 - t.Fatal("reconnect did not replay CancelAttempt for retained lease") 388 388 - } 389 389 - 390 390 - if err := m.onEventBatch(sess, &millv1.EventBatch{ 391 391 - Epoch: sess.epoch, 392 392 - Events: []*millv1.Event{ 393 393 - { 394 394 - Seqno: 1, 395 395 - LeaseId: lease.id, 396 396 - Payload: &millv1.Event_AttemptResult{ 397 397 - AttemptResult: &millv1.AttemptResult{ 398 398 - Status: millv1.TerminalStatus_CANCELLED, 399 399 - }, 400 400 - }, 401 401 - }, 402 402 - }, 403 403 - }); err != nil { 404 404 - t.Fatalf("onEventBatch: %v", err) 405 405 - } 406 406 - m.mu.Lock() 407 407 - _, retained = m.leases[lease.id] 408 408 - m.mu.Unlock() 409 409 - if retained { 410 410 - t.Fatal("terminal result did not clean retained cancelled lease") 411 411 - } 412 412 - if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 0 { 413 413 - t.Fatalf("durable leases after terminal = %+v, err = %v; want none", rows, err) 414 414 - } 415 415 - slot.Release() 416 416 - } 417 417 - 418 418 - func TestSessionRequestCancelledBeforeRegistrationOrSend(t *testing.T) { 419 419 - sent := 0 420 420 - sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(*millproto.Message) error { 421 421 - sent++ 422 422 - return nil 423 423 - }), discardLogger()) 424 424 - ctx, cancel := context.WithCancel(context.Background()) 425 425 - cancel() 426 426 - _, err := sess.request(ctx, "lease-1", &millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: "lease-1"}}) 427 427 - if !errors.Is(err, context.Canceled) { 428 428 - t.Fatalf("request error = %v, want context.Canceled", err) 429 429 - } 430 430 - if sent != 0 { 431 431 - t.Fatalf("request sent %d messages for an already-cancelled context, want 0", sent) 432 432 - } 433 433 - sess.mu.Lock() 434 434 - pending := len(sess.pending) 435 435 - sess.mu.Unlock() 436 436 - if pending != 0 { 437 437 - t.Fatalf("request left %d pending waiters, want 0", pending) 438 438 - } 439 439 - } 440 440 - 441 441 - func TestAttachSessionReplacesSilentIncumbentButRejectsActiveDuplicate(t *testing.T) { 442 442 - m := New(discardLogger(), Config{ReconnectGrace: time.Minute}) 443 443 - old := newSession("node-1", "inc-old", nil, nopEncoder(), discardLogger()) 444 444 - transportClosed := make(chan struct{}) 445 445 - old.closeTransport = func() error { 446 446 - close(transportClosed) 447 447 - return nil 448 448 - } 449 449 - if _, ok := m.attachSession(old); !ok { 450 450 - t.Fatal("first attach rejected") 451 451 - } 452 452 - m.mu.Lock() 453 453 - old.lastSeen = time.Now().Add(-2 * m.cfg.ReconnectGrace) 454 454 - m.mu.Unlock() 455 455 - 456 456 - replacement := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) 457 457 - if _, ok := m.attachSession(replacement); !ok { 458 458 - t.Fatal("silent incumbent blocked authenticated replacement") 459 459 - } 460 460 - select { 461 461 - case <-transportClosed: 462 462 - default: 463 463 - t.Fatal("replacing a silent incumbent did not close its transport") 464 464 - } 465 465 - m.mu.Lock() 466 466 - replacement.lastSeen = time.Now().Add(-2 * m.cfg.ReconnectGrace) 467 467 - m.mu.Unlock() 468 468 - if err := replacement.dispatch(m, &millproto.Message{NodeSnapshot: &millv1.NodeSnapshot{Seqno: 1}}); err != nil { 469 469 - t.Fatalf("periodic snapshot dispatch: %v", err) 470 470 - } 471 471 - if _, ok := m.attachSession(newSession("node-1", "inc-dup", nil, nopEncoder(), discardLogger())); ok { 472 472 - t.Fatal("active replacement did not reject a duplicate session") 473 473 - } 474 474 - } 475 475 - 476 476 - func TestPlaceReleasesRemoteReservationWhenInitialPersistenceFails(t *testing.T) { 477 477 - m, bdb := restoreTestMill(t, Config{BidTimeout: time.Second}) 478 478 - if err := bdb.Close(); err != nil { 479 479 - t.Fatalf("close db: %v", err) 480 480 - } 481 481 - released := make(chan string, 1) 482 482 - var sess *millSession 483 483 - sess = addCandidateSession(t, m, "node-1", nil, 0, scriptedEncoder(func(msg *millproto.Message) error { 484 484 - switch { 485 485 - case msg.GetReserveSeat() != nil: 486 486 - leaseID := msg.GetReserveSeat().GetLeaseId() 487 487 - sess.deliver(leaseID, &millproto.Message{ReserveResult: &millv1.ReserveResult{ 488 488 - LeaseId: leaseID, 489 489 - Accepted: true, 490 490 - }}) 491 491 - case msg.GetReleaseLease() != nil: 492 492 - released <- msg.GetReleaseLease().GetLeaseId() 493 493 - } 494 494 - return nil 495 495 - })) 496 496 - 497 497 - ctx, cancel := context.WithTimeout(context.Background(), time.Second) 498 498 - defer cancel() 499 499 - slot, err := m.place( 500 500 - ctx, 501 501 - "dummy", 502 502 - models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, 503 503 - testWorkflow("build"), 504 504 - ) 505 505 - if err == nil { 506 506 - t.Fatal("place succeeded after reserved lease persistence failed") 507 507 - } 508 508 - if slot != nil { 509 509 - t.Fatalf("place returned slot %T after persistence failure", slot) 510 510 - } 511 511 - select { 512 512 - case leaseID := <-released: 513 513 - if leaseID == "" { 514 514 - t.Fatal("ReleaseLease had empty lease id") 515 515 - } 516 516 - case <-time.After(time.Second): 517 517 - t.Fatal("persistence failure did not compensate with ReleaseLease") 518 518 - } 519 519 - m.mu.Lock() 520 520 - leases := len(m.leases) 521 521 - m.mu.Unlock() 522 522 - if leases != 0 { 523 523 - t.Fatalf("mill published %d leases after initial persistence failure, want 0", leases) 524 524 - } 525 525 - } 526 526 - 527 527 - func TestGapsAndDuplicates(t *testing.T) { 528 528 - m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 529 529 - sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 530 530 - m.attachSession(sess) 531 531 - 532 532 - owned := newLease("lease-1", "node-1", "inc-1", "dummy") 533 533 - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 534 534 - m.mu.Lock() 535 535 - m.leases[owned.id] = owned 536 536 - m.mu.Unlock() 537 537 - 538 538 - err := m.onEventBatch(sess, &millv1.EventBatch{ 539 539 - Epoch: sess.epoch, 540 540 - Events: []*millv1.Event{ 541 541 - { 542 542 - Seqno: 0, 543 543 - LeaseId: owned.id, 544 544 - Payload: &millv1.Event_StatusEvent{ 545 545 - StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING}, 546 546 - }, 547 547 - }, 548 548 - }, 549 549 - }) 550 550 - if err != nil { 551 551 - t.Fatalf("expected duplicate to be skipped without error, got: %v", err) 552 552 - } 553 553 - 554 554 - err = m.onEventBatch(sess, &millv1.EventBatch{ 555 555 - Epoch: sess.epoch, 556 556 - Events: []*millv1.Event{ 557 557 - { 558 558 - Seqno: 2, 559 559 - LeaseId: owned.id, 560 560 - Payload: &millv1.Event_StatusEvent{ 561 561 - StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING}, 562 562 - }, 563 563 - }, 564 564 - }, 565 565 - }) 566 566 - if err == nil { 567 567 - t.Fatal("expected error due to seqno gap, got nil") 568 568 - } 569 569 - } 570 570 - 571 571 - func TestAtomicBatchRollback(t *testing.T) { 572 572 - m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 573 573 - sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 574 574 - m.attachSession(sess) 575 575 - 576 576 - owned := newLease("lease-1", "node-1", "inc-1", "dummy") 577 577 - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 578 578 - m.mu.Lock() 579 579 - m.leases[owned.id] = owned 580 580 - m.mu.Unlock() 581 581 - 582 582 - if _, err := bdb.Exec(` 583 583 - create trigger reject_status_event 584 584 - before insert on events 585 585 - begin 586 586 - select raise(abort, 'forced status event failure'); 587 587 - end 588 588 - `); err != nil { 589 589 - t.Fatalf("failed to create fail trigger: %v", err) 590 590 - } 591 591 - defer bdb.Exec("drop trigger reject_status_event") 592 592 - 593 593 - err := m.onEventBatch(sess, &millv1.EventBatch{ 594 594 - Epoch: sess.epoch, 595 595 - Events: []*millv1.Event{ 596 596 - { 597 597 - Seqno: 1, 598 598 - LeaseId: owned.id, 599 599 - Payload: &millv1.Event_StatusEvent{ 600 600 - StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING}, 601 601 - }, 602 602 - }, 603 603 - }, 604 604 - }) 605 605 - if err == nil { 606 606 - t.Fatal("expected status event insertion to fail due to trigger") 607 607 - } 608 608 - 609 609 - m.mu.Lock() 610 610 - seqno := m.nodeSeqno["node-1/inc-1"] 611 611 - m.mu.Unlock() 612 612 - if seqno != 0 { 613 613 - t.Fatalf("expected seqno 0 due to rollback, got %d", seqno) 614 614 - } 615 615 - } 616 616 - 617 617 - func TestTerminalBeforeACK(t *testing.T) { 618 618 - m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 619 619 - 620 620 - ackSent := make(chan struct{}) 621 621 - sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 622 622 - if msg.GetAck() != nil { 623 623 - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 624 624 - st, err := bdb.GetStatus(wid) 625 625 - if err != nil || st.Status != "success" { 626 626 - t.Errorf("expected terminal status success at ACK time, got status: %v, err: %v", st, err) 627 627 - } 628 628 - close(ackSent) 629 629 - } 630 630 - return nil 631 631 - }), discardLogger()) 632 632 - m.attachSession(sess) 633 633 - 634 634 - owned := newLease("lease-1", "node-1", "inc-1", "dummy") 635 635 - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 636 636 - m.mu.Lock() 637 637 - m.leases[owned.id] = owned 638 638 - m.mu.Unlock() 639 639 - 640 640 - err := m.onEventBatch(sess, &millv1.EventBatch{ 641 641 - Epoch: sess.epoch, 642 642 - Events: []*millv1.Event{ 643 643 - { 644 644 - Seqno: 1, 645 645 - LeaseId: owned.id, 646 646 - Payload: &millv1.Event_AttemptResult{ 647 647 - AttemptResult: &millv1.AttemptResult{Status: millv1.TerminalStatus_SUCCESS}, 648 648 - }, 649 649 - }, 650 650 - }, 651 651 - }) 652 652 - if err != nil { 653 653 - t.Fatalf("onEventBatch: %v", err) 654 654 - } 655 655 - 656 656 - select { 657 657 - case <-ackSent: 658 658 - case <-time.After(2 * time.Second): 659 659 - t.Fatal("ACK was not sent") 660 660 - } 661 661 - } 662 662 - 663 663 - func TestExecutorRestartEmptySnapshot(t *testing.T) { 664 664 - m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 665 665 - 666 666 - lease := newLease("lease-1", "node-1", "inc-old", "dummy") 667 667 - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 668 668 - m.mu.Lock() 669 669 - m.leases[lease.id] = lease 670 670 - m.mu.Unlock() 671 671 - if err := m.persistLease(lease, leaseRowRunning); err != nil { 672 672 - t.Fatalf("persistLease: %v", err) 673 673 - } 674 674 - 675 675 - sess := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) 676 676 - m.attachSession(sess) 677 677 - 678 678 - err := m.onSnapshot(sess, &millv1.NodeSnapshot{ 679 679 - Seqno: 1, 680 680 - ActiveLeaseIds: nil, 681 681 - }) 682 682 - if err != nil { 683 683 - t.Fatalf("onSnapshot: %v", err) 684 684 - } 685 685 - 686 686 - m.mu.Lock() 687 687 - _, stillActive := m.leases["lease-1"] 688 688 - m.mu.Unlock() 689 689 - if stillActive { 690 690 - t.Fatal("expected old epoch lease to be reconciled and failed") 691 691 - } 692 692 - } 693 693 - 694 694 - func TestReplacementLostBeforeSnapshotFailsOldEpochLease(t *testing.T) { 695 695 - m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 696 696 - lease := newLease("lease-1", "node-1", "inc-old", "dummy") 697 697 - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 698 698 - lease.setState(leaseRunning) 699 699 - if err := m.persistLease(lease, leaseRowRunning); err != nil { 700 700 - t.Fatalf("persistLease: %v", err) 701 701 - } 702 702 - m.mu.Lock() 703 703 - m.leases[lease.id] = lease 704 704 - m.mu.Unlock() 705 705 - 706 706 - replacement := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) 707 707 - if _, ok := m.attachSession(replacement); !ok { 708 708 - t.Fatal("attachSession rejected replacement") 709 709 - } 710 710 - replacement.disconnected = true 711 711 - m.failLeasesAfterGrace(replacement) 712 712 - 713 713 - m.mu.Lock() 714 714 - _, stillActive := m.leases[lease.id] 715 715 - m.mu.Unlock() 716 716 - if stillActive { 717 717 - t.Fatal("replacement loss stranded old-epoch lease") 718 718 - } 719 719 - } 720 720 - 721 721 - func TestSeqRegression(t *testing.T) { 722 722 - m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 723 723 - sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 724 724 - m.attachSession(sess) 725 725 - 726 726 - err := m.onSnapshot(sess, &millv1.NodeSnapshot{ 727 727 - Seqno: 5, 728 728 - }) 729 729 - if err != nil { 730 730 - t.Fatalf("first snapshot: %v", err) 731 731 - } 732 732 - 733 733 - err = m.onSnapshot(sess, &millv1.NodeSnapshot{ 734 734 - Seqno: 4, 735 735 - }) 736 736 - if err == nil { 737 737 - t.Fatal("expected seqno regression to be rejected") 738 738 - } 739 739 - } 740 740 - 741 741 - func TestClaimedSweep(t *testing.T) { 742 742 - m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 743 743 - 744 744 - lease := newLease("lease-1", "node-1", "inc-1", "dummy") 745 745 - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 746 746 - lease.orphaned = true 747 747 - lease.claimed = false 748 748 - m.mu.Lock() 749 749 - m.leases[lease.id] = lease 750 750 - m.mu.Unlock() 751 751 - 752 752 - sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 753 753 - m.attachSession(sess) 754 754 - err := m.onSnapshot(sess, &millv1.NodeSnapshot{ 755 755 - Seqno: 1, 756 756 - ActiveLeaseIds: []string{"lease-1"}, 757 757 - }) 758 758 - if err != nil { 759 759 - t.Fatalf("onSnapshot: %v", err) 760 760 - } 761 761 - 762 762 - m.detachSession(sess) 763 763 - 764 764 - m.sweepUnclaimedOrphans() 765 765 - 766 766 - m.mu.Lock() 767 767 - _, stillRunning := m.leases["lease-1"] 768 768 - m.mu.Unlock() 769 769 - if !stillRunning { 770 770 - t.Fatal("claimed lease was incorrectly swept by startup sweep") 771 771 - } 772 772 - } 773 773 - 774 774 - func TestCancelDeadline(t *testing.T) { 775 775 - m, _ := restoreTestMill(t, Config{CancelTimeout: 50 * time.Millisecond}) 776 776 - 777 777 - sessClosed := make(chan struct{}) 778 778 - sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 779 779 - return nil 780 780 - }), discardLogger()) 781 781 - sess.closeTransport = func() error { 782 782 - close(sessClosed) 783 783 - return nil 784 784 - } 785 785 - m.attachSession(sess) 786 786 - 787 787 - lease := newLease("lease-1", "node-1", "inc-1", "dummy") 788 788 - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 789 789 - lease.setState(leaseRunning) 790 790 - m.mu.Lock() 791 791 - m.leases[lease.id] = lease 792 792 - m.mu.Unlock() 793 793 - 794 794 - m.destroy(lease.wid) 795 795 - 796 796 - select { 797 797 - case <-sessClosed: 798 798 - case <-time.After(1 * time.Second): 799 799 - t.Fatal("session was not closed after cancel deadline expiration") 800 800 - } 801 801 - } 802 802 - 803 803 - func TestCleanupRetry(t *testing.T) { 804 804 - m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 805 805 - 806 806 - lease := newLease("lease-1", "node-1", "inc-1", "dummy") 807 807 - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 808 808 - if err := m.persistLease(lease, leaseRowRunning); err != nil { 809 809 - t.Fatalf("persist lease: %v", err) 810 810 - } 811 811 - m.mu.Lock() 812 812 - m.leases[lease.id] = lease 813 813 - m.mu.Unlock() 814 814 - 815 815 - if _, err := bdb.Exec(` 816 816 - create trigger reject_cleanup_delete 817 817 - before delete on mill_leases 818 818 - begin 819 819 - select raise(abort, 'forced delete failure'); 820 820 - end 821 821 - `); err != nil { 822 822 - t.Fatalf("failed to create fail trigger: %v", err) 823 823 - } 824 824 - 825 825 - err := m.cleanupLease(lease) 826 826 - if err == nil { 827 827 - t.Fatal("expected cleanupLease to fail") 828 828 - } 829 829 - 830 830 - m.mu.Lock() 831 831 - _, stillRunning := m.leases["lease-1"] 832 832 - m.mu.Unlock() 833 833 - if !stillRunning { 834 834 - t.Fatal("lease was removed from memory despite cleanup failure") 835 835 - } 836 836 - 837 837 - if _, err := bdb.Exec("drop trigger reject_cleanup_delete"); err != nil { 838 838 - t.Fatalf("drop trigger: %v", err) 839 839 - } 840 840 - 841 841 - err = m.cleanupLease(lease) 842 842 - if err != nil { 843 843 - t.Fatalf("expected retry cleanup to succeed, got: %v", err) 844 844 - } 845 845 - 846 846 - m.mu.Lock() 847 847 - _, stillRunning = m.leases["lease-1"] 848 848 - m.mu.Unlock() 849 849 - if stillRunning { 850 850 - t.Fatal("lease still in memory after successful cleanup retry") 851 851 - } 852 852 - } 853 853 - 854 854 - func TestBoundedBidding(t *testing.T) { 855 855 - m := New(discardLogger(), Config{TopK: 2, BidTimeout: 10 * time.Millisecond}) 856 856 - 857 857 - addCandidateSession(t, m, "node-1", nil, 0, nil) 858 858 - addCandidateSession(t, m, "node-2", nil, 0, nil) 859 859 - addCandidateSession(t, m, "node-3", nil, 0, nil) 860 860 - addCandidateSession(t, m, "node-4", nil, 0, nil) 861 861 - addCandidateSession(t, m, "node-5", nil, 0, nil) 862 862 - 863 863 - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) 864 864 - defer cancel() 865 865 - 866 866 - lease, err := m.bid(ctx, "dummy", models.WorkflowId{}, testWorkflow("build")) 867 867 - if err != nil { 868 868 - t.Fatalf("bid: %v", err) 869 869 - } 870 870 - if lease != nil { 871 871 - t.Fatalf("did not expect a lease, got %+v", lease) 872 872 - } 873 873 - }

-1679

spindle/mill/proto/gen/mill.pb.go

Reviewed

··· 1 1 - // Code generated by protoc-gen-go. DO NOT EDIT. 2 2 - // versions: 3 3 - // protoc-gen-go v1.36.11 4 4 - // protoc (unknown) 5 5 - // source: spindle/mill/v1/mill.proto 6 6 - 7 7 - package millv1 8 8 - 9 9 - import ( 10 10 - _ "buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go/buf/validate" 11 11 - protoreflect "google.golang.org/protobuf/reflect/protoreflect" 12 12 - protoimpl "google.golang.org/protobuf/runtime/protoimpl" 13 13 - reflect "reflect" 14 14 - sync "sync" 15 15 - unsafe "unsafe" 16 16 - ) 17 17 - 18 18 - const ( 19 19 - // Verify that this generated code is sufficiently up-to-date. 20 20 - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 21 21 - // Verify that runtime/protoimpl is sufficiently up-to-date. 22 22 - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 23 23 - ) 24 24 - 25 25 - type RejectClass int32 26 26 - 27 27 - const ( 28 28 - RejectClass_REJECT_CLASS_UNSPECIFIED RejectClass = 0 29 29 - RejectClass_REJECT_CLASS_TRANSIENT RejectClass = 1 30 30 - RejectClass_REJECT_CLASS_INCOMPATIBLE RejectClass = 2 31 31 - ) 32 32 - 33 33 - // Enum value maps for RejectClass. 34 34 - var ( 35 35 - RejectClass_name = map[int32]string{ 36 36 - 0: "REJECT_CLASS_UNSPECIFIED", 37 37 - 1: "REJECT_CLASS_TRANSIENT", 38 38 - 2: "REJECT_CLASS_INCOMPATIBLE", 39 39 - } 40 40 - RejectClass_value = map[string]int32{ 41 41 - "REJECT_CLASS_UNSPECIFIED": 0, 42 42 - "REJECT_CLASS_TRANSIENT": 1, 43 43 - "REJECT_CLASS_INCOMPATIBLE": 2, 44 44 - } 45 45 - ) 46 46 - 47 47 - func (x RejectClass) Enum() *RejectClass { 48 48 - p := new(RejectClass) 49 49 - *p = x 50 50 - return p 51 51 - } 52 52 - 53 53 - func (x RejectClass) String() string { 54 54 - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 55 55 - } 56 56 - 57 57 - func (RejectClass) Descriptor() protoreflect.EnumDescriptor { 58 58 - return file_spindle_mill_v1_mill_proto_enumTypes[0].Descriptor() 59 59 - } 60 60 - 61 61 - func (RejectClass) Type() protoreflect.EnumType { 62 62 - return &file_spindle_mill_v1_mill_proto_enumTypes[0] 63 63 - } 64 64 - 65 65 - func (x RejectClass) Number() protoreflect.EnumNumber { 66 66 - return protoreflect.EnumNumber(x) 67 67 - } 68 68 - 69 69 - // Deprecated: Use RejectClass.Descriptor instead. 70 70 - func (RejectClass) EnumDescriptor() ([]byte, []int) { 71 71 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{0} 72 72 - } 73 73 - 74 74 - type NonterminalStatus int32 75 75 - 76 76 - const ( 77 77 - NonterminalStatus_NONTERMINAL_STATUS_UNSPECIFIED NonterminalStatus = 0 78 78 - NonterminalStatus_RUNNING NonterminalStatus = 1 79 79 - ) 80 80 - 81 81 - // Enum value maps for NonterminalStatus. 82 82 - var ( 83 83 - NonterminalStatus_name = map[int32]string{ 84 84 - 0: "NONTERMINAL_STATUS_UNSPECIFIED", 85 85 - 1: "RUNNING", 86 86 - } 87 87 - NonterminalStatus_value = map[string]int32{ 88 88 - "NONTERMINAL_STATUS_UNSPECIFIED": 0, 89 89 - "RUNNING": 1, 90 90 - } 91 91 - ) 92 92 - 93 93 - func (x NonterminalStatus) Enum() *NonterminalStatus { 94 94 - p := new(NonterminalStatus) 95 95 - *p = x 96 96 - return p 97 97 - } 98 98 - 99 99 - func (x NonterminalStatus) String() string { 100 100 - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 101 101 - } 102 102 - 103 103 - func (NonterminalStatus) Descriptor() protoreflect.EnumDescriptor { 104 104 - return file_spindle_mill_v1_mill_proto_enumTypes[1].Descriptor() 105 105 - } 106 106 - 107 107 - func (NonterminalStatus) Type() protoreflect.EnumType { 108 108 - return &file_spindle_mill_v1_mill_proto_enumTypes[1] 109 109 - } 110 110 - 111 111 - func (x NonterminalStatus) Number() protoreflect.EnumNumber { 112 112 - return protoreflect.EnumNumber(x) 113 113 - } 114 114 - 115 115 - // Deprecated: Use NonterminalStatus.Descriptor instead. 116 116 - func (NonterminalStatus) EnumDescriptor() ([]byte, []int) { 117 117 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{1} 118 118 - } 119 119 - 120 120 - type TerminalStatus int32 121 121 - 122 122 - const ( 123 123 - TerminalStatus_TERMINAL_STATUS_UNSPECIFIED TerminalStatus = 0 124 124 - TerminalStatus_SUCCESS TerminalStatus = 1 125 125 - TerminalStatus_FAILED TerminalStatus = 2 126 126 - TerminalStatus_TIMEOUT TerminalStatus = 3 127 127 - TerminalStatus_CANCELLED TerminalStatus = 4 128 128 - ) 129 129 - 130 130 - // Enum value maps for TerminalStatus. 131 131 - var ( 132 132 - TerminalStatus_name = map[int32]string{ 133 133 - 0: "TERMINAL_STATUS_UNSPECIFIED", 134 134 - 1: "SUCCESS", 135 135 - 2: "FAILED", 136 136 - 3: "TIMEOUT", 137 137 - 4: "CANCELLED", 138 138 - } 139 139 - TerminalStatus_value = map[string]int32{ 140 140 - "TERMINAL_STATUS_UNSPECIFIED": 0, 141 141 - "SUCCESS": 1, 142 142 - "FAILED": 2, 143 143 - "TIMEOUT": 3, 144 144 - "CANCELLED": 4, 145 145 - } 146 146 - ) 147 147 - 148 148 - func (x TerminalStatus) Enum() *TerminalStatus { 149 149 - p := new(TerminalStatus) 150 150 - *p = x 151 151 - return p 152 152 - } 153 153 - 154 154 - func (x TerminalStatus) String() string { 155 155 - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 156 156 - } 157 157 - 158 158 - func (TerminalStatus) Descriptor() protoreflect.EnumDescriptor { 159 159 - return file_spindle_mill_v1_mill_proto_enumTypes[2].Descriptor() 160 160 - } 161 161 - 162 162 - func (TerminalStatus) Type() protoreflect.EnumType { 163 163 - return &file_spindle_mill_v1_mill_proto_enumTypes[2] 164 164 - } 165 165 - 166 166 - func (x TerminalStatus) Number() protoreflect.EnumNumber { 167 167 - return protoreflect.EnumNumber(x) 168 168 - } 169 169 - 170 170 - // Deprecated: Use TerminalStatus.Descriptor instead. 171 171 - func (TerminalStatus) EnumDescriptor() ([]byte, []int) { 172 172 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{2} 173 173 - } 174 174 - 175 175 - // executor identity, sent on connect 176 176 - type Hello struct { 177 177 - state protoimpl.MessageState `protogen:"open.v1"` 178 178 - ProtocolVersion uint32 `protobuf:"varint,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"` 179 179 - // GOARCH of the node, informational only 180 180 - Arch string `protobuf:"bytes,2,opt,name=arch,proto3" json:"arch,omitempty"` 181 181 - // operator-defined labels, matched against runs_on 182 182 - Labels []string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty"` 183 183 - Epoch string `protobuf:"bytes,4,opt,name=epoch,proto3" json:"epoch,omitempty"` 184 184 - unknownFields protoimpl.UnknownFields 185 185 - sizeCache protoimpl.SizeCache 186 186 - } 187 187 - 188 188 - func (x *Hello) Reset() { 189 189 - *x = Hello{} 190 190 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[0] 191 191 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 192 192 - ms.StoreMessageInfo(mi) 193 193 - } 194 194 - 195 195 - func (x *Hello) String() string { 196 196 - return protoimpl.X.MessageStringOf(x) 197 197 - } 198 198 - 199 199 - func (*Hello) ProtoMessage() {} 200 200 - 201 201 - func (x *Hello) ProtoReflect() protoreflect.Message { 202 202 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[0] 203 203 - if x != nil { 204 204 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 205 205 - if ms.LoadMessageInfo() == nil { 206 206 - ms.StoreMessageInfo(mi) 207 207 - } 208 208 - return ms 209 209 - } 210 210 - return mi.MessageOf(x) 211 211 - } 212 212 - 213 213 - // Deprecated: Use Hello.ProtoReflect.Descriptor instead. 214 214 - func (*Hello) Descriptor() ([]byte, []int) { 215 215 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{0} 216 216 - } 217 217 - 218 218 - func (x *Hello) GetProtocolVersion() uint32 { 219 219 - if x != nil { 220 220 - return x.ProtocolVersion 221 221 - } 222 222 - return 0 223 223 - } 224 224 - 225 225 - func (x *Hello) GetArch() string { 226 226 - if x != nil { 227 227 - return x.Arch 228 228 - } 229 229 - return "" 230 230 - } 231 231 - 232 232 - func (x *Hello) GetLabels() []string { 233 233 - if x != nil { 234 234 - return x.Labels 235 235 - } 236 236 - return nil 237 237 - } 238 238 - 239 239 - func (x *Hello) GetEpoch() string { 240 240 - if x != nil { 241 241 - return x.Epoch 242 242 - } 243 243 - return "" 244 244 - } 245 245 - 246 246 - // reconnect state for an existing epoch 247 247 - type Resume struct { 248 248 - state protoimpl.MessageState `protogen:"open.v1"` 249 249 - Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"` 250 250 - AckSeqno uint64 `protobuf:"varint,2,opt,name=ack_seqno,json=ackSeqno,proto3" json:"ack_seqno,omitempty"` 251 251 - unknownFields protoimpl.UnknownFields 252 252 - sizeCache protoimpl.SizeCache 253 253 - } 254 254 - 255 255 - func (x *Resume) Reset() { 256 256 - *x = Resume{} 257 257 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[1] 258 258 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 259 259 - ms.StoreMessageInfo(mi) 260 260 - } 261 261 - 262 262 - func (x *Resume) String() string { 263 263 - return protoimpl.X.MessageStringOf(x) 264 264 - } 265 265 - 266 266 - func (*Resume) ProtoMessage() {} 267 267 - 268 268 - func (x *Resume) ProtoReflect() protoreflect.Message { 269 269 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[1] 270 270 - if x != nil { 271 271 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 272 272 - if ms.LoadMessageInfo() == nil { 273 273 - ms.StoreMessageInfo(mi) 274 274 - } 275 275 - return ms 276 276 - } 277 277 - return mi.MessageOf(x) 278 278 - } 279 279 - 280 280 - // Deprecated: Use Resume.ProtoReflect.Descriptor instead. 281 281 - func (*Resume) Descriptor() ([]byte, []int) { 282 282 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{1} 283 283 - } 284 284 - 285 285 - func (x *Resume) GetEpoch() string { 286 286 - if x != nil { 287 287 - return x.Epoch 288 288 - } 289 289 - return "" 290 290 - } 291 291 - 292 292 - func (x *Resume) GetAckSeqno() uint64 { 293 293 - if x != nil { 294 294 - return x.AckSeqno 295 295 - } 296 296 - return 0 297 297 - } 298 298 - 299 299 - // per-engine state of a node 300 300 - type EngineAvailability struct { 301 301 - state protoimpl.MessageState `protogen:"open.v1"` 302 302 - Available bool `protobuf:"varint,1,opt,name=available,proto3" json:"available,omitempty"` 303 303 - // opaque engine-defined load metrics, higher means more loaded 304 304 - Load map[string]float64 `protobuf:"bytes,2,rep,name=load,proto3" json:"load,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"fixed64,2,opt,name=value"` 305 305 - unknownFields protoimpl.UnknownFields 306 306 - sizeCache protoimpl.SizeCache 307 307 - } 308 308 - 309 309 - func (x *EngineAvailability) Reset() { 310 310 - *x = EngineAvailability{} 311 311 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[2] 312 312 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 313 313 - ms.StoreMessageInfo(mi) 314 314 - } 315 315 - 316 316 - func (x *EngineAvailability) String() string { 317 317 - return protoimpl.X.MessageStringOf(x) 318 318 - } 319 319 - 320 320 - func (*EngineAvailability) ProtoMessage() {} 321 321 - 322 322 - func (x *EngineAvailability) ProtoReflect() protoreflect.Message { 323 323 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[2] 324 324 - if x != nil { 325 325 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 326 326 - if ms.LoadMessageInfo() == nil { 327 327 - ms.StoreMessageInfo(mi) 328 328 - } 329 329 - return ms 330 330 - } 331 331 - return mi.MessageOf(x) 332 332 - } 333 333 - 334 334 - // Deprecated: Use EngineAvailability.ProtoReflect.Descriptor instead. 335 335 - func (*EngineAvailability) Descriptor() ([]byte, []int) { 336 336 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{2} 337 337 - } 338 338 - 339 339 - func (x *EngineAvailability) GetAvailable() bool { 340 340 - if x != nil { 341 341 - return x.Available 342 342 - } 343 343 - return false 344 344 - } 345 345 - 346 346 - func (x *EngineAvailability) GetLoad() map[string]float64 { 347 347 - if x != nil { 348 348 - return x.Load 349 349 - } 350 350 - return nil 351 351 - } 352 352 - 353 353 - // full node state 354 354 - type NodeSnapshot struct { 355 355 - state protoimpl.MessageState `protogen:"open.v1"` 356 356 - Seqno uint64 `protobuf:"varint,1,opt,name=seqno,proto3" json:"seqno,omitempty"` 357 357 - Engines map[string]*EngineAvailability `protobuf:"bytes,2,rep,name=engines,proto3" json:"engines,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` 358 358 - // every lease the executor currently holds, reserved or running 359 359 - ActiveLeaseIds []string `protobuf:"bytes,3,rep,name=active_lease_ids,json=activeLeaseIds,proto3" json:"active_lease_ids,omitempty"` 360 360 - unknownFields protoimpl.UnknownFields 361 361 - sizeCache protoimpl.SizeCache 362 362 - } 363 363 - 364 364 - func (x *NodeSnapshot) Reset() { 365 365 - *x = NodeSnapshot{} 366 366 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[3] 367 367 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 368 368 - ms.StoreMessageInfo(mi) 369 369 - } 370 370 - 371 371 - func (x *NodeSnapshot) String() string { 372 372 - return protoimpl.X.MessageStringOf(x) 373 373 - } 374 374 - 375 375 - func (*NodeSnapshot) ProtoMessage() {} 376 376 - 377 377 - func (x *NodeSnapshot) ProtoReflect() protoreflect.Message { 378 378 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[3] 379 379 - if x != nil { 380 380 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 381 381 - if ms.LoadMessageInfo() == nil { 382 382 - ms.StoreMessageInfo(mi) 383 383 - } 384 384 - return ms 385 385 - } 386 386 - return mi.MessageOf(x) 387 387 - } 388 388 - 389 389 - // Deprecated: Use NodeSnapshot.ProtoReflect.Descriptor instead. 390 390 - func (*NodeSnapshot) Descriptor() ([]byte, []int) { 391 391 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{3} 392 392 - } 393 393 - 394 394 - func (x *NodeSnapshot) GetSeqno() uint64 { 395 395 - if x != nil { 396 396 - return x.Seqno 397 397 - } 398 398 - return 0 399 399 - } 400 400 - 401 401 - func (x *NodeSnapshot) GetEngines() map[string]*EngineAvailability { 402 402 - if x != nil { 403 403 - return x.Engines 404 404 - } 405 405 - return nil 406 406 - } 407 407 - 408 408 - func (x *NodeSnapshot) GetActiveLeaseIds() []string { 409 409 - if x != nil { 410 410 - return x.ActiveLeaseIds 411 411 - } 412 412 - return nil 413 413 - } 414 414 - 415 415 - // a seat reservation, carries no secrets 416 416 - type ReserveSeat struct { 417 417 - state protoimpl.MessageState `protogen:"open.v1"` 418 418 - LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 419 419 - TargetEngine string `protobuf:"bytes,2,opt,name=target_engine,json=targetEngine,proto3" json:"target_engine,omitempty"` 420 420 - RawPipelineJson string `protobuf:"bytes,3,opt,name=raw_pipeline_json,json=rawPipelineJson,proto3" json:"raw_pipeline_json,omitempty"` 421 421 - RawWorkflowJson string `protobuf:"bytes,4,opt,name=raw_workflow_json,json=rawWorkflowJson,proto3" json:"raw_workflow_json,omitempty"` 422 422 - // pipeline id, the executor reconstructs the exact WorkflowId from it 423 423 - Knot string `protobuf:"bytes,5,opt,name=knot,proto3" json:"knot,omitempty"` 424 424 - Rkey string `protobuf:"bytes,6,opt,name=rkey,proto3" json:"rkey,omitempty"` 425 425 - TtlSeconds uint32 `protobuf:"varint,7,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"` 426 426 - unknownFields protoimpl.UnknownFields 427 427 - sizeCache protoimpl.SizeCache 428 428 - } 429 429 - 430 430 - func (x *ReserveSeat) Reset() { 431 431 - *x = ReserveSeat{} 432 432 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[4] 433 433 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 434 434 - ms.StoreMessageInfo(mi) 435 435 - } 436 436 - 437 437 - func (x *ReserveSeat) String() string { 438 438 - return protoimpl.X.MessageStringOf(x) 439 439 - } 440 440 - 441 441 - func (*ReserveSeat) ProtoMessage() {} 442 442 - 443 443 - func (x *ReserveSeat) ProtoReflect() protoreflect.Message { 444 444 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[4] 445 445 - if x != nil { 446 446 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 447 447 - if ms.LoadMessageInfo() == nil { 448 448 - ms.StoreMessageInfo(mi) 449 449 - } 450 450 - return ms 451 451 - } 452 452 - return mi.MessageOf(x) 453 453 - } 454 454 - 455 455 - // Deprecated: Use ReserveSeat.ProtoReflect.Descriptor instead. 456 456 - func (*ReserveSeat) Descriptor() ([]byte, []int) { 457 457 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{4} 458 458 - } 459 459 - 460 460 - func (x *ReserveSeat) GetLeaseId() string { 461 461 - if x != nil { 462 462 - return x.LeaseId 463 463 - } 464 464 - return "" 465 465 - } 466 466 - 467 467 - func (x *ReserveSeat) GetTargetEngine() string { 468 468 - if x != nil { 469 469 - return x.TargetEngine 470 470 - } 471 471 - return "" 472 472 - } 473 473 - 474 474 - func (x *ReserveSeat) GetRawPipelineJson() string { 475 475 - if x != nil { 476 476 - return x.RawPipelineJson 477 477 - } 478 478 - return "" 479 479 - } 480 480 - 481 481 - func (x *ReserveSeat) GetRawWorkflowJson() string { 482 482 - if x != nil { 483 483 - return x.RawWorkflowJson 484 484 - } 485 485 - return "" 486 486 - } 487 487 - 488 488 - func (x *ReserveSeat) GetKnot() string { 489 489 - if x != nil { 490 490 - return x.Knot 491 491 - } 492 492 - return "" 493 493 - } 494 494 - 495 495 - func (x *ReserveSeat) GetRkey() string { 496 496 - if x != nil { 497 497 - return x.Rkey 498 498 - } 499 499 - return "" 500 500 - } 501 501 - 502 502 - func (x *ReserveSeat) GetTtlSeconds() uint32 { 503 503 - if x != nil { 504 504 - return x.TtlSeconds 505 505 - } 506 506 - return 0 507 507 - } 508 508 - 509 509 - type ReserveResult struct { 510 510 - state protoimpl.MessageState `protogen:"open.v1"` 511 511 - LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 512 512 - Accepted bool `protobuf:"varint,2,opt,name=accepted,proto3" json:"accepted,omitempty"` 513 513 - RejectReason string `protobuf:"bytes,3,opt,name=reject_reason,json=rejectReason,proto3" json:"reject_reason,omitempty"` 514 514 - RejectClass RejectClass `protobuf:"varint,4,opt,name=reject_class,json=rejectClass,proto3,enum=spindle.mill.v1.RejectClass" json:"reject_class,omitempty"` 515 515 - unknownFields protoimpl.UnknownFields 516 516 - sizeCache protoimpl.SizeCache 517 517 - } 518 518 - 519 519 - func (x *ReserveResult) Reset() { 520 520 - *x = ReserveResult{} 521 521 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[5] 522 522 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 523 523 - ms.StoreMessageInfo(mi) 524 524 - } 525 525 - 526 526 - func (x *ReserveResult) String() string { 527 527 - return protoimpl.X.MessageStringOf(x) 528 528 - } 529 529 - 530 530 - func (*ReserveResult) ProtoMessage() {} 531 531 - 532 532 - func (x *ReserveResult) ProtoReflect() protoreflect.Message { 533 533 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[5] 534 534 - if x != nil { 535 535 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 536 536 - if ms.LoadMessageInfo() == nil { 537 537 - ms.StoreMessageInfo(mi) 538 538 - } 539 539 - return ms 540 540 - } 541 541 - return mi.MessageOf(x) 542 542 - } 543 543 - 544 544 - // Deprecated: Use ReserveResult.ProtoReflect.Descriptor instead. 545 545 - func (*ReserveResult) Descriptor() ([]byte, []int) { 546 546 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{5} 547 547 - } 548 548 - 549 549 - func (x *ReserveResult) GetLeaseId() string { 550 550 - if x != nil { 551 551 - return x.LeaseId 552 552 - } 553 553 - return "" 554 554 - } 555 555 - 556 556 - func (x *ReserveResult) GetAccepted() bool { 557 557 - if x != nil { 558 558 - return x.Accepted 559 559 - } 560 560 - return false 561 561 - } 562 562 - 563 563 - func (x *ReserveResult) GetRejectReason() string { 564 564 - if x != nil { 565 565 - return x.RejectReason 566 566 - } 567 567 - return "" 568 568 - } 569 569 - 570 570 - func (x *ReserveResult) GetRejectClass() RejectClass { 571 571 - if x != nil { 572 572 - return x.RejectClass 573 573 - } 574 574 - return RejectClass_REJECT_CLASS_UNSPECIFIED 575 575 - } 576 576 - 577 577 - // a single unlocked secret 578 578 - type Secret struct { 579 579 - state protoimpl.MessageState `protogen:"open.v1"` 580 580 - Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` 581 581 - Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` 582 582 - unknownFields protoimpl.UnknownFields 583 583 - sizeCache protoimpl.SizeCache 584 584 - } 585 585 - 586 586 - func (x *Secret) Reset() { 587 587 - *x = Secret{} 588 588 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[6] 589 589 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 590 590 - ms.StoreMessageInfo(mi) 591 591 - } 592 592 - 593 593 - func (x *Secret) String() string { 594 594 - return protoimpl.X.MessageStringOf(x) 595 595 - } 596 596 - 597 597 - func (*Secret) ProtoMessage() {} 598 598 - 599 599 - func (x *Secret) ProtoReflect() protoreflect.Message { 600 600 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[6] 601 601 - if x != nil { 602 602 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 603 603 - if ms.LoadMessageInfo() == nil { 604 604 - ms.StoreMessageInfo(mi) 605 605 - } 606 606 - return ms 607 607 - } 608 608 - return mi.MessageOf(x) 609 609 - } 610 610 - 611 611 - // Deprecated: Use Secret.ProtoReflect.Descriptor instead. 612 612 - func (*Secret) Descriptor() ([]byte, []int) { 613 613 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{6} 614 614 - } 615 615 - 616 616 - func (x *Secret) GetKey() string { 617 617 - if x != nil { 618 618 - return x.Key 619 619 - } 620 620 - return "" 621 621 - } 622 622 - 623 623 - func (x *Secret) GetValue() string { 624 624 - if x != nil { 625 625 - return x.Value 626 626 - } 627 627 - return "" 628 628 - } 629 629 - 630 630 - // promotes a reservation to a running job and hands over the secrets 631 631 - type CommitLease struct { 632 632 - state protoimpl.MessageState `protogen:"open.v1"` 633 633 - LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 634 634 - Secrets []*Secret `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty"` 635 635 - unknownFields protoimpl.UnknownFields 636 636 - sizeCache protoimpl.SizeCache 637 637 - } 638 638 - 639 639 - func (x *CommitLease) Reset() { 640 640 - *x = CommitLease{} 641 641 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] 642 642 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 643 643 - ms.StoreMessageInfo(mi) 644 644 - } 645 645 - 646 646 - func (x *CommitLease) String() string { 647 647 - return protoimpl.X.MessageStringOf(x) 648 648 - } 649 649 - 650 650 - func (*CommitLease) ProtoMessage() {} 651 651 - 652 652 - func (x *CommitLease) ProtoReflect() protoreflect.Message { 653 653 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] 654 654 - if x != nil { 655 655 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 656 656 - if ms.LoadMessageInfo() == nil { 657 657 - ms.StoreMessageInfo(mi) 658 658 - } 659 659 - return ms 660 660 - } 661 661 - return mi.MessageOf(x) 662 662 - } 663 663 - 664 664 - // Deprecated: Use CommitLease.ProtoReflect.Descriptor instead. 665 665 - func (*CommitLease) Descriptor() ([]byte, []int) { 666 666 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{7} 667 667 - } 668 668 - 669 669 - func (x *CommitLease) GetLeaseId() string { 670 670 - if x != nil { 671 671 - return x.LeaseId 672 672 - } 673 673 - return "" 674 674 - } 675 675 - 676 676 - func (x *CommitLease) GetSecrets() []*Secret { 677 677 - if x != nil { 678 678 - return x.Secrets 679 679 - } 680 680 - return nil 681 681 - } 682 682 - 683 683 - type Committed struct { 684 684 - state protoimpl.MessageState `protogen:"open.v1"` 685 685 - LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 686 686 - unknownFields protoimpl.UnknownFields 687 687 - sizeCache protoimpl.SizeCache 688 688 - } 689 689 - 690 690 - func (x *Committed) Reset() { 691 691 - *x = Committed{} 692 692 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] 693 693 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 694 694 - ms.StoreMessageInfo(mi) 695 695 - } 696 696 - 697 697 - func (x *Committed) String() string { 698 698 - return protoimpl.X.MessageStringOf(x) 699 699 - } 700 700 - 701 701 - func (*Committed) ProtoMessage() {} 702 702 - 703 703 - func (x *Committed) ProtoReflect() protoreflect.Message { 704 704 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] 705 705 - if x != nil { 706 706 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 707 707 - if ms.LoadMessageInfo() == nil { 708 708 - ms.StoreMessageInfo(mi) 709 709 - } 710 710 - return ms 711 711 - } 712 712 - return mi.MessageOf(x) 713 713 - } 714 714 - 715 715 - // Deprecated: Use Committed.ProtoReflect.Descriptor instead. 716 716 - func (*Committed) Descriptor() ([]byte, []int) { 717 717 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{8} 718 718 - } 719 719 - 720 720 - func (x *Committed) GetLeaseId() string { 721 721 - if x != nil { 722 722 - return x.LeaseId 723 723 - } 724 724 - return "" 725 725 - } 726 726 - 727 727 - // drops a reservation that was never committed 728 728 - type ReleaseLease struct { 729 729 - state protoimpl.MessageState `protogen:"open.v1"` 730 730 - LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 731 731 - unknownFields protoimpl.UnknownFields 732 732 - sizeCache protoimpl.SizeCache 733 733 - } 734 734 - 735 735 - func (x *ReleaseLease) Reset() { 736 736 - *x = ReleaseLease{} 737 737 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] 738 738 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 739 739 - ms.StoreMessageInfo(mi) 740 740 - } 741 741 - 742 742 - func (x *ReleaseLease) String() string { 743 743 - return protoimpl.X.MessageStringOf(x) 744 744 - } 745 745 - 746 746 - func (*ReleaseLease) ProtoMessage() {} 747 747 - 748 748 - func (x *ReleaseLease) ProtoReflect() protoreflect.Message { 749 749 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] 750 750 - if x != nil { 751 751 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 752 752 - if ms.LoadMessageInfo() == nil { 753 753 - ms.StoreMessageInfo(mi) 754 754 - } 755 755 - return ms 756 756 - } 757 757 - return mi.MessageOf(x) 758 758 - } 759 759 - 760 760 - // Deprecated: Use ReleaseLease.ProtoReflect.Descriptor instead. 761 761 - func (*ReleaseLease) Descriptor() ([]byte, []int) { 762 762 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{9} 763 763 - } 764 764 - 765 765 - func (x *ReleaseLease) GetLeaseId() string { 766 766 - if x != nil { 767 767 - return x.LeaseId 768 768 - } 769 769 - return "" 770 770 - } 771 771 - 772 772 - // cancels a running attempt 773 773 - type CancelAttempt struct { 774 774 - state protoimpl.MessageState `protogen:"open.v1"` 775 775 - LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 776 776 - Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` 777 777 - unknownFields protoimpl.UnknownFields 778 778 - sizeCache protoimpl.SizeCache 779 779 - } 780 780 - 781 781 - func (x *CancelAttempt) Reset() { 782 782 - *x = CancelAttempt{} 783 783 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] 784 784 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 785 785 - ms.StoreMessageInfo(mi) 786 786 - } 787 787 - 788 788 - func (x *CancelAttempt) String() string { 789 789 - return protoimpl.X.MessageStringOf(x) 790 790 - } 791 791 - 792 792 - func (*CancelAttempt) ProtoMessage() {} 793 793 - 794 794 - func (x *CancelAttempt) ProtoReflect() protoreflect.Message { 795 795 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] 796 796 - if x != nil { 797 797 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 798 798 - if ms.LoadMessageInfo() == nil { 799 799 - ms.StoreMessageInfo(mi) 800 800 - } 801 801 - return ms 802 802 - } 803 803 - return mi.MessageOf(x) 804 804 - } 805 805 - 806 806 - // Deprecated: Use CancelAttempt.ProtoReflect.Descriptor instead. 807 807 - func (*CancelAttempt) Descriptor() ([]byte, []int) { 808 808 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{10} 809 809 - } 810 810 - 811 811 - func (x *CancelAttempt) GetLeaseId() string { 812 812 - if x != nil { 813 813 - return x.LeaseId 814 814 - } 815 815 - return "" 816 816 - } 817 817 - 818 818 - func (x *CancelAttempt) GetReason() string { 819 819 - if x != nil { 820 820 - return x.Reason 821 821 - } 822 822 - return "" 823 823 - } 824 824 - 825 825 - type CancelAck struct { 826 826 - state protoimpl.MessageState `protogen:"open.v1"` 827 827 - LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 828 828 - unknownFields protoimpl.UnknownFields 829 829 - sizeCache protoimpl.SizeCache 830 830 - } 831 831 - 832 832 - func (x *CancelAck) Reset() { 833 833 - *x = CancelAck{} 834 834 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] 835 835 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 836 836 - ms.StoreMessageInfo(mi) 837 837 - } 838 838 - 839 839 - func (x *CancelAck) String() string { 840 840 - return protoimpl.X.MessageStringOf(x) 841 841 - } 842 842 - 843 843 - func (*CancelAck) ProtoMessage() {} 844 844 - 845 845 - func (x *CancelAck) ProtoReflect() protoreflect.Message { 846 846 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] 847 847 - if x != nil { 848 848 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 849 849 - if ms.LoadMessageInfo() == nil { 850 850 - ms.StoreMessageInfo(mi) 851 851 - } 852 852 - return ms 853 853 - } 854 854 - return mi.MessageOf(x) 855 855 - } 856 856 - 857 857 - // Deprecated: Use CancelAck.ProtoReflect.Descriptor instead. 858 858 - func (*CancelAck) Descriptor() ([]byte, []int) { 859 859 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{11} 860 860 - } 861 861 - 862 862 - func (x *CancelAck) GetLeaseId() string { 863 863 - if x != nil { 864 864 - return x.LeaseId 865 865 - } 866 866 - return "" 867 867 - } 868 868 - 869 869 - type StatusEvent struct { 870 870 - state protoimpl.MessageState `protogen:"open.v1"` 871 871 - Status NonterminalStatus `protobuf:"varint,1,opt,name=status,proto3,enum=spindle.mill.v1.NonterminalStatus" json:"status,omitempty"` 872 872 - Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` 873 873 - ExitCode int64 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` 874 874 - unknownFields protoimpl.UnknownFields 875 875 - sizeCache protoimpl.SizeCache 876 876 - } 877 877 - 878 878 - func (x *StatusEvent) Reset() { 879 879 - *x = StatusEvent{} 880 880 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] 881 881 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 882 882 - ms.StoreMessageInfo(mi) 883 883 - } 884 884 - 885 885 - func (x *StatusEvent) String() string { 886 886 - return protoimpl.X.MessageStringOf(x) 887 887 - } 888 888 - 889 889 - func (*StatusEvent) ProtoMessage() {} 890 890 - 891 891 - func (x *StatusEvent) ProtoReflect() protoreflect.Message { 892 892 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] 893 893 - if x != nil { 894 894 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 895 895 - if ms.LoadMessageInfo() == nil { 896 896 - ms.StoreMessageInfo(mi) 897 897 - } 898 898 - return ms 899 899 - } 900 900 - return mi.MessageOf(x) 901 901 - } 902 902 - 903 903 - // Deprecated: Use StatusEvent.ProtoReflect.Descriptor instead. 904 904 - func (*StatusEvent) Descriptor() ([]byte, []int) { 905 905 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{12} 906 906 - } 907 907 - 908 908 - func (x *StatusEvent) GetStatus() NonterminalStatus { 909 909 - if x != nil { 910 910 - return x.Status 911 911 - } 912 912 - return NonterminalStatus_NONTERMINAL_STATUS_UNSPECIFIED 913 913 - } 914 914 - 915 915 - func (x *StatusEvent) GetError() string { 916 916 - if x != nil { 917 917 - return x.Error 918 918 - } 919 919 - return "" 920 920 - } 921 921 - 922 922 - func (x *StatusEvent) GetExitCode() int64 { 923 923 - if x != nil { 924 924 - return x.ExitCode 925 925 - } 926 926 - return 0 927 927 - } 928 928 - 929 929 - type LogArtifact struct { 930 930 - state protoimpl.MessageState `protogen:"open.v1"` 931 931 - Ref string `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` 932 932 - Hash string `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"` 933 933 - unknownFields protoimpl.UnknownFields 934 934 - sizeCache protoimpl.SizeCache 935 935 - } 936 936 - 937 937 - func (x *LogArtifact) Reset() { 938 938 - *x = LogArtifact{} 939 939 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] 940 940 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 941 941 - ms.StoreMessageInfo(mi) 942 942 - } 943 943 - 944 944 - func (x *LogArtifact) String() string { 945 945 - return protoimpl.X.MessageStringOf(x) 946 946 - } 947 947 - 948 948 - func (*LogArtifact) ProtoMessage() {} 949 949 - 950 950 - func (x *LogArtifact) ProtoReflect() protoreflect.Message { 951 951 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] 952 952 - if x != nil { 953 953 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 954 954 - if ms.LoadMessageInfo() == nil { 955 955 - ms.StoreMessageInfo(mi) 956 956 - } 957 957 - return ms 958 958 - } 959 959 - return mi.MessageOf(x) 960 960 - } 961 961 - 962 962 - // Deprecated: Use LogArtifact.ProtoReflect.Descriptor instead. 963 963 - func (*LogArtifact) Descriptor() ([]byte, []int) { 964 964 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{13} 965 965 - } 966 966 - 967 967 - func (x *LogArtifact) GetRef() string { 968 968 - if x != nil { 969 969 - return x.Ref 970 970 - } 971 971 - return "" 972 972 - } 973 973 - 974 974 - func (x *LogArtifact) GetHash() string { 975 975 - if x != nil { 976 976 - return x.Hash 977 977 - } 978 978 - return "" 979 979 - } 980 980 - 981 981 - // terminal outcome of an attempt 982 982 - type AttemptResult struct { 983 983 - state protoimpl.MessageState `protogen:"open.v1"` 984 984 - Status TerminalStatus `protobuf:"varint,1,opt,name=status,proto3,enum=spindle.mill.v1.TerminalStatus" json:"status,omitempty"` 985 985 - Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` 986 986 - ExitCode int64 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` 987 987 - LogArtifact *LogArtifact `protobuf:"bytes,4,opt,name=log_artifact,json=logArtifact,proto3" json:"log_artifact,omitempty"` 988 988 - unknownFields protoimpl.UnknownFields 989 989 - sizeCache protoimpl.SizeCache 990 990 - } 991 991 - 992 992 - func (x *AttemptResult) Reset() { 993 993 - *x = AttemptResult{} 994 994 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] 995 995 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 996 996 - ms.StoreMessageInfo(mi) 997 997 - } 998 998 - 999 999 - func (x *AttemptResult) String() string { 1000 1000 - return protoimpl.X.MessageStringOf(x) 1001 1001 - } 1002 1002 - 1003 1003 - func (*AttemptResult) ProtoMessage() {} 1004 1004 - 1005 1005 - func (x *AttemptResult) ProtoReflect() protoreflect.Message { 1006 1006 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] 1007 1007 - if x != nil { 1008 1008 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1009 1009 - if ms.LoadMessageInfo() == nil { 1010 1010 - ms.StoreMessageInfo(mi) 1011 1011 - } 1012 1012 - return ms 1013 1013 - } 1014 1014 - return mi.MessageOf(x) 1015 1015 - } 1016 1016 - 1017 1017 - // Deprecated: Use AttemptResult.ProtoReflect.Descriptor instead. 1018 1018 - func (*AttemptResult) Descriptor() ([]byte, []int) { 1019 1019 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{14} 1020 1020 - } 1021 1021 - 1022 1022 - func (x *AttemptResult) GetStatus() TerminalStatus { 1023 1023 - if x != nil { 1024 1024 - return x.Status 1025 1025 - } 1026 1026 - return TerminalStatus_TERMINAL_STATUS_UNSPECIFIED 1027 1027 - } 1028 1028 - 1029 1029 - func (x *AttemptResult) GetError() string { 1030 1030 - if x != nil { 1031 1031 - return x.Error 1032 1032 - } 1033 1033 - return "" 1034 1034 - } 1035 1035 - 1036 1036 - func (x *AttemptResult) GetExitCode() int64 { 1037 1037 - if x != nil { 1038 1038 - return x.ExitCode 1039 1039 - } 1040 1040 - return 0 1041 1041 - } 1042 1042 - 1043 1043 - func (x *AttemptResult) GetLogArtifact() *LogArtifact { 1044 1044 - if x != nil { 1045 1045 - return x.LogArtifact 1046 1046 - } 1047 1047 - return nil 1048 1048 - } 1049 1049 - 1050 1050 - // live non-replay log frame 1051 1051 - type LiveLog struct { 1052 1052 - state protoimpl.MessageState `protogen:"open.v1"` 1053 1053 - LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 1054 1054 - RawJson []byte `protobuf:"bytes,2,opt,name=raw_json,json=rawJson,proto3" json:"raw_json,omitempty"` 1055 1055 - unknownFields protoimpl.UnknownFields 1056 1056 - sizeCache protoimpl.SizeCache 1057 1057 - } 1058 1058 - 1059 1059 - func (x *LiveLog) Reset() { 1060 1060 - *x = LiveLog{} 1061 1061 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] 1062 1062 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1063 1063 - ms.StoreMessageInfo(mi) 1064 1064 - } 1065 1065 - 1066 1066 - func (x *LiveLog) String() string { 1067 1067 - return protoimpl.X.MessageStringOf(x) 1068 1068 - } 1069 1069 - 1070 1070 - func (*LiveLog) ProtoMessage() {} 1071 1071 - 1072 1072 - func (x *LiveLog) ProtoReflect() protoreflect.Message { 1073 1073 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] 1074 1074 - if x != nil { 1075 1075 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1076 1076 - if ms.LoadMessageInfo() == nil { 1077 1077 - ms.StoreMessageInfo(mi) 1078 1078 - } 1079 1079 - return ms 1080 1080 - } 1081 1081 - return mi.MessageOf(x) 1082 1082 - } 1083 1083 - 1084 1084 - // Deprecated: Use LiveLog.ProtoReflect.Descriptor instead. 1085 1085 - func (*LiveLog) Descriptor() ([]byte, []int) { 1086 1086 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{15} 1087 1087 - } 1088 1088 - 1089 1089 - func (x *LiveLog) GetLeaseId() string { 1090 1090 - if x != nil { 1091 1091 - return x.LeaseId 1092 1092 - } 1093 1093 - return "" 1094 1094 - } 1095 1095 - 1096 1096 - func (x *LiveLog) GetRawJson() []byte { 1097 1097 - if x != nil { 1098 1098 - return x.RawJson 1099 1099 - } 1100 1100 - return nil 1101 1101 - } 1102 1102 - 1103 1103 - // one event in the executor's stream back to the mill 1104 1104 - type Event struct { 1105 1105 - state protoimpl.MessageState `protogen:"open.v1"` 1106 1106 - Seqno uint64 `protobuf:"varint,1,opt,name=seqno,proto3" json:"seqno,omitempty"` 1107 1107 - LeaseId string `protobuf:"bytes,2,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` 1108 1108 - // Types that are valid to be assigned to Payload: 1109 1109 - // 1110 1110 - // *Event_StatusEvent 1111 1111 - // *Event_AttemptResult 1112 1112 - Payload isEvent_Payload `protobuf_oneof:"payload"` 1113 1113 - unknownFields protoimpl.UnknownFields 1114 1114 - sizeCache protoimpl.SizeCache 1115 1115 - } 1116 1116 - 1117 1117 - func (x *Event) Reset() { 1118 1118 - *x = Event{} 1119 1119 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] 1120 1120 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1121 1121 - ms.StoreMessageInfo(mi) 1122 1122 - } 1123 1123 - 1124 1124 - func (x *Event) String() string { 1125 1125 - return protoimpl.X.MessageStringOf(x) 1126 1126 - } 1127 1127 - 1128 1128 - func (*Event) ProtoMessage() {} 1129 1129 - 1130 1130 - func (x *Event) ProtoReflect() protoreflect.Message { 1131 1131 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] 1132 1132 - if x != nil { 1133 1133 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1134 1134 - if ms.LoadMessageInfo() == nil { 1135 1135 - ms.StoreMessageInfo(mi) 1136 1136 - } 1137 1137 - return ms 1138 1138 - } 1139 1139 - return mi.MessageOf(x) 1140 1140 - } 1141 1141 - 1142 1142 - // Deprecated: Use Event.ProtoReflect.Descriptor instead. 1143 1143 - func (*Event) Descriptor() ([]byte, []int) { 1144 1144 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16} 1145 1145 - } 1146 1146 - 1147 1147 - func (x *Event) GetSeqno() uint64 { 1148 1148 - if x != nil { 1149 1149 - return x.Seqno 1150 1150 - } 1151 1151 - return 0 1152 1152 - } 1153 1153 - 1154 1154 - func (x *Event) GetLeaseId() string { 1155 1155 - if x != nil { 1156 1156 - return x.LeaseId 1157 1157 - } 1158 1158 - return "" 1159 1159 - } 1160 1160 - 1161 1161 - func (x *Event) GetPayload() isEvent_Payload { 1162 1162 - if x != nil { 1163 1163 - return x.Payload 1164 1164 - } 1165 1165 - return nil 1166 1166 - } 1167 1167 - 1168 1168 - func (x *Event) GetStatusEvent() *StatusEvent { 1169 1169 - if x != nil { 1170 1170 - if x, ok := x.Payload.(*Event_StatusEvent); ok { 1171 1171 - return x.StatusEvent 1172 1172 - } 1173 1173 - } 1174 1174 - return nil 1175 1175 - } 1176 1176 - 1177 1177 - func (x *Event) GetAttemptResult() *AttemptResult { 1178 1178 - if x != nil { 1179 1179 - if x, ok := x.Payload.(*Event_AttemptResult); ok { 1180 1180 - return x.AttemptResult 1181 1181 - } 1182 1182 - } 1183 1183 - return nil 1184 1184 - } 1185 1185 - 1186 1186 - type isEvent_Payload interface { 1187 1187 - isEvent_Payload() 1188 1188 - } 1189 1189 - 1190 1190 - type Event_StatusEvent struct { 1191 1191 - StatusEvent *StatusEvent `protobuf:"bytes,3,opt,name=status_event,json=statusEvent,proto3,oneof"` 1192 1192 - } 1193 1193 - 1194 1194 - type Event_AttemptResult struct { 1195 1195 - AttemptResult *AttemptResult `protobuf:"bytes,4,opt,name=attempt_result,json=attemptResult,proto3,oneof"` 1196 1196 - } 1197 1197 - 1198 1198 - func (*Event_StatusEvent) isEvent_Payload() {} 1199 1199 - 1200 1200 - func (*Event_AttemptResult) isEvent_Payload() {} 1201 1201 - 1202 1202 - // a flushed bundle of events 1203 1203 - type EventBatch struct { 1204 1204 - state protoimpl.MessageState `protogen:"open.v1"` 1205 1205 - Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"` 1206 1206 - Events []*Event `protobuf:"bytes,2,rep,name=events,proto3" json:"events,omitempty"` 1207 1207 - unknownFields protoimpl.UnknownFields 1208 1208 - sizeCache protoimpl.SizeCache 1209 1209 - } 1210 1210 - 1211 1211 - func (x *EventBatch) Reset() { 1212 1212 - *x = EventBatch{} 1213 1213 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] 1214 1214 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1215 1215 - ms.StoreMessageInfo(mi) 1216 1216 - } 1217 1217 - 1218 1218 - func (x *EventBatch) String() string { 1219 1219 - return protoimpl.X.MessageStringOf(x) 1220 1220 - } 1221 1221 - 1222 1222 - func (*EventBatch) ProtoMessage() {} 1223 1223 - 1224 1224 - func (x *EventBatch) ProtoReflect() protoreflect.Message { 1225 1225 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] 1226 1226 - if x != nil { 1227 1227 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1228 1228 - if ms.LoadMessageInfo() == nil { 1229 1229 - ms.StoreMessageInfo(mi) 1230 1230 - } 1231 1231 - return ms 1232 1232 - } 1233 1233 - return mi.MessageOf(x) 1234 1234 - } 1235 1235 - 1236 1236 - // Deprecated: Use EventBatch.ProtoReflect.Descriptor instead. 1237 1237 - func (*EventBatch) Descriptor() ([]byte, []int) { 1238 1238 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17} 1239 1239 - } 1240 1240 - 1241 1241 - func (x *EventBatch) GetEpoch() string { 1242 1242 - if x != nil { 1243 1243 - return x.Epoch 1244 1244 - } 1245 1245 - return "" 1246 1246 - } 1247 1247 - 1248 1248 - func (x *EventBatch) GetEvents() []*Event { 1249 1249 - if x != nil { 1250 1250 - return x.Events 1251 1251 - } 1252 1252 - return nil 1253 1253 - } 1254 1254 - 1255 1255 - // all events up to and including up_to_seqno are durably processed 1256 1256 - type Ack struct { 1257 1257 - state protoimpl.MessageState `protogen:"open.v1"` 1258 1258 - Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"` 1259 1259 - UpToSeqno uint64 `protobuf:"varint,2,opt,name=up_to_seqno,json=upToSeqno,proto3" json:"up_to_seqno,omitempty"` 1260 1260 - unknownFields protoimpl.UnknownFields 1261 1261 - sizeCache protoimpl.SizeCache 1262 1262 - } 1263 1263 - 1264 1264 - func (x *Ack) Reset() { 1265 1265 - *x = Ack{} 1266 1266 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] 1267 1267 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1268 1268 - ms.StoreMessageInfo(mi) 1269 1269 - } 1270 1270 - 1271 1271 - func (x *Ack) String() string { 1272 1272 - return protoimpl.X.MessageStringOf(x) 1273 1273 - } 1274 1274 - 1275 1275 - func (*Ack) ProtoMessage() {} 1276 1276 - 1277 1277 - func (x *Ack) ProtoReflect() protoreflect.Message { 1278 1278 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] 1279 1279 - if x != nil { 1280 1280 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1281 1281 - if ms.LoadMessageInfo() == nil { 1282 1282 - ms.StoreMessageInfo(mi) 1283 1283 - } 1284 1284 - return ms 1285 1285 - } 1286 1286 - return mi.MessageOf(x) 1287 1287 - } 1288 1288 - 1289 1289 - // Deprecated: Use Ack.ProtoReflect.Descriptor instead. 1290 1290 - func (*Ack) Descriptor() ([]byte, []int) { 1291 1291 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18} 1292 1292 - } 1293 1293 - 1294 1294 - func (x *Ack) GetEpoch() string { 1295 1295 - if x != nil { 1296 1296 - return x.Epoch 1297 1297 - } 1298 1298 - return "" 1299 1299 - } 1300 1300 - 1301 1301 - func (x *Ack) GetUpToSeqno() uint64 { 1302 1302 - if x != nil { 1303 1303 - return x.UpToSeqno 1304 1304 - } 1305 1305 - return 0 1306 1306 - } 1307 1307 - 1308 1308 - type Message struct { 1309 1309 - state protoimpl.MessageState `protogen:"open.v1"` 1310 1310 - Hello *Hello `protobuf:"bytes,1,opt,name=hello,proto3" json:"hello,omitempty"` 1311 1311 - Resume *Resume `protobuf:"bytes,2,opt,name=resume,proto3" json:"resume,omitempty"` 1312 1312 - NodeSnapshot *NodeSnapshot `protobuf:"bytes,3,opt,name=node_snapshot,json=nodeSnapshot,proto3" json:"node_snapshot,omitempty"` 1313 1313 - ReserveSeat *ReserveSeat `protobuf:"bytes,4,opt,name=reserve_seat,json=reserveSeat,proto3" json:"reserve_seat,omitempty"` 1314 1314 - ReserveResult *ReserveResult `protobuf:"bytes,5,opt,name=reserve_result,json=reserveResult,proto3" json:"reserve_result,omitempty"` 1315 1315 - CommitLease *CommitLease `protobuf:"bytes,6,opt,name=commit_lease,json=commitLease,proto3" json:"commit_lease,omitempty"` 1316 1316 - Committed *Committed `protobuf:"bytes,7,opt,name=committed,proto3" json:"committed,omitempty"` 1317 1317 - ReleaseLease *ReleaseLease `protobuf:"bytes,8,opt,name=release_lease,json=releaseLease,proto3" json:"release_lease,omitempty"` 1318 1318 - CancelAttempt *CancelAttempt `protobuf:"bytes,9,opt,name=cancel_attempt,json=cancelAttempt,proto3" json:"cancel_attempt,omitempty"` 1319 1319 - CancelAck *CancelAck `protobuf:"bytes,10,opt,name=cancel_ack,json=cancelAck,proto3" json:"cancel_ack,omitempty"` 1320 1320 - EventBatch *EventBatch `protobuf:"bytes,11,opt,name=event_batch,json=eventBatch,proto3" json:"event_batch,omitempty"` 1321 1321 - Ack *Ack `protobuf:"bytes,12,opt,name=ack,proto3" json:"ack,omitempty"` 1322 1322 - LiveLog *LiveLog `protobuf:"bytes,13,opt,name=live_log,json=liveLog,proto3" json:"live_log,omitempty"` 1323 1323 - unknownFields protoimpl.UnknownFields 1324 1324 - sizeCache protoimpl.SizeCache 1325 1325 - } 1326 1326 - 1327 1327 - func (x *Message) Reset() { 1328 1328 - *x = Message{} 1329 1329 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] 1330 1330 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1331 1331 - ms.StoreMessageInfo(mi) 1332 1332 - } 1333 1333 - 1334 1334 - func (x *Message) String() string { 1335 1335 - return protoimpl.X.MessageStringOf(x) 1336 1336 - } 1337 1337 - 1338 1338 - func (*Message) ProtoMessage() {} 1339 1339 - 1340 1340 - func (x *Message) ProtoReflect() protoreflect.Message { 1341 1341 - mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] 1342 1342 - if x != nil { 1343 1343 - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 1344 1344 - if ms.LoadMessageInfo() == nil { 1345 1345 - ms.StoreMessageInfo(mi) 1346 1346 - } 1347 1347 - return ms 1348 1348 - } 1349 1349 - return mi.MessageOf(x) 1350 1350 - } 1351 1351 - 1352 1352 - // Deprecated: Use Message.ProtoReflect.Descriptor instead. 1353 1353 - func (*Message) Descriptor() ([]byte, []int) { 1354 1354 - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{19} 1355 1355 - } 1356 1356 - 1357 1357 - func (x *Message) GetHello() *Hello { 1358 1358 - if x != nil { 1359 1359 - return x.Hello 1360 1360 - } 1361 1361 - return nil 1362 1362 - } 1363 1363 - 1364 1364 - func (x *Message) GetResume() *Resume { 1365 1365 - if x != nil { 1366 1366 - return x.Resume 1367 1367 - } 1368 1368 - return nil 1369 1369 - } 1370 1370 - 1371 1371 - func (x *Message) GetNodeSnapshot() *NodeSnapshot { 1372 1372 - if x != nil { 1373 1373 - return x.NodeSnapshot 1374 1374 - } 1375 1375 - return nil 1376 1376 - } 1377 1377 - 1378 1378 - func (x *Message) GetReserveSeat() *ReserveSeat { 1379 1379 - if x != nil { 1380 1380 - return x.ReserveSeat 1381 1381 - } 1382 1382 - return nil 1383 1383 - } 1384 1384 - 1385 1385 - func (x *Message) GetReserveResult() *ReserveResult { 1386 1386 - if x != nil { 1387 1387 - return x.ReserveResult 1388 1388 - } 1389 1389 - return nil 1390 1390 - } 1391 1391 - 1392 1392 - func (x *Message) GetCommitLease() *CommitLease { 1393 1393 - if x != nil { 1394 1394 - return x.CommitLease 1395 1395 - } 1396 1396 - return nil 1397 1397 - } 1398 1398 - 1399 1399 - func (x *Message) GetCommitted() *Committed { 1400 1400 - if x != nil { 1401 1401 - return x.Committed 1402 1402 - } 1403 1403 - return nil 1404 1404 - } 1405 1405 - 1406 1406 - func (x *Message) GetReleaseLease() *ReleaseLease { 1407 1407 - if x != nil { 1408 1408 - return x.ReleaseLease 1409 1409 - } 1410 1410 - return nil 1411 1411 - } 1412 1412 - 1413 1413 - func (x *Message) GetCancelAttempt() *CancelAttempt { 1414 1414 - if x != nil { 1415 1415 - return x.CancelAttempt 1416 1416 - } 1417 1417 - return nil 1418 1418 - } 1419 1419 - 1420 1420 - func (x *Message) GetCancelAck() *CancelAck { 1421 1421 - if x != nil { 1422 1422 - return x.CancelAck 1423 1423 - } 1424 1424 - return nil 1425 1425 - } 1426 1426 - 1427 1427 - func (x *Message) GetEventBatch() *EventBatch { 1428 1428 - if x != nil { 1429 1429 - return x.EventBatch 1430 1430 - } 1431 1431 - return nil 1432 1432 - } 1433 1433 - 1434 1434 - func (x *Message) GetAck() *Ack { 1435 1435 - if x != nil { 1436 1436 - return x.Ack 1437 1437 - } 1438 1438 - return nil 1439 1439 - } 1440 1440 - 1441 1441 - func (x *Message) GetLiveLog() *LiveLog { 1442 1442 - if x != nil { 1443 1443 - return x.LiveLog 1444 1444 - } 1445 1445 - return nil 1446 1446 - } 1447 1447 - 1448 1448 - var File_spindle_mill_v1_mill_proto protoreflect.FileDescriptor 1449 1449 - 1450 1450 - const file_spindle_mill_v1_mill_proto_rawDesc = "" + 1451 1451 - "\n" + 1452 1452 - "\x1aspindle/mill/v1/mill.proto\x12\x0fspindle.mill.v1\x1a\x1bbuf/validate/validate.proto\"}\n" + 1453 1453 - "\x05Hello\x12)\n" + 1454 1454 - "\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x12\x12\n" + 1455 1455 - "\x04arch\x18\x02 \x01(\tR\x04arch\x12\x16\n" + 1456 1456 - "\x06labels\x18\x03 \x03(\tR\x06labels\x12\x1d\n" + 1457 1457 - "\x05epoch\x18\x04 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\"D\n" + 1458 1458 - "\x06Resume\x12\x1d\n" + 1459 1459 - "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1b\n" + 1460 1460 - "\tack_seqno\x18\x02 \x01(\x04R\backSeqno\"\xae\x01\n" + 1461 1461 - "\x12EngineAvailability\x12\x1c\n" + 1462 1462 - "\tavailable\x18\x01 \x01(\bR\tavailable\x12A\n" + 1463 1463 - "\x04load\x18\x02 \x03(\v2-.spindle.mill.v1.EngineAvailability.LoadEntryR\x04load\x1a7\n" + 1464 1464 - "\tLoadEntry\x12\x10\n" + 1465 1465 - "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + 1466 1466 - "\x05value\x18\x02 \x01(\x01R\x05value:\x028\x01\"\xfe\x01\n" + 1467 1467 - "\fNodeSnapshot\x12\x1d\n" + 1468 1468 - "\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12D\n" + 1469 1469 - "\aengines\x18\x02 \x03(\v2*.spindle.mill.v1.NodeSnapshot.EnginesEntryR\aengines\x12(\n" + 1470 1470 - "\x10active_lease_ids\x18\x03 \x03(\tR\x0eactiveLeaseIds\x1a_\n" + 1471 1471 - "\fEnginesEntry\x12\x10\n" + 1472 1472 - "\x03key\x18\x01 \x01(\tR\x03key\x129\n" + 1473 1473 - "\x05value\x18\x02 \x01(\v2#.spindle.mill.v1.EngineAvailabilityR\x05value:\x028\x01\"\x80\x02\n" + 1474 1474 - "\vReserveSeat\x12\"\n" + 1475 1475 - "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12,\n" + 1476 1476 - "\rtarget_engine\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\ftargetEngine\x12*\n" + 1477 1477 - "\x11raw_pipeline_json\x18\x03 \x01(\tR\x0frawPipelineJson\x12*\n" + 1478 1478 - "\x11raw_workflow_json\x18\x04 \x01(\tR\x0frawWorkflowJson\x12\x12\n" + 1479 1479 - "\x04knot\x18\x05 \x01(\tR\x04knot\x12\x12\n" + 1480 1480 - "\x04rkey\x18\x06 \x01(\tR\x04rkey\x12\x1f\n" + 1481 1481 - "\vttl_seconds\x18\a \x01(\rR\n" + 1482 1482 - "ttlSeconds\"\xbf\x01\n" + 1483 1483 - "\rReserveResult\x12\"\n" + 1484 1484 - "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x1a\n" + 1485 1485 - "\baccepted\x18\x02 \x01(\bR\baccepted\x12#\n" + 1486 1486 - "\rreject_reason\x18\x03 \x01(\tR\frejectReason\x12I\n" + 1487 1487 - "\freject_class\x18\x04 \x01(\x0e2\x1c.spindle.mill.v1.RejectClassB\b\xbaH\x05\x82\x01\x02\x10\x01R\vrejectClass\"0\n" + 1488 1488 - "\x06Secret\x12\x10\n" + 1489 1489 - "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + 1490 1490 - "\x05value\x18\x02 \x01(\tR\x05value\"d\n" + 1491 1491 - "\vCommitLease\x12\"\n" + 1492 1492 - "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x121\n" + 1493 1493 - "\asecrets\x18\x02 \x03(\v2\x17.spindle.mill.v1.SecretR\asecrets\"/\n" + 1494 1494 - "\tCommitted\x12\"\n" + 1495 1495 - "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"2\n" + 1496 1496 - "\fReleaseLease\x12\"\n" + 1497 1497 - "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"K\n" + 1498 1498 - "\rCancelAttempt\x12\"\n" + 1499 1499 - "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x16\n" + 1500 1500 - "\x06reason\x18\x02 \x01(\tR\x06reason\"/\n" + 1501 1501 - "\tCancelAck\x12\"\n" + 1502 1502 - "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"\x93\x01\n" + 1503 1503 - "\vStatusEvent\x12F\n" + 1504 1504 - "\x06status\x18\x01 \x01(\x0e2\".spindle.mill.v1.NonterminalStatusB\n" + 1505 1505 - "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" + 1506 1506 - "\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" + 1507 1507 - "\texit_code\x18\x03 \x01(\x03R\bexitCode\"E\n" + 1508 1508 - "\vLogArtifact\x12\x19\n" + 1509 1509 - "\x03ref\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x03ref\x12\x1b\n" + 1510 1510 - "\x04hash\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x04hash\"\xd3\x01\n" + 1511 1511 - "\rAttemptResult\x12C\n" + 1512 1512 - "\x06status\x18\x01 \x01(\x0e2\x1f.spindle.mill.v1.TerminalStatusB\n" + 1513 1513 - "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" + 1514 1514 - "\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" + 1515 1515 - "\texit_code\x18\x03 \x01(\x03R\bexitCode\x12?\n" + 1516 1516 - "\flog_artifact\x18\x04 \x01(\v2\x1c.spindle.mill.v1.LogArtifactR\vlogArtifact\"Q\n" + 1517 1517 - "\aLiveLog\x12\"\n" + 1518 1518 - "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\"\n" + 1519 1519 - "\braw_json\x18\x02 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\xe8\x01\n" + 1520 1520 - "\x05Event\x12\x1d\n" + 1521 1521 - "\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12\"\n" + 1522 1522 - "\blease_id\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12A\n" + 1523 1523 - "\fstatus_event\x18\x03 \x01(\v2\x1c.spindle.mill.v1.StatusEventH\x00R\vstatusEvent\x12G\n" + 1524 1524 - "\x0eattempt_result\x18\x04 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResultB\x10\n" + 1525 1525 - "\apayload\x12\x05\xbaH\x02\b\x01\"e\n" + 1526 1526 - "\n" + 1527 1527 - "EventBatch\x12\x1d\n" + 1528 1528 - "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x128\n" + 1529 1529 - "\x06events\x18\x02 \x03(\v2\x16.spindle.mill.v1.EventB\b\xbaH\x05\x92\x01\x02\b\x01R\x06events\"D\n" + 1530 1530 - "\x03Ack\x12\x1d\n" + 1531 1531 - "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1e\n" + 1532 1532 - "\vup_to_seqno\x18\x02 \x01(\x04R\tupToSeqno\"\xb8\a\n" + 1533 1533 - "\aMessage\x12,\n" + 1534 1534 - "\x05hello\x18\x01 \x01(\v2\x16.spindle.mill.v1.HelloR\x05hello\x12/\n" + 1535 1535 - "\x06resume\x18\x02 \x01(\v2\x17.spindle.mill.v1.ResumeR\x06resume\x12B\n" + 1536 1536 - "\rnode_snapshot\x18\x03 \x01(\v2\x1d.spindle.mill.v1.NodeSnapshotR\fnodeSnapshot\x12?\n" + 1537 1537 - "\freserve_seat\x18\x04 \x01(\v2\x1c.spindle.mill.v1.ReserveSeatR\vreserveSeat\x12E\n" + 1538 1538 - "\x0ereserve_result\x18\x05 \x01(\v2\x1e.spindle.mill.v1.ReserveResultR\rreserveResult\x12?\n" + 1539 1539 - "\fcommit_lease\x18\x06 \x01(\v2\x1c.spindle.mill.v1.CommitLeaseR\vcommitLease\x128\n" + 1540 1540 - "\tcommitted\x18\a \x01(\v2\x1a.spindle.mill.v1.CommittedR\tcommitted\x12B\n" + 1541 1541 - "\rrelease_lease\x18\b \x01(\v2\x1d.spindle.mill.v1.ReleaseLeaseR\freleaseLease\x12E\n" + 1542 1542 - "\x0ecancel_attempt\x18\t \x01(\v2\x1e.spindle.mill.v1.CancelAttemptR\rcancelAttempt\x129\n" + 1543 1543 - "\n" + 1544 1544 - "cancel_ack\x18\n" + 1545 1545 - " \x01(\v2\x1a.spindle.mill.v1.CancelAckR\tcancelAck\x12<\n" + 1546 1546 - "\vevent_batch\x18\v \x01(\v2\x1b.spindle.mill.v1.EventBatchR\n" + 1547 1547 - "eventBatch\x12&\n" + 1548 1548 - "\x03ack\x18\f \x01(\v2\x14.spindle.mill.v1.AckR\x03ack\x123\n" + 1549 1549 - "\blive_log\x18\r \x01(\v2\x18.spindle.mill.v1.LiveLogR\aliveLog:\xa5\x01\xbaH\xa1\x01\"\x9e\x01\n" + 1550 1550 - "\x05hello\n" + 1551 1551 - "\x06resume\n" + 1552 1552 - "\rnode_snapshot\n" + 1553 1553 - "\freserve_seat\n" + 1554 1554 - "\x0ereserve_result\n" + 1555 1555 - "\fcommit_lease\n" + 1556 1556 - "\tcommitted\n" + 1557 1557 - "\rrelease_lease\n" + 1558 1558 - "\x0ecancel_attempt\n" + 1559 1559 - "\n" + 1560 1560 - "cancel_ack\n" + 1561 1561 - "\vevent_batch\n" + 1562 1562 - "\x03ack\n" + 1563 1563 - "\blive_log\x10\x01*f\n" + 1564 1564 - "\vRejectClass\x12\x1c\n" + 1565 1565 - "\x18REJECT_CLASS_UNSPECIFIED\x10\x00\x12\x1a\n" + 1566 1566 - "\x16REJECT_CLASS_TRANSIENT\x10\x01\x12\x1d\n" + 1567 1567 - "\x19REJECT_CLASS_INCOMPATIBLE\x10\x02*D\n" + 1568 1568 - "\x11NonterminalStatus\x12\"\n" + 1569 1569 - "\x1eNONTERMINAL_STATUS_UNSPECIFIED\x10\x00\x12\v\n" + 1570 1570 - "\aRUNNING\x10\x01*f\n" + 1571 1571 - "\x0eTerminalStatus\x12\x1f\n" + 1572 1572 - "\x1bTERMINAL_STATUS_UNSPECIFIED\x10\x00\x12\v\n" + 1573 1573 - "\aSUCCESS\x10\x01\x12\n" + 1574 1574 - "\n" + 1575 1575 - "\x06FAILED\x10\x02\x12\v\n" + 1576 1576 - "\aTIMEOUT\x10\x03\x12\r\n" + 1577 1577 - "\tCANCELLED\x10\x04B0Z.tangled.org/core/spindle/mill/proto/gen;millv1b\x06proto3" 1578 1578 - 1579 1579 - var ( 1580 1580 - file_spindle_mill_v1_mill_proto_rawDescOnce sync.Once 1581 1581 - file_spindle_mill_v1_mill_proto_rawDescData []byte 1582 1582 - ) 1583 1583 - 1584 1584 - func file_spindle_mill_v1_mill_proto_rawDescGZIP() []byte { 1585 1585 - file_spindle_mill_v1_mill_proto_rawDescOnce.Do(func() { 1586 1586 - file_spindle_mill_v1_mill_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc))) 1587 1587 - }) 1588 1588 - return file_spindle_mill_v1_mill_proto_rawDescData 1589 1589 - } 1590 1590 - 1591 1591 - var file_spindle_mill_v1_mill_proto_enumTypes = make([]protoimpl.EnumInfo, 3) 1592 1592 - var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 22) 1593 1593 - var file_spindle_mill_v1_mill_proto_goTypes = []any{ 1594 1594 - (RejectClass)(0), // 0: spindle.mill.v1.RejectClass 1595 1595 - (NonterminalStatus)(0), // 1: spindle.mill.v1.NonterminalStatus 1596 1596 - (TerminalStatus)(0), // 2: spindle.mill.v1.TerminalStatus 1597 1597 - (*Hello)(nil), // 3: spindle.mill.v1.Hello 1598 1598 - (*Resume)(nil), // 4: spindle.mill.v1.Resume 1599 1599 - (*EngineAvailability)(nil), // 5: spindle.mill.v1.EngineAvailability 1600 1600 - (*NodeSnapshot)(nil), // 6: spindle.mill.v1.NodeSnapshot 1601 1601 - (*ReserveSeat)(nil), // 7: spindle.mill.v1.ReserveSeat 1602 1602 - (*ReserveResult)(nil), // 8: spindle.mill.v1.ReserveResult 1603 1603 - (*Secret)(nil), // 9: spindle.mill.v1.Secret 1604 1604 - (*CommitLease)(nil), // 10: spindle.mill.v1.CommitLease 1605 1605 - (*Committed)(nil), // 11: spindle.mill.v1.Committed 1606 1606 - (*ReleaseLease)(nil), // 12: spindle.mill.v1.ReleaseLease 1607 1607 - (*CancelAttempt)(nil), // 13: spindle.mill.v1.CancelAttempt 1608 1608 - (*CancelAck)(nil), // 14: spindle.mill.v1.CancelAck 1609 1609 - (*StatusEvent)(nil), // 15: spindle.mill.v1.StatusEvent 1610 1610 - (*LogArtifact)(nil), // 16: spindle.mill.v1.LogArtifact 1611 1611 - (*AttemptResult)(nil), // 17: spindle.mill.v1.AttemptResult 1612 1612 - (*LiveLog)(nil), // 18: spindle.mill.v1.LiveLog 1613 1613 - (*Event)(nil), // 19: spindle.mill.v1.Event 1614 1614 - (*EventBatch)(nil), // 20: spindle.mill.v1.EventBatch 1615 1615 - (*Ack)(nil), // 21: spindle.mill.v1.Ack 1616 1616 - (*Message)(nil), // 22: spindle.mill.v1.Message 1617 1617 - nil, // 23: spindle.mill.v1.EngineAvailability.LoadEntry 1618 1618 - nil, // 24: spindle.mill.v1.NodeSnapshot.EnginesEntry 1619 1619 - } 1620 1620 - var file_spindle_mill_v1_mill_proto_depIdxs = []int32{ 1621 1621 - 23, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry 1622 1622 - 24, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry 1623 1623 - 0, // 2: spindle.mill.v1.ReserveResult.reject_class:type_name -> spindle.mill.v1.RejectClass 1624 1624 - 9, // 3: spindle.mill.v1.CommitLease.secrets:type_name -> spindle.mill.v1.Secret 1625 1625 - 1, // 4: spindle.mill.v1.StatusEvent.status:type_name -> spindle.mill.v1.NonterminalStatus 1626 1626 - 2, // 5: spindle.mill.v1.AttemptResult.status:type_name -> spindle.mill.v1.TerminalStatus 1627 1627 - 16, // 6: spindle.mill.v1.AttemptResult.log_artifact:type_name -> spindle.mill.v1.LogArtifact 1628 1628 - 15, // 7: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent 1629 1629 - 17, // 8: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult 1630 1630 - 19, // 9: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event 1631 1631 - 3, // 10: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello 1632 1632 - 4, // 11: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume 1633 1633 - 6, // 12: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot 1634 1634 - 7, // 13: spindle.mill.v1.Message.reserve_seat:type_name -> spindle.mill.v1.ReserveSeat 1635 1635 - 8, // 14: spindle.mill.v1.Message.reserve_result:type_name -> spindle.mill.v1.ReserveResult 1636 1636 - 10, // 15: spindle.mill.v1.Message.commit_lease:type_name -> spindle.mill.v1.CommitLease 1637 1637 - 11, // 16: spindle.mill.v1.Message.committed:type_name -> spindle.mill.v1.Committed 1638 1638 - 12, // 17: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease 1639 1639 - 13, // 18: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt 1640 1640 - 14, // 19: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck 1641 1641 - 20, // 20: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch 1642 1642 - 21, // 21: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack 1643 1643 - 18, // 22: spindle.mill.v1.Message.live_log:type_name -> spindle.mill.v1.LiveLog 1644 1644 - 5, // 23: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability 1645 1645 - 24, // [24:24] is the sub-list for method output_type 1646 1646 - 24, // [24:24] is the sub-list for method input_type 1647 1647 - 24, // [24:24] is the sub-list for extension type_name 1648 1648 - 24, // [24:24] is the sub-list for extension extendee 1649 1649 - 0, // [0:24] is the sub-list for field type_name 1650 1650 - } 1651 1651 - 1652 1652 - func init() { file_spindle_mill_v1_mill_proto_init() } 1653 1653 - func file_spindle_mill_v1_mill_proto_init() { 1654 1654 - if File_spindle_mill_v1_mill_proto != nil { 1655 1655 - return 1656 1656 - } 1657 1657 - file_spindle_mill_v1_mill_proto_msgTypes[16].OneofWrappers = []any{ 1658 1658 - (*Event_StatusEvent)(nil), 1659 1659 - (*Event_AttemptResult)(nil), 1660 1660 - } 1661 1661 - type x struct{} 1662 1662 - out := protoimpl.TypeBuilder{ 1663 1663 - File: protoimpl.DescBuilder{ 1664 1664 - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 1665 1665 - RawDescriptor: unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc)), 1666 1666 - NumEnums: 3, 1667 1667 - NumMessages: 22, 1668 1668 - NumExtensions: 0, 1669 1669 - NumServices: 0, 1670 1670 - }, 1671 1671 - GoTypes: file_spindle_mill_v1_mill_proto_goTypes, 1672 1672 - DependencyIndexes: file_spindle_mill_v1_mill_proto_depIdxs, 1673 1673 - EnumInfos: file_spindle_mill_v1_mill_proto_enumTypes, 1674 1674 - MessageInfos: file_spindle_mill_v1_mill_proto_msgTypes, 1675 1675 - }.Build() 1676 1676 - File_spindle_mill_v1_mill_proto = out.File 1677 1677 - file_spindle_mill_v1_mill_proto_goTypes = nil 1678 1678 - file_spindle_mill_v1_mill_proto_depIdxs = nil 1679 1679 - }

-103

spindle/mill/proto/protocol.go

Reviewed

··· 1 1 - // package millproto carries the mill<->executor session protocol. a new message 2 2 - // vocabulary over the same length-prefixed protobuf framing the spindle already 3 3 - // uses to talk to the microVM guest (see spindle/agentproto). only the framing 4 4 - // pattern is shared. the messages are entirely separate 5 5 - package millproto 6 6 - 7 7 - import ( 8 8 - "encoding/binary" 9 9 - "fmt" 10 10 - "io" 11 11 - "sync" 12 12 - 13 13 - "buf.build/go/protovalidate" 14 14 - "google.golang.org/protobuf/proto" 15 15 - 16 16 - millv1 "tangled.org/core/spindle/mill/proto/gen" 17 17 - ) 18 18 - 19 19 - const ( 20 20 - ProtocolVersion = 1 21 21 - // generous vs agentproto's 1 MiB. a ReserveSeat carries the raw pipeline and 22 22 - // workflow JSON, and streamed log lines can be chunky 23 23 - MaxMessageBytes = 8 * 1024 * 1024 24 24 - ) 25 25 - 26 26 - type Message = millv1.Message 27 27 - 28 28 - var validator protovalidate.Validator 29 29 - 30 30 - func init() { 31 31 - var err error 32 32 - validator, err = protovalidate.New() 33 33 - if err != nil { 34 34 - panic(fmt.Errorf("failed to initialize protovalidate validator: %w", err)) 35 35 - } 36 36 - } 37 37 - 38 38 - type Encoder struct { 39 39 - mu sync.Mutex 40 40 - w io.Writer 41 41 - } 42 42 - 43 43 - func NewEncoder(w io.Writer) *Encoder { 44 44 - return &Encoder{w: w} 45 45 - } 46 46 - 47 47 - func (e *Encoder) Encode(msg *Message) error { 48 48 - if err := validator.Validate(msg); err != nil { 49 49 - return fmt.Errorf("validate fleet message: %w", err) 50 50 - } 51 51 - 52 52 - data, err := proto.Marshal(msg) 53 53 - if err != nil { 54 54 - return fmt.Errorf("marshal fleet message: %w", err) 55 55 - } 56 56 - if len(data) > MaxMessageBytes { 57 57 - return fmt.Errorf("fleet message exceeded %d bytes", MaxMessageBytes) 58 58 - } 59 59 - 60 60 - // single write of header and payload maps to exactly one websocket binary 61 61 - // frame when the writer is a ws stream 62 62 - frame := make([]byte, 4+len(data)) 63 63 - binary.BigEndian.PutUint32(frame[:4], uint32(len(data))) 64 64 - copy(frame[4:], data) 65 65 - 66 66 - e.mu.Lock() 67 67 - defer e.mu.Unlock() 68 68 - _, err = e.w.Write(frame) 69 69 - return err 70 70 - } 71 71 - 72 72 - type Decoder struct { 73 73 - r io.Reader 74 74 - } 75 75 - 76 76 - func NewDecoder(r io.Reader) *Decoder { 77 77 - return &Decoder{r: r} 78 78 - } 79 79 - 80 80 - func (d *Decoder) Decode() (*Message, error) { 81 81 - msg := &Message{} 82 82 - var header [4]byte 83 83 - if _, err := io.ReadFull(d.r, header[:]); err != nil { 84 84 - return msg, err 85 85 - } 86 86 - 87 87 - size := binary.BigEndian.Uint32(header[:]) 88 88 - if size > MaxMessageBytes { 89 89 - return msg, fmt.Errorf("fleet message exceeded %d bytes", MaxMessageBytes) 90 90 - } 91 91 - 92 92 - data := make([]byte, size) 93 93 - if _, err := io.ReadFull(d.r, data); err != nil { 94 94 - return msg, err 95 95 - } 96 96 - if err := proto.Unmarshal(data, msg); err != nil { 97 97 - return msg, fmt.Errorf("parse fleet message: %w", err) 98 98 - } 99 99 - if err := validator.Validate(msg); err != nil { 100 100 - return msg, fmt.Errorf("validate fleet message: %w", err) 101 101 - } 102 102 - return msg, nil 103 103 - }

-227

spindle/mill/proto/protocol_test.go

Reviewed

··· 1 1 - package millproto 2 2 - 3 3 - import ( 4 4 - "bytes" 5 5 - "encoding/binary" 6 6 - "testing" 7 7 - 8 8 - millv1 "tangled.org/core/spindle/mill/proto/gen" 9 9 - ) 10 10 - 11 11 - func TestEncodeDecodeRoundTrip(t *testing.T) { 12 12 - var buf bytes.Buffer 13 13 - enc := NewEncoder(&buf) 14 14 - 15 15 - want := &Message{ 16 16 - ReserveSeat: &millv1.ReserveSeat{ 17 17 - LeaseId: "lease-1", 18 18 - TargetEngine: "microvm", 19 19 - RawWorkflowJson: `{"name":"build"}`, 20 20 - Knot: "knot.example", 21 21 - Rkey: "abc123", 22 22 - TtlSeconds: 30, 23 23 - }, 24 24 - } 25 25 - if err := enc.Encode(want); err != nil { 26 26 - t.Fatalf("Encode() error = %v", err) 27 27 - } 28 28 - 29 29 - got, err := NewDecoder(&buf).Decode() 30 30 - if err != nil { 31 31 - t.Fatalf("Decode() error = %v", err) 32 32 - } 33 33 - rs := got.GetReserveSeat() 34 34 - if rs == nil { 35 35 - t.Fatal("decoded message missing reserve_seat") 36 36 - } 37 37 - if rs.LeaseId != "lease-1" || rs.TargetEngine != "microvm" || rs.TtlSeconds != 30 { 38 38 - t.Fatalf("round-trip mismatch: %+v", rs) 39 39 - } 40 40 - } 41 41 - 42 42 - func TestDecoderRejectsOversizedMessage(t *testing.T) { 43 43 - var tooLarge bytes.Buffer 44 44 - var header [4]byte 45 45 - binary.BigEndian.PutUint32(header[:], MaxMessageBytes+1) 46 46 - tooLarge.Write(header[:]) 47 47 - 48 48 - if _, err := NewDecoder(&tooLarge).Decode(); err == nil { 49 49 - t.Fatal("expected oversized message error") 50 50 - } 51 51 - } 52 52 - 53 53 - func TestValidationRules(t *testing.T) { 54 54 - tests := []struct { 55 55 - name string 56 56 - msg *Message 57 57 - wantErr bool 58 58 - }{ 59 59 - { 60 60 - name: "valid ack message", 61 61 - msg: &Message{ 62 62 - Ack: &millv1.Ack{ 63 63 - Epoch: "inc-1", 64 64 - UpToSeqno: 5, 65 65 - }, 66 66 - }, 67 67 - wantErr: false, 68 68 - }, 69 69 - { 70 70 - name: "valid hello message", 71 71 - msg: &Message{ 72 72 - Hello: &millv1.Hello{ 73 73 - ProtocolVersion: 1, 74 74 - Arch: "amd64", 75 75 - Labels: []string{"linux"}, 76 76 - Epoch: "inc-1", 77 77 - }, 78 78 - }, 79 79 - wantErr: false, 80 80 - }, 81 81 - { 82 82 - name: "invalid message with zero payloads", 83 83 - msg: &Message{}, 84 84 - wantErr: true, 85 85 - }, 86 86 - { 87 87 - name: "invalid message with multiple payloads", 88 88 - msg: &Message{ 89 89 - Ack: &millv1.Ack{Epoch: "inc-1", UpToSeqno: 5}, 90 90 - Committed: &millv1.Committed{LeaseId: "x"}, 91 91 - }, 92 92 - wantErr: true, 93 93 - }, 94 94 - { 95 95 - name: "invalid ack message missing epoch", 96 96 - msg: &Message{ 97 97 - Ack: &millv1.Ack{ 98 98 - UpToSeqno: 5, 99 99 - }, 100 100 - }, 101 101 - wantErr: true, 102 102 - }, 103 103 - { 104 104 - name: "invalid node snapshot with zero seqno", 105 105 - msg: &Message{ 106 106 - NodeSnapshot: &millv1.NodeSnapshot{ 107 107 - Seqno: 0, 108 108 - }, 109 109 - }, 110 110 - wantErr: true, 111 111 - }, 112 112 - { 113 113 - name: "valid node snapshot with positive seqno", 114 114 - msg: &Message{ 115 115 - NodeSnapshot: &millv1.NodeSnapshot{ 116 116 - Seqno: 1, 117 117 - }, 118 118 - }, 119 119 - wantErr: false, 120 120 - }, 121 121 - { 122 122 - name: "invalid stream batch with zero seqno entry", 123 123 - msg: &Message{ 124 124 - EventBatch: &millv1.EventBatch{ 125 125 - Epoch: "inc-1", 126 126 - Events: []*millv1.Event{ 127 127 - { 128 128 - Seqno: 0, 129 129 - LeaseId: "lease-1", 130 130 - Payload: &millv1.Event_StatusEvent{ 131 131 - StatusEvent: &millv1.StatusEvent{ 132 132 - Status: millv1.NonterminalStatus_RUNNING, 133 133 - }, 134 134 - }, 135 135 - }, 136 136 - }, 137 137 - }, 138 138 - }, 139 139 - wantErr: true, 140 140 - }, 141 141 - { 142 142 - name: "invalid stream batch with empty entries", 143 143 - msg: &Message{ 144 144 - EventBatch: &millv1.EventBatch{ 145 145 - Epoch: "inc-1", 146 146 - Events: []*millv1.Event{}, 147 147 - }, 148 148 - }, 149 149 - wantErr: true, 150 150 - }, 151 151 - { 152 152 - name: "invalid reserve result with unknown enum", 153 153 - msg: &Message{ 154 154 - ReserveResult: &millv1.ReserveResult{ 155 155 - LeaseId: "lease-1", 156 156 - RejectClass: millv1.RejectClass(99), 157 157 - }, 158 158 - }, 159 159 - wantErr: true, 160 160 - }, 161 161 - { 162 162 - name: "invalid stream entry with malformed oneof (empty payload)", 163 163 - msg: &Message{ 164 164 - EventBatch: &millv1.EventBatch{ 165 165 - Epoch: "inc-1", 166 166 - Events: []*millv1.Event{ 167 167 - { 168 168 - Seqno: 1, 169 169 - LeaseId: "lease-1", 170 170 - Payload: nil, 171 171 - }, 172 172 - }, 173 173 - }, 174 174 - }, 175 175 - wantErr: true, 176 176 - }, 177 177 - { 178 178 - name: "valid stream batch status event", 179 179 - msg: &Message{ 180 180 - EventBatch: &millv1.EventBatch{ 181 181 - Epoch: "inc-1", 182 182 - Events: []*millv1.Event{ 183 183 - { 184 184 - Seqno: 1, 185 185 - LeaseId: "lease-1", 186 186 - Payload: &millv1.Event_StatusEvent{ 187 187 - StatusEvent: &millv1.StatusEvent{ 188 188 - Status: millv1.NonterminalStatus_RUNNING, 189 189 - }, 190 190 - }, 191 191 - }, 192 192 - }, 193 193 - }, 194 194 - }, 195 195 - wantErr: false, 196 196 - }, 197 197 - { 198 198 - name: "valid stream batch attempt result", 199 199 - msg: &Message{ 200 200 - EventBatch: &millv1.EventBatch{ 201 201 - Epoch: "inc-1", 202 202 - Events: []*millv1.Event{ 203 203 - { 204 204 - Seqno: 1, 205 205 - LeaseId: "lease-1", 206 206 - Payload: &millv1.Event_AttemptResult{ 207 207 - AttemptResult: &millv1.AttemptResult{ 208 208 - Status: millv1.TerminalStatus_SUCCESS, 209 209 - }, 210 210 - }, 211 211 - }, 212 212 - }, 213 213 - }, 214 214 - }, 215 215 - wantErr: false, 216 216 - }, 217 217 - } 218 218 - 219 219 - for _, tc := range tests { 220 220 - t.Run(tc.name, func(t *testing.T) { 221 221 - err := validator.Validate(tc.msg) 222 222 - if (err != nil) != tc.wantErr { 223 223 - t.Fatalf("Validate() error = %v, wantErr = %v", err, tc.wantErr) 224 224 - } 225 225 - }) 226 226 - } 227 227 - }

-187

spindle/mill/proto/spindle/mill/v1/mill.proto

Reviewed

··· 1 1 - syntax = "proto3"; 2 2 - 3 3 - package spindle.mill.v1; 4 4 - 5 5 - import "buf/validate/validate.proto"; 6 6 - 7 7 - option go_package = "tangled.org/core/spindle/mill/proto/gen;millv1"; 8 8 - 9 9 - // executor identity, sent on connect 10 10 - message Hello { 11 11 - uint32 protocol_version = 1; 12 12 - // GOARCH of the node, informational only 13 13 - string arch = 2; 14 14 - // operator-defined labels, matched against runs_on 15 15 - repeated string labels = 3; 16 16 - string epoch = 4 [(buf.validate.field).string.min_len = 1]; 17 17 - } 18 18 - 19 19 - // reconnect state for an existing epoch 20 20 - message Resume { 21 21 - string epoch = 1 [(buf.validate.field).string.min_len = 1]; 22 22 - uint64 ack_seqno = 2; 23 23 - } 24 24 - 25 25 - // per-engine state of a node 26 26 - message EngineAvailability { 27 27 - bool available = 1; 28 28 - // opaque engine-defined load metrics, higher means more loaded 29 29 - map<string, double> load = 2; 30 30 - } 31 31 - 32 32 - // full node state 33 33 - message NodeSnapshot { 34 34 - uint64 seqno = 1 [(buf.validate.field).uint64.gt = 0]; 35 35 - map<string, EngineAvailability> engines = 2; 36 36 - // every lease the executor currently holds, reserved or running 37 37 - repeated string active_lease_ids = 3; 38 38 - } 39 39 - 40 40 - // a seat reservation, carries no secrets 41 41 - message ReserveSeat { 42 42 - string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 43 43 - string target_engine = 2 [(buf.validate.field).string.min_len = 1]; 44 44 - string raw_pipeline_json = 3; 45 45 - string raw_workflow_json = 4; 46 46 - // pipeline id, the executor reconstructs the exact WorkflowId from it 47 47 - string knot = 5; 48 48 - string rkey = 6; 49 49 - uint32 ttl_seconds = 7; 50 50 - } 51 51 - 52 52 - enum RejectClass { 53 53 - REJECT_CLASS_UNSPECIFIED = 0; 54 54 - REJECT_CLASS_TRANSIENT = 1; 55 55 - REJECT_CLASS_INCOMPATIBLE = 2; 56 56 - } 57 57 - 58 58 - message ReserveResult { 59 59 - string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 60 60 - bool accepted = 2; 61 61 - string reject_reason = 3; 62 62 - RejectClass reject_class = 4 [(buf.validate.field).enum.defined_only = true]; 63 63 - } 64 64 - 65 65 - // a single unlocked secret 66 66 - message Secret { 67 67 - string key = 1; 68 68 - string value = 2; 69 69 - } 70 70 - 71 71 - // promotes a reservation to a running job and hands over the secrets 72 72 - message CommitLease { 73 73 - string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 74 74 - repeated Secret secrets = 2; 75 75 - } 76 76 - 77 77 - message Committed { 78 78 - string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 79 79 - } 80 80 - 81 81 - // drops a reservation that was never committed 82 82 - message ReleaseLease { 83 83 - string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 84 84 - } 85 85 - 86 86 - // cancels a running attempt 87 87 - message CancelAttempt { 88 88 - string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 89 89 - string reason = 2; 90 90 - } 91 91 - 92 92 - message CancelAck { 93 93 - string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 94 94 - } 95 95 - 96 96 - enum NonterminalStatus { 97 97 - NONTERMINAL_STATUS_UNSPECIFIED = 0; 98 98 - RUNNING = 1; 99 99 - } 100 100 - 101 101 - enum TerminalStatus { 102 102 - TERMINAL_STATUS_UNSPECIFIED = 0; 103 103 - SUCCESS = 1; 104 104 - FAILED = 2; 105 105 - TIMEOUT = 3; 106 106 - CANCELLED = 4; 107 107 - } 108 108 - 109 109 - message StatusEvent { 110 110 - NonterminalStatus status = 1 [(buf.validate.field).enum = { 111 111 - defined_only: true 112 112 - not_in: 0 113 113 - }]; 114 114 - string error = 2 [(buf.validate.field).string.max_bytes = 65536]; 115 115 - int64 exit_code = 3; 116 116 - } 117 117 - 118 118 - message LogArtifact { 119 119 - string ref = 1 [(buf.validate.field).string.min_len = 1]; 120 120 - string hash = 2 [(buf.validate.field).string.min_len = 1]; 121 121 - } 122 122 - 123 123 - // terminal outcome of an attempt 124 124 - message AttemptResult { 125 125 - TerminalStatus status = 1 [(buf.validate.field).enum = { 126 126 - defined_only: true 127 127 - not_in: 0 128 128 - }]; 129 129 - string error = 2 [(buf.validate.field).string.max_bytes = 65536]; 130 130 - int64 exit_code = 3; 131 131 - LogArtifact log_artifact = 4; 132 132 - } 133 133 - 134 134 - // live non-replay log frame 135 135 - message LiveLog { 136 136 - string lease_id = 1 [(buf.validate.field).string.min_len = 1]; 137 137 - bytes raw_json = 2 [(buf.validate.field).bytes.min_len = 1]; 138 138 - } 139 139 - 140 140 - // one event in the executor's stream back to the mill 141 141 - message Event { 142 142 - uint64 seqno = 1 [(buf.validate.field).uint64.gt = 0]; 143 143 - string lease_id = 2 [(buf.validate.field).string.min_len = 1]; 144 144 - 145 145 - oneof payload { 146 146 - option (buf.validate.oneof).required = true; 147 147 - StatusEvent status_event = 3; 148 148 - AttemptResult attempt_result = 4; 149 149 - } 150 150 - } 151 151 - 152 152 - // a flushed bundle of events 153 153 - message EventBatch { 154 154 - string epoch = 1 [(buf.validate.field).string.min_len = 1]; 155 155 - repeated Event events = 2 [(buf.validate.field).repeated.min_items = 1]; 156 156 - } 157 157 - 158 158 - // all events up to and including up_to_seqno are durably processed 159 159 - message Ack { 160 160 - string epoch = 1 [(buf.validate.field).string.min_len = 1]; 161 161 - uint64 up_to_seqno = 2; 162 162 - } 163 163 - 164 164 - message Message { 165 165 - option (buf.validate.message).oneof = { 166 166 - fields: [ 167 167 - "hello", "resume", "node_snapshot", "reserve_seat", "reserve_result", 168 168 - "commit_lease", "committed", "release_lease", "cancel_attempt", 169 169 - "cancel_ack", "event_batch", "ack", "live_log" 170 170 - ], 171 171 - required: true 172 172 - }; 173 173 - 174 174 - Hello hello = 1; 175 175 - Resume resume = 2; 176 176 - NodeSnapshot node_snapshot = 3; 177 177 - ReserveSeat reserve_seat = 4; 178 178 - ReserveResult reserve_result = 5; 179 179 - CommitLease commit_lease = 6; 180 180 - Committed committed = 7; 181 181 - ReleaseLease release_lease = 8; 182 182 - CancelAttempt cancel_attempt = 9; 183 183 - CancelAck cancel_ack = 10; 184 184 - EventBatch event_batch = 11; 185 185 - Ack ack = 12; 186 186 - LiveLog live_log = 13; 187 187 - }

-58

spindle/mill/proto/ws.go

Reviewed

··· 1 1 - package millproto 2 2 - 3 3 - import ( 4 4 - "io" 5 5 - "sync" 6 6 - 7 7 - "github.com/gorilla/websocket" 8 8 - ) 9 9 - 10 10 - // adapts a gorilla websocket connection to an io.ReadWriteCloser so the 11 11 - // length-prefixed fleet framing rides over it. each Encode produces exactly one 12 12 - // binary frame. the reader reassembles the byte stream across frames 13 13 - type WSStream struct { 14 14 - conn *websocket.Conn 15 15 - 16 16 - rmu sync.Mutex 17 17 - r io.Reader // current message reader, advanced as frames are consumed 18 18 - 19 19 - wmu sync.Mutex 20 20 - } 21 21 - 22 22 - func NewWSStream(conn *websocket.Conn) *WSStream { 23 23 - return &WSStream{conn: conn} 24 24 - } 25 25 - 26 26 - func (s *WSStream) Read(p []byte) (int, error) { 27 27 - s.rmu.Lock() 28 28 - defer s.rmu.Unlock() 29 29 - for { 30 30 - if s.r == nil { 31 31 - _, r, err := s.conn.NextReader() 32 32 - if err != nil { 33 33 - return 0, err 34 34 - } 35 35 - s.r = r 36 36 - } 37 37 - n, err := s.r.Read(p) 38 38 - if err == io.EOF { 39 39 - s.r = nil 40 40 - if n > 0 { 41 41 - return n, nil 42 42 - } 43 43 - continue 44 44 - } 45 45 - return n, err 46 46 - } 47 47 - } 48 48 - 49 49 - func (s *WSStream) Write(p []byte) (int, error) { 50 50 - s.wmu.Lock() 51 51 - defer s.wmu.Unlock() 52 52 - if err := s.conn.WriteMessage(websocket.BinaryMessage, p); err != nil { 53 53 - return 0, err 54 54 - } 55 55 - return len(p), nil 56 56 - } 57 57 - 58 58 - func (s *WSStream) Close() error { return s.conn.Close() }

-221

spindle/mill/restore.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "fmt" 5 5 - "tangled.org/core/spindle/db" 6 6 - millproto "tangled.org/core/spindle/mill/proto" 7 7 - millv1 "tangled.org/core/spindle/mill/proto/gen" 8 8 - "tangled.org/core/spindle/models" 9 9 - "time" 10 10 - ) 11 11 - 12 12 - const ( 13 13 - leaseRowReserved = "reserved" 14 14 - leaseRowRunning = "running" 15 15 - ) 16 16 - 17 17 - func (m *Mill) persistLease(lease *RemoteLease, state string) error { 18 18 - if m.db == nil { 19 19 - return nil 20 20 - } 21 21 - return m.db.SaveMillLease(db.MillLease{ 22 22 - LeaseID: lease.id, 23 23 - NodeID: lease.nodeID, 24 24 - Epoch: lease.epoch, 25 25 - Engine: lease.engine, 26 26 - Knot: lease.wid.Knot, 27 27 - Rkey: lease.wid.Rkey, 28 28 - Workflow: lease.wid.Name, 29 29 - State: state, 30 30 - }) 31 31 - } 32 32 - 33 33 - func (m *Mill) RestoreState() error { 34 34 - if m.db == nil { 35 35 - return nil 36 36 - } 37 37 - cursors, err := m.db.ListExecutorCursors() 38 38 - if err != nil { 39 39 - return err 40 40 - } 41 41 - rows, err := m.db.ListMillLeases() 42 42 - if err != nil { 43 43 - return err 44 44 - } 45 45 - 46 46 - m.mu.Lock() 47 47 - for _, c := range cursors { 48 48 - m.nodeSeqno[c.NodeID+"/"+c.Epoch] = c.AckedSeqno 49 49 - } 50 50 - for _, r := range rows { 51 51 - lease := newLease(r.LeaseID, r.NodeID, r.Epoch, r.Engine) 52 52 - lease.wid = models.WorkflowId{ 53 53 - PipelineId: models.PipelineId{Knot: r.Knot, Rkey: r.Rkey}, 54 54 - Name: r.Workflow, 55 55 - } 56 56 - // restored leases start as orphans, an executor must reclaim it via 57 57 - // its first snapshot, or the sweep will fail it 58 58 - lease.orphaned = true 59 59 - lease.claimed = false 60 60 - if r.State == leaseRowRunning { 61 61 - lease.state = leaseRunning 62 62 - } 63 63 - m.leases[r.LeaseID] = lease 64 64 - } 65 65 - restored := len(rows) 66 66 - m.mu.Unlock() 67 67 - 68 68 - if restored > 0 { 69 69 - m.l.Info("restored mill leases from previous run", "leases", restored, "cursors", len(cursors)) 70 70 - // executors get a grace window to reconnect and claim their leases 71 71 - time.AfterFunc(m.cfg.ReconnectGrace, m.sweepUnclaimedOrphans) 72 72 - } 73 73 - return nil 74 74 - } 75 75 - func (m *Mill) sweepUnclaimedOrphans() { 76 76 - m.mu.Lock() 77 77 - var unclaimed []*RemoteLease 78 78 - for _, lease := range m.leases { 79 79 - if !lease.orphaned { 80 80 - continue 81 81 - } 82 82 - if lease.claimed { 83 83 - continue 84 84 - } 85 85 - if lease.getState() == leaseDone { 86 86 - continue 87 87 - } 88 88 - if sess := m.sessions[lease.nodeID]; sess == nil || sess.disconnected { 89 89 - unclaimed = append(unclaimed, lease) 90 90 - } 91 91 - } 92 92 - m.mu.Unlock() 93 93 - 94 94 - retry := false 95 95 - reason := "executor did not reconnect after mill restart" 96 96 - for _, lease := range unclaimed { 97 97 - m.l.Warn("failing unclaimed restored lease", "lease", lease.id, "node", lease.nodeID) 98 98 - if err := m.finishOrphan(lease, string(models.StatusKindFailed), &reason, nil); err != nil { 99 99 - m.l.Error("finish unclaimed restored lease", "lease", lease.id, "err", err) 100 100 - retry = true 101 101 - } 102 102 - } 103 103 - if retry { 104 104 - time.AfterFunc(5*time.Second, m.sweepUnclaimedOrphans) 105 105 - } 106 106 - } 107 107 - 108 108 - func (m *Mill) reconcileLeases(sess *millSession, activeLeaseIDs []string) error { 109 109 - active := make(map[string]struct{}, len(activeLeaseIDs)) 110 110 - for _, id := range activeLeaseIDs { 111 111 - active[id] = struct{}{} 112 112 - } 113 113 - 114 114 - known := make(map[string]struct{}) 115 115 - m.mu.Lock() 116 116 - var gone []*RemoteLease 117 117 - for _, lease := range m.leases { 118 118 - if lease.nodeID == sess.nodeID { 119 119 - known[lease.id] = struct{}{} 120 120 - if lease.epoch != sess.epoch { 121 121 - gone = append(gone, lease) 122 122 - } else if _, ok := active[lease.id]; !ok { 123 123 - gone = append(gone, lease) 124 124 - } 125 125 - } 126 126 - } 127 127 - for _, lease := range m.reservations { 128 128 - if lease.nodeID == sess.nodeID && lease.epoch == sess.epoch { 129 129 - known[lease.id] = struct{}{} 130 130 - } 131 131 - } 132 132 - m.mu.Unlock() 133 133 - var unknown []string 134 134 - for id := range active { 135 135 - if _, ok := known[id]; !ok { 136 136 - unknown = append(unknown, id) 137 137 - } 138 138 - } 139 139 - 140 140 - for _, lease := range gone { 141 141 - status := string(models.StatusKindFailed) 142 142 - reason := "executor no longer holds lease" 143 143 - if cancelled, cancelReason := lease.cancelRequested(); cancelled { 144 144 - status = string(models.StatusKindCancelled) 145 145 - reason = cancelReason 146 146 - } 147 147 - m.l.Warn("finishing reconciled lease", "lease", lease.id, "node", sess.nodeID, "leaseInc", lease.epoch, "sessInc", sess.epoch) 148 148 - if lease.orphaned { 149 149 - if err := m.finishOrphan(lease, status, &reason, nil); err != nil { 150 150 - return err 151 151 - } 152 152 - } else if err := m.finishLiveLease(lease, status, reason); err != nil { 153 153 - return err 154 154 - } 155 155 - } 156 156 - for _, id := range unknown { 157 157 - if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{LeaseId: id, Reason: "lease is not owned by this mill"}}); err != nil { 158 158 - return fmt.Errorf("cancel unknown executor lease %q: %w", id, err) 159 159 - } 160 160 - } 161 161 - return nil 162 162 - } 163 163 - 164 164 - func (m *Mill) completeLeaseRow(lease *RemoteLease, status string, errMsg *string, exitCode *int64) error { 165 165 - if m.db == nil { 166 166 - return nil 167 167 - } 168 168 - return m.db.CompleteMillLease( 169 169 - lease.id, 170 170 - string(lease.wid.PipelineId.AtUri()), 171 171 - lease.wid.Name, 172 172 - status, 173 173 - errMsg, 174 174 - exitCode, 175 175 - m.n, 176 176 - ) 177 177 - } 178 178 - 179 179 - func (m *Mill) finishLiveLease(lease *RemoteLease, status, reason string) error { 180 180 - lease.finishMu.Lock() 181 181 - defer lease.finishMu.Unlock() 182 182 - if lease.getState() == leaseDone { 183 183 - return nil 184 184 - } 185 185 - if err := m.completeLeaseRow(lease, status, &reason, nil); err != nil { 186 186 - return err 187 187 - } 188 188 - lease.deliverTerminal(&millv1.AttemptResult{ 189 189 - Status: mapTerminalStatusString(status), 190 190 - Error: reason, 191 191 - }) 192 192 - return m.cleanupLeaseLocked(lease) 193 193 - } 194 194 - 195 195 - func (m *Mill) finishOrphan(lease *RemoteLease, status string, errMsg *string, exitCode *int64) error { 196 196 - lease.finishMu.Lock() 197 197 - defer lease.finishMu.Unlock() 198 198 - if lease.getState() == leaseDone { 199 199 - return nil 200 200 - } 201 201 - if err := m.completeLeaseRow(lease, status, errMsg, exitCode); err != nil { 202 202 - return err 203 203 - } 204 204 - lease.markDone() 205 205 - return m.cleanupLeaseLocked(lease) 206 206 - } 207 207 - 208 208 - func mapTerminalStatusString(s string) millv1.TerminalStatus { 209 209 - switch s { 210 210 - case "success": 211 211 - return millv1.TerminalStatus_SUCCESS 212 212 - case "failed": 213 213 - return millv1.TerminalStatus_FAILED 214 214 - case "timeout": 215 215 - return millv1.TerminalStatus_TIMEOUT 216 216 - case "cancelled": 217 217 - return millv1.TerminalStatus_CANCELLED 218 218 - default: 219 219 - return millv1.TerminalStatus_TERMINAL_STATUS_UNSPECIFIED 220 220 - } 221 221 - }

-386

spindle/mill/restore_test.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "path/filepath" 6 6 - "testing" 7 7 - "time" 8 8 - 9 9 - "tangled.org/core/notifier" 10 10 - "tangled.org/core/spindle/db" 11 11 - millproto "tangled.org/core/spindle/mill/proto" 12 12 - millv1 "tangled.org/core/spindle/mill/proto/gen" 13 13 - "tangled.org/core/spindle/models" 14 14 - ) 15 15 - 16 16 - func restoreTestMill(t *testing.T, cfg Config) (*Mill, *db.DB) { 17 17 - t.Helper() 18 18 - bdb, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "mill.db")) 19 19 - if err != nil { 20 20 - t.Fatalf("db.Make: %v", err) 21 21 - } 22 22 - t.Cleanup(func() { bdb.Close() }) 23 23 - n := notifier.New() 24 24 - m := New(discardLogger(), cfg) 25 25 - m.Attach(bdb, &n) 26 26 - return m, bdb 27 27 - } 28 28 - 29 29 - func restoredMill(t *testing.T, bdb *db.DB, cfg Config) *Mill { 30 30 - t.Helper() 31 31 - n := notifier.New() 32 32 - m := New(discardLogger(), cfg) 33 33 - m.Attach(bdb, &n) 34 34 - if err := m.RestoreState(); err != nil { 35 35 - t.Fatalf("RestoreState: %v", err) 36 36 - } 37 37 - return m 38 38 - } 39 39 - 40 40 - func TestRestoreStateRebuildsLeasesAndCursors(t *testing.T) { 41 41 - _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 42 42 - 43 43 - if err := bdb.SaveMillLease(db.MillLease{ 44 44 - LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 45 45 - Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, 46 46 - }); err != nil { 47 47 - t.Fatalf("SaveMillLease: %v", err) 48 48 - } 49 49 - if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 7) }); err != nil { 50 50 - t.Fatalf("AdvanceCursor: %v", err) 51 51 - } 52 52 - 53 53 - m2 := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 54 54 - 55 55 - m2.mu.Lock() 56 56 - lease := m2.leases["lease-1"] 57 57 - seqno := m2.nodeSeqno["node-1/inc-1"] 58 58 - m2.mu.Unlock() 59 59 - 60 60 - if lease == nil { 61 61 - t.Fatal("restored mill has no lease-1") 62 62 - } 63 63 - if !lease.orphaned { 64 64 - t.Fatal("restored lease is not orphaned; a terminal would be delivered to a waiter that does not exist") 65 65 - } 66 66 - if lease.getState() != leaseRunning { 67 67 - t.Fatalf("restored lease state = %v, want leaseRunning", lease.getState()) 68 68 - } 69 69 - wantWid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"} 70 70 - if lease.wid != wantWid { 71 71 - t.Fatalf("restored lease wid = %+v, want %+v", lease.wid, wantWid) 72 72 - } 73 73 - if seqno != 7 { 74 74 - t.Fatalf("restored cursor = %d, want 7", seqno) 75 75 - } 76 76 - } 77 77 - 78 78 - func TestOrphanTerminalAuthorsStatusRow(t *testing.T) { 79 79 - _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 80 80 - if err := bdb.SaveMillLease(db.MillLease{ 81 81 - LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 82 82 - Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, 83 83 - }); err != nil { 84 84 - t.Fatalf("SaveMillLease: %v", err) 85 85 - } 86 86 - if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 3) }); err != nil { 87 87 - t.Fatalf("AdvanceCursor: %v", err) 88 88 - } 89 89 - 90 90 - m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 91 91 - 92 92 - sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 93 93 - resume, ok := m.attachSession(sess) 94 94 - if !ok { 95 95 - t.Fatal("attachSession rejected the reconnecting executor") 96 96 - } 97 97 - if resume != 3 { 98 98 - t.Fatalf("attachSession resume seqno = %d, want restored cursor 3", resume) 99 99 - } 100 100 - 101 101 - _ = m.onEventBatch(sess, &millv1.EventBatch{ 102 102 - Epoch: sess.epoch, 103 103 - Events: []*millv1.Event{ 104 104 - { 105 105 - Seqno: 4, 106 106 - LeaseId: "lease-1", 107 107 - Payload: &millv1.Event_AttemptResult{ 108 108 - AttemptResult: &millv1.AttemptResult{ 109 109 - Status: millv1.TerminalStatus_SUCCESS, 110 110 - }, 111 111 - }, 112 112 - }, 113 113 - }, 114 114 - }) 115 115 - 116 116 - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"} 117 117 - st, err := bdb.GetStatus(wid) 118 118 - if err != nil { 119 119 - t.Fatalf("GetStatus after orphan terminal: %v", err) 120 120 - } 121 121 - if st.Status != string(models.StatusKindSuccess) { 122 122 - t.Fatalf("orphan terminal authored status %q, want success", st.Status) 123 123 - } 124 124 - 125 125 - m.mu.Lock() 126 126 - _, still := m.leases["lease-1"] 127 127 - m.mu.Unlock() 128 128 - if still { 129 129 - t.Fatal("finished orphan still in the lease map") 130 130 - } 131 131 - if rows, _ := bdb.ListMillLeases(); len(rows) != 0 { 132 132 - t.Fatalf("finished orphan still persisted: %+v", rows) 133 133 - } 134 134 - } 135 135 - 136 136 - func TestSnapshotReconciliationFailsDroppedOrphans(t *testing.T) { 137 137 - _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 138 138 - for _, l := range []db.MillLease{ 139 139 - {LeaseID: "lease-kept", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", Knot: "k", Rkey: "r1", Workflow: "w", State: leaseRowRunning}, 140 140 - {LeaseID: "lease-gone", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", Knot: "k", Rkey: "r2", Workflow: "w", State: leaseRowRunning}, 141 141 - } { 142 142 - if err := bdb.SaveMillLease(l); err != nil { 143 143 - t.Fatalf("SaveMillLease(%s): %v", l.LeaseID, err) 144 144 - } 145 145 - } 146 146 - 147 147 - m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 148 148 - 149 149 - sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 150 150 - m.attachSession(sess) 151 151 - 152 152 - m.onSnapshot(sess, &millv1.NodeSnapshot{ 153 153 - Seqno: 1, 154 154 - ActiveLeaseIds: []string{"lease-kept"}, 155 155 - }) 156 156 - 157 157 - m.mu.Lock() 158 158 - _, kept := m.leases["lease-kept"] 159 159 - _, gone := m.leases["lease-gone"] 160 160 - m.mu.Unlock() 161 161 - if !kept { 162 162 - t.Fatal("reconciliation dropped a lease the executor still holds") 163 163 - } 164 164 - if gone { 165 165 - t.Fatal("reconciliation kept a lease the executor no longer holds") 166 166 - } 167 167 - 168 168 - st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r2"}, Name: "w"}) 169 169 - if err != nil { 170 170 - t.Fatalf("GetStatus for dropped orphan: %v", err) 171 171 - } 172 172 - if st.Status != string(models.StatusKindFailed) { 173 173 - t.Fatalf("dropped orphan authored status %q, want failed", st.Status) 174 174 - } 175 175 - } 176 176 - func TestSnapshotReconciliationPreservesRequestedCancellation(t *testing.T) { 177 177 - m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 178 178 - lease := newLease("lease-1", "node-1", "inc-old", "dummy") 179 179 - lease.wid = models.WorkflowId{ 180 180 - PipelineId: models.PipelineId{Knot: "k", Rkey: "r1"}, 181 181 - Name: "w", 182 182 - } 183 183 - lease.setState(leaseRunning) 184 184 - lease.requestCancel("workflow destroyed") 185 185 - if err := m.persistLease(lease, leaseRowRunning); err != nil { 186 186 - t.Fatalf("persistLease: %v", err) 187 187 - } 188 188 - m.mu.Lock() 189 189 - m.leases[lease.id] = lease 190 190 - m.mu.Unlock() 191 191 - 192 192 - sess := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) 193 193 - m.attachSession(sess) 194 194 - if err := m.onSnapshot(sess, &millv1.NodeSnapshot{Seqno: 1}); err != nil { 195 195 - t.Fatalf("onSnapshot: %v", err) 196 196 - } 197 197 - 198 198 - st, err := bdb.GetStatus(lease.wid) 199 199 - if err != nil { 200 200 - t.Fatalf("GetStatus: %v", err) 201 201 - } 202 202 - if st.Status != string(models.StatusKindCancelled) { 203 203 - t.Fatalf("reconciled status = %q, want cancelled", st.Status) 204 204 - } 205 205 - } 206 206 - 207 207 - func TestSnapshotReconciliationCancelsUnknownExecutorLease(t *testing.T) { 208 208 - m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 209 209 - cancelled := make(chan string, 1) 210 210 - sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { 211 211 - if cancel := msg.GetCancelAttempt(); cancel != nil { 212 212 - cancelled <- cancel.GetLeaseId() 213 213 - } 214 214 - return nil 215 215 - }), discardLogger()) 216 216 - m.attachSession(sess) 217 217 - 218 218 - if err := m.onSnapshot(sess, &millv1.NodeSnapshot{ 219 219 - Seqno: 1, 220 220 - ActiveLeaseIds: []string{"executor-only"}, 221 221 - }); err != nil { 222 222 - t.Fatalf("onSnapshot: %v", err) 223 223 - } 224 224 - select { 225 225 - case id := <-cancelled: 226 226 - if id != "executor-only" { 227 227 - t.Fatalf("cancelled lease = %q, want executor-only", id) 228 228 - } 229 229 - default: 230 230 - t.Fatal("snapshot reconciliation left an executor-only lease running") 231 231 - } 232 232 - } 233 233 - 234 234 - func TestSweepFailsOrphansOfAbsentExecutors(t *testing.T) { 235 235 - _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 236 236 - if err := bdb.SaveMillLease(db.MillLease{ 237 237 - LeaseID: "lease-1", NodeID: "node-absent", Epoch: "inc-absent", Engine: "dummy", 238 238 - Knot: "k", Rkey: "r1", Workflow: "w", State: leaseRowReserved, 239 239 - }); err != nil { 240 240 - t.Fatalf("SaveMillLease: %v", err) 241 241 - } 242 242 - 243 243 - m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 244 244 - m.sweepUnclaimedOrphans() 245 245 - 246 246 - m.mu.Lock() 247 247 - _, still := m.leases["lease-1"] 248 248 - m.mu.Unlock() 249 249 - if still { 250 250 - t.Fatal("sweep kept an orphan whose executor never reconnected") 251 251 - } 252 252 - st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r1"}, Name: "w"}) 253 253 - if err != nil { 254 254 - t.Fatalf("GetStatus after sweep: %v", err) 255 255 - } 256 256 - if st.Status != string(models.StatusKindFailed) { 257 257 - t.Fatalf("sweep authored status %q, want failed", st.Status) 258 258 - } 259 259 - if rows, _ := bdb.ListMillLeases(); len(rows) != 0 { 260 260 - t.Fatalf("swept orphan still persisted: %+v", rows) 261 261 - } 262 262 - } 263 263 - 264 264 - func TestAckSeqnoPersistsCursor(t *testing.T) { 265 265 - m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 266 266 - sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 267 267 - m.attachSession(sess) 268 268 - 269 269 - owned := newLease("lease-1", "node-1", "inc-1", "dummy") 270 270 - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} 271 271 - m.mu.Lock() 272 272 - m.leases[owned.id] = owned 273 273 - m.mu.Unlock() 274 274 - 275 275 - err := m.onEventBatch(sess, &millv1.EventBatch{ 276 276 - Epoch: sess.epoch, 277 277 - Events: []*millv1.Event{ 278 278 - { 279 279 - Seqno: 1, 280 280 - LeaseId: owned.id, 281 281 - Payload: &millv1.Event_StatusEvent{ 282 282 - StatusEvent: &millv1.StatusEvent{ 283 283 - Status: millv1.NonterminalStatus_RUNNING, 284 284 - }, 285 285 - }, 286 286 - }, 287 287 - }, 288 288 - }) 289 289 - if err != nil { 290 290 - t.Fatalf("onEventBatch: %v", err) 291 291 - } 292 292 - 293 293 - cursors, err := bdb.ListExecutorCursors() 294 294 - if err != nil { 295 295 - t.Fatalf("ListExecutorCursors: %v", err) 296 296 - } 297 297 - if len(cursors) != 1 || cursors[0].AckedSeqno != 1 { 298 298 - t.Fatalf("persisted cursor = %+v, want seqno 1", cursors) 299 299 - } 300 300 - } 301 301 - 302 302 - func TestOrphanTerminalFailureKeepsLeaseAndSeqnoRetryable(t *testing.T) { 303 303 - _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) 304 304 - if err := bdb.SaveMillLease(db.MillLease{ 305 305 - LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", 306 306 - Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, 307 307 - }); err != nil { 308 308 - t.Fatalf("SaveMillLease: %v", err) 309 309 - } 310 310 - if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 3) }); err != nil { 311 311 - t.Fatalf("AdvanceCursor: %v", err) 312 312 - } 313 313 - if _, err := bdb.Exec(` 314 314 - create trigger reject_orphan_lease_delete 315 315 - before delete on mill_leases 316 316 - begin 317 317 - select raise(abort, 'forced delete failure'); 318 318 - end 319 319 - `); err != nil { 320 320 - t.Fatalf("create failure trigger: %v", err) 321 321 - } 322 322 - 323 323 - m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) 324 324 - sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) 325 325 - if _, ok := m.attachSession(sess); !ok { 326 326 - t.Fatal("attachSession rejected reconnect") 327 327 - } 328 328 - batch := &millv1.EventBatch{ 329 329 - Epoch: sess.epoch, 330 330 - Events: []*millv1.Event{ 331 331 - { 332 332 - Seqno: 4, 333 333 - LeaseId: "lease-1", 334 334 - Payload: &millv1.Event_AttemptResult{ 335 335 - AttemptResult: &millv1.AttemptResult{ 336 336 - Status: millv1.TerminalStatus_SUCCESS, 337 337 - }, 338 338 - }, 339 339 - }, 340 340 - }, 341 341 - } 342 342 - if err := m.onEventBatch(sess, batch); err == nil { 343 343 - t.Fatal("orphan terminal stream succeeded despite forced transaction failure") 344 344 - } 345 345 - 346 346 - m.mu.Lock() 347 347 - lease := m.leases["lease-1"] 348 348 - seqno := m.nodeSeqno["node-1/inc-1"] 349 349 - m.mu.Unlock() 350 350 - if lease == nil || lease.getState() == leaseDone { 351 351 - t.Fatal("failed orphan completion made the in-memory lease unretryable") 352 352 - } 353 353 - if seqno != 3 { 354 354 - t.Fatalf("in-memory stream seqno = %d, want 3", seqno) 355 355 - } 356 356 - if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 1 { 357 357 - t.Fatalf("durable leases after transaction rollback = %+v, err = %v; want retained lease", rows, err) 358 358 - } 359 359 - var events int 360 360 - if err := bdb.QueryRow(`select count(*) from events`).Scan(&events); err != nil { 361 361 - t.Fatalf("count events: %v", err) 362 362 - } 363 363 - if events != 0 { 364 364 - t.Fatalf("terminal events after transaction rollback = %d, want 0", events) 365 365 - } 366 366 - 367 367 - if _, err := bdb.Exec(`drop trigger reject_orphan_lease_delete`); err != nil { 368 368 - t.Fatalf("drop failure trigger: %v", err) 369 369 - } 370 370 - if err := m.onEventBatch(sess, batch); err != nil { 371 371 - t.Fatalf("retry orphan terminal: %v", err) 372 372 - } 373 373 - m.mu.Lock() 374 374 - _, still := m.leases["lease-1"] 375 375 - seqno = m.nodeSeqno["node-1/inc-1"] 376 376 - m.mu.Unlock() 377 377 - if still { 378 378 - t.Fatal("successful orphan completion retained in-memory lease") 379 379 - } 380 380 - if seqno != 4 { 381 381 - t.Fatalf("in-memory stream seqno after retry = %d, want 4", seqno) 382 382 - } 383 383 - if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 0 { 384 384 - t.Fatalf("durable leases after successful retry = %+v, err = %v; want none", rows, err) 385 385 - } 386 386 - }

-160

spindle/mill/session.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "errors" 6 6 - "log/slog" 7 7 - "sync" 8 8 - "time" 9 9 - 10 10 - millproto "tangled.org/core/spindle/mill/proto" 11 11 - millv1 "tangled.org/core/spindle/mill/proto/gen" 12 12 - ) 13 13 - 14 14 - var errSessionClosed = errors.New("mill: executor session closed") 15 15 - 16 16 - // one live websocket to an executor. many leases and async streams share 17 17 - // it, so one reader goroutine demuxes by message type and correlates by 18 18 - // lease ID. never hold locks across decodes 19 19 - type millSession struct { 20 20 - nodeID string 21 21 - epoch string 22 22 - labels []string 23 23 - enc messageEncoder 24 24 - l *slog.Logger 25 25 - closeTransport func() error 26 26 - 27 27 - // snapshot, disconnected and graceTimer are guarded by Mill.mu, the 28 28 - // fleet ranks across sessions under its own lock 29 29 - snapshot *millv1.NodeSnapshot 30 30 - disconnected bool 31 31 - graceTimer *time.Timer 32 32 - lastSeen time.Time 33 33 - 34 34 - mu sync.Mutex 35 35 - pending map[string]chan *millproto.Message // maps lease ID to response waiter 36 36 - 37 37 - closeOnce sync.Once 38 38 - closed chan struct{} 39 39 - } 40 40 - 41 41 - type messageEncoder interface { 42 42 - Encode(*millproto.Message) error 43 43 - } 44 44 - 45 45 - func newSession(nodeID string, epoch string, labels []string, enc messageEncoder, l *slog.Logger) *millSession { 46 46 - return &millSession{ 47 47 - nodeID: nodeID, 48 48 - epoch: epoch, 49 49 - labels: labels, 50 50 - enc: enc, 51 51 - l: l, 52 52 - pending: make(map[string]chan *millproto.Message), 53 53 - closed: make(chan struct{}), 54 54 - lastSeen: time.Now(), 55 55 - } 56 56 - } 57 57 - 58 58 - // caller holds Mill.mu 59 59 - func (s *millSession) live(grace time.Duration) bool { 60 60 - return !s.disconnected && time.Since(s.lastSeen) <= grace 61 61 - } 62 62 - 63 63 - func (s *millSession) send(msg *millproto.Message) error { 64 64 - return s.enc.Encode(msg) 65 65 - } 66 66 - 67 67 - func (s *millSession) close() { 68 68 - s.closeOnce.Do(func() { 69 69 - close(s.closed) 70 70 - if s.closeTransport != nil { 71 71 - _ = s.closeTransport() 72 72 - } 73 73 - }) 74 74 - } 75 75 - 76 76 - // one-shot waiter for the next response on the lease, cancel unregisters it 77 77 - func (s *millSession) await(leaseID string) (<-chan *millproto.Message, func()) { 78 78 - ch := make(chan *millproto.Message, 1) 79 79 - s.mu.Lock() 80 80 - s.pending[leaseID] = ch 81 81 - s.mu.Unlock() 82 82 - return ch, func() { 83 83 - s.mu.Lock() 84 84 - if s.pending[leaseID] == ch { 85 85 - delete(s.pending, leaseID) 86 86 - } 87 87 - s.mu.Unlock() 88 88 - } 89 89 - } 90 90 - 91 91 - func (s *millSession) deliver(leaseID string, msg *millproto.Message) { 92 92 - s.mu.Lock() 93 93 - ch := s.pending[leaseID] 94 94 - delete(s.pending, leaseID) 95 95 - s.mu.Unlock() 96 96 - if ch != nil { 97 97 - select { 98 98 - case ch <- msg: 99 99 - default: 100 100 - } 101 101 - } 102 102 - } 103 103 - 104 104 - // sends a message and waits for its response, respecting ctx and session closure 105 105 - func (s *millSession) request(ctx context.Context, leaseID string, msg *millproto.Message) (*millproto.Message, error) { 106 106 - if err := ctx.Err(); err != nil { 107 107 - return nil, err 108 108 - } 109 109 - ch, cancel := s.await(leaseID) 110 110 - defer cancel() 111 111 - 112 112 - if err := s.send(msg); err != nil { 113 113 - return nil, err 114 114 - } 115 115 - 116 116 - select { 117 117 - case resp := <-ch: 118 118 - return resp, nil 119 119 - case <-ctx.Done(): 120 120 - return nil, ctx.Err() 121 121 - case <-s.closed: 122 122 - return nil, errSessionClosed 123 123 - } 124 124 - } 125 125 - 126 126 - // demuxes frames until the decoder errors (connection gone) 127 127 - func (s *millSession) readLoop(m *Mill, dec *millproto.Decoder) error { 128 128 - for { 129 129 - msg, err := dec.Decode() 130 130 - if err != nil { 131 131 - return err 132 132 - } 133 133 - if err := s.dispatch(m, msg); err != nil { 134 134 - return err 135 135 - } 136 136 - } 137 137 - } 138 138 - 139 139 - func (s *millSession) dispatch(m *Mill, msg *millproto.Message) error { 140 140 - if !m.touchSession(s) { 141 141 - return errSessionClosed 142 142 - } 143 143 - switch { 144 144 - case msg.GetNodeSnapshot() != nil: 145 145 - return m.onSnapshot(s, msg.GetNodeSnapshot()) 146 146 - case msg.GetReserveResult() != nil: 147 147 - s.deliver(msg.GetReserveResult().GetLeaseId(), msg) 148 148 - case msg.GetCommitted() != nil: 149 149 - s.deliver(msg.GetCommitted().GetLeaseId(), msg) 150 150 - case msg.GetEventBatch() != nil: 151 151 - return m.onEventBatch(s, msg.GetEventBatch()) 152 152 - case msg.GetCancelAck() != nil: 153 153 - m.onCancelAck(s, msg.GetCancelAck()) 154 154 - case msg.GetLiveLog() != nil: 155 155 - return m.onLiveLog(s, msg.GetLiveLog()) 156 156 - default: 157 157 - s.l.Warn("session received unexpected message", "node", s.nodeID) 158 158 - } 159 159 - return nil 160 160 - }

-23

spindle/mill/token.go

Reviewed

··· 1 1 - package mill 2 2 - 3 3 - import ( 4 4 - "crypto/rand" 5 5 - "crypto/sha256" 6 6 - "encoding/base64" 7 7 - "encoding/hex" 8 8 - ) 9 9 - 10 10 - func GenerateToken() (string, error) { 11 11 - var b [32]byte 12 12 - if _, err := rand.Read(b[:]); err != nil { 13 13 - return "", err 14 14 - } 15 15 - return base64.RawURLEncoding.EncodeToString(b[:]), nil 16 16 - } 17 17 - 18 18 - // hashes the token for persistence and comparison. a database leak never 19 19 - // exposes a usable token, and lookup avoids a secret-dependent comparison 20 20 - func HashToken(token string) string { 21 21 - sum := sha256.Sum256([]byte(token)) 22 22 - return hex.EncodeToString(sum[:]) 23 23 - }

+153 -221

spindle/server.go

Reviewed

··· 39 39 "tangled.org/core/spindle/engines/dummy" 40 40 "tangled.org/core/spindle/engines/nixery" 41 41 "tangled.org/core/spindle/git" 42 42 - "tangled.org/core/spindle/mill" 43 43 - "tangled.org/core/spindle/mill/executor" 44 42 "tangled.org/core/spindle/models" 45 43 "tangled.org/core/spindle/secrets" 46 44 "tangled.org/core/spindle/xrpc" ··· 65 63 l *slog.Logger 66 64 n *notifier.Notifier 67 65 engs map[string]models.Engine 68 68 - jobWake chan struct{} 69 66 cfg *config.Config 70 67 ks *eventconsumer.Consumer 71 68 res *idresolver.Resolver ··· 74 71 motd []byte 75 72 motdMu sync.RWMutex 76 73 rootCtx context.Context 77 77 - store artifactstore.Store 74 74 + jobWake chan struct{} 78 75 stores *artifactstore.Stores 79 76 reader artifactstore.Reader 80 80 - // set only when this spindle hosts the mill or joins one as an executor 81 81 - mill *mill.Mill 82 82 - exec *executor.Executor 83 77 } 84 78 85 79 // New creates a new Spindle server with the provided configuration and engines. 86 80 func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { 87 81 logger := log.FromContext(ctx) 88 88 - n := notifier.New() 89 82 90 90 - if cfg.Role == config.RoleExecutor { 91 91 - if err := cleanupOrphanRepos(ctx, d, logger); err != nil { 92 92 - return nil, fmt.Errorf("failed to run startup cleanup: %w", err) 93 93 - } 94 94 - } else if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { 95 95 - return nil, fmt.Errorf("failed to run startup migrations: %w", err) 96 96 - } 97 97 - 98 98 - spindle := &Spindle{ 99 99 - db: d, 100 100 - l: logger, 101 101 - n: &n, 102 102 - engs: engines, 103 103 - cfg: cfg, 104 104 - motd: defaultMotd, 105 105 - rootCtx: ctx, 106 106 - jobWake: make(chan struct{}, 1), 107 107 - } 108 108 - diskFallback := "" 109 109 - if cfg.Role == config.RoleStandalone { 110 110 - diskFallback = cfg.Server.LogDir 111 111 - if cfg.ArtifactStores.Disk.Dir == "" { 112 112 - logger.Warn("using SPINDLE_SERVER_LOG_DIR as the implicit disk artifact store; configure SPINDLE_ARTIFACT_STORES_DISK_DIR explicitly") 113 113 - } 114 114 - } 115 115 - stores, err := artifactstore.NewStores(cfg.ArtifactStores, diskFallback, cfg.LegacyS3.LogBucket) 116 116 - if err != nil { 117 117 - return nil, fmt.Errorf("failed to setup artifact stores: %w", err) 118 118 - } 119 119 - spindle.stores = stores 120 120 - if cfg.LegacyS3.LogBucket != "" { 121 121 - logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") 122 122 - } 123 123 - if cfg.Role == config.RoleStandalone { 124 124 - spindle.reader = stores 125 125 - } else { 126 126 - name := cfg.Mill.ArtifactStore 127 127 - if name == "" { 128 128 - names := stores.Names() 129 129 - if len(names) != 1 { 130 130 - return nil, fmt.Errorf("%s requires SPINDLE_MILL_ARTIFACT_STORE when %d artifact stores are configured", cfg.Role, len(names)) 131 131 - } 132 132 - name = names[0] 133 133 - logger.Warn("SPINDLE_MILL_ARTIFACT_STORE is not set; inferred the only configured store", "store", name) 134 134 - } 135 135 - store, ok := stores.Store(name) 136 136 - if !ok { 137 137 - return nil, fmt.Errorf("SPINDLE_MILL_ARTIFACT_STORE=%q is not configured", name) 138 138 - } 139 139 - spindle.store = store 140 140 - spindle.reader = store 141 141 - } 142 142 - if cfg.Role == config.RoleExecutor { 143 143 - return spindle, nil 144 144 - } 145 145 - 146 83 e, err := rbac.NewEnforcer(cfg.Server.DBPath) 147 84 if err != nil { 148 85 return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) 149 86 } 150 87 e.E.EnableAutoSave(true) 151 151 - spindle.e = e 152 88 89 89 + n := notifier.New() 90 90 + 91 91 + var vault secrets.Manager 153 92 switch cfg.Server.Secrets.Provider { 154 93 case "openbao": 155 94 if cfg.Server.Secrets.OpenBao.ProxyAddr == "" { 156 95 return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider") 157 96 } 158 158 - spindle.vault, err = secrets.NewOpenBaoManager( 97 97 + vault, err = secrets.NewOpenBaoManager( 159 98 cfg.Server.Secrets.OpenBao.ProxyAddr, 160 99 logger, 161 100 secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount), ··· 165 104 } 166 105 logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount) 167 106 case "sqlite", "": 168 168 - spindle.vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) 107 107 + vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) 169 108 if err != nil { 170 109 return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err) 171 110 } ··· 174 113 return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) 175 114 } 176 115 116 116 + if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { 117 117 + return nil, fmt.Errorf("failed to run startup migrations: %w", err) 118 118 + } 119 119 + 177 120 collections := []string{ 178 121 tangled.SpindleMemberNSID, 179 122 tangled.RepoNSID, ··· 184 127 if err != nil { 185 128 return nil, fmt.Errorf("failed to setup jetstream client: %w", err) 186 129 } 187 187 - spindle.jc = jc 188 130 jc.AddDid(cfg.Server.Owner) 189 131 // pull records are created by arbitrary users too, same hack as in tap 190 132 jc.ExemptCollection(tangled.RepoPullNSID) ··· 208 150 } 209 151 } 210 152 211 211 - spindle.res = idresolver.DefaultResolver(cfg.Server.PlcUrl) 212 212 - spindle.verify = repoverify.New(spindle.res, cfg.Server.Dev) 153 153 + resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl) 213 154 155 155 + spindle := &Spindle{ 156 156 + jc: jc, 157 157 + e: e, 158 158 + db: d, 159 159 + l: logger, 160 160 + n: &n, 161 161 + engs: engines, 162 162 + cfg: cfg, 163 163 + res: resolver, 164 164 + verify: repoverify.New(resolver, cfg.Server.Dev), 165 165 + vault: vault, 166 166 + motd: defaultMotd, 167 167 + rootCtx: ctx, 168 168 + jobWake: make(chan struct{}, 1), 169 169 + } 170 170 + diskFallback := cfg.Server.LogDir 171 171 + if cfg.ArtifactStores.Disk.Dir == "" { 172 172 + logger.Warn("using SPINDLE_SERVER_LOG_DIR as the implicit disk artifact store; configure SPINDLE_ARTIFACT_STORES_DISK_DIR explicitly") 173 173 + } 174 174 + stores, err := artifactstore.NewStores(cfg.ArtifactStores, diskFallback, cfg.LegacyS3.LogBucket) 175 175 + if err != nil { 176 176 + return nil, fmt.Errorf("failed to setup artifact stores: %w", err) 177 177 + } 178 178 + spindle.stores = stores 179 179 + spindle.reader = stores 180 180 + if cfg.LegacyS3.LogBucket != "" { 181 181 + logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") 182 182 + } 183 183 + 214 184 err = e.AddSpindle(rbacDomain) 215 185 if err != nil { 216 186 ··· 238 208 ccfg.Logger = log.SubLogger(logger, "eventconsumer") 239 209 ccfg.ProcessFunc = spindle.processKnotStream 240 210 ccfg.CursorStore = cursorStore 211 211 + ccfg.WorkerCount = 16 212 212 + ccfg.QueueSize = 200 241 213 if cfg.Server.Dev { 242 214 ccfg.RetryInterval = 5 * time.Second 243 215 ccfg.MaxRetryInterval = 10 * time.Second ··· 256 228 ccfg.Sources[src] = struct{}{} 257 229 } 258 230 spindle.ks = eventconsumer.NewConsumer(*ccfg) 231 231 + 259 232 if cfg.Server.Tap.Embed { 260 233 pw, err := randomAdminPassword() 261 234 if err != nil { ··· 268 241 269 242 return spindle, nil 270 243 } 244 244 + 245 245 + // DB returns the database instance. 271 246 func (s *Spindle) DB() *db.DB { 272 247 return s.db 273 248 } ··· 306 281 return s.motd 307 282 } 308 283 309 309 - // runs the server. blocks 284 284 + // Start starts the Spindle server (blocking). 310 285 func (s *Spindle) Start(ctx context.Context) error { 311 311 - // only standalone runs the local queue. mill hosts place directly onto 312 312 - // executors, and executors only run jobs explicitly assigned by a mill 286 286 + // starts a job queue runner in the background 313 287 s.StartJobWorkers(ctx) 314 288 315 315 - // an executor dials out to its mill and takes work from it 316 316 - if s.exec != nil { 317 317 - go s.exec.Connect(ctx) 318 318 - } 319 319 - 289 289 + // Stop vault token renewal if it implements Stopper 320 290 if stopper, ok := s.vault.(secrets.Stopper); ok { 321 291 defer stopper.Stop() 322 292 } 323 293 324 324 - if s.cfg.Role != config.RoleExecutor { 325 325 - tapCtx, tapCancel := context.WithCancel(ctx) 294 294 + tapCtx, tapCancel := context.WithCancel(ctx) 326 295 327 327 - if s.cfg.Server.Tap.Embed { 328 328 - emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) 329 329 - if err != nil { 330 330 - tapCancel() 331 331 - return fmt.Errorf("starting embedded tap: %w", err) 332 332 - } 333 333 - s.embedTap = emb 334 334 - defer func() { 335 335 - tapCancel() 336 336 - s.embedTap.Shutdown() 337 337 - }() 338 338 - 339 339 - go s.watchTapDrain(tapCtx, tapCancel) 340 340 - } else { 341 341 - defer tapCancel() 296 296 + if s.cfg.Server.Tap.Embed { 297 297 + emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) 298 298 + if err != nil { 299 299 + tapCancel() 300 300 + return fmt.Errorf("starting embedded tap: %w", err) 342 301 } 343 343 - 344 344 - go func() { 345 345 - s.l.Info("starting knot event consumer") 346 346 - s.ks.Start(ctx) 302 302 + s.embedTap = emb 303 303 + defer func() { 304 304 + tapCancel() 305 305 + s.embedTap.Shutdown() 347 306 }() 348 307 349 349 - s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) 350 350 - s.tap.Start(tapCtx) 308 308 + go s.watchTapDrain(tapCtx, tapCancel) 309 309 + } else { 310 310 + defer tapCancel() 351 311 } 352 312 313 313 + go func() { 314 314 + s.l.Info("starting knot event consumer") 315 315 + s.ks.Start(ctx) 316 316 + }() 317 317 + 318 318 + s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) 319 319 + s.tap.Start(tapCtx) 320 320 + 353 321 s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr) 354 322 return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router()) 355 323 } ··· 394 362 return fmt.Errorf("failed to setup db: %w", err) 395 363 } 396 364 397 397 - logger := log.FromContext(ctx) 398 398 - 399 399 - var engines map[string]models.Engine 400 400 - var m *mill.Mill 401 401 - 402 402 - if cfg.Role == config.RoleMill { 403 403 - // mill host: register engines that place jobs on executors instead of 404 404 - // running them. all names share one Mill 405 405 - m = mill.New(log.SubLogger(logger, "mill"), mill.Config{ 406 406 - LogDir: cfg.Server.LogDir, 407 407 - MaxPending: cfg.Mill.MaxPending, 408 408 - ReconnectGrace: cfg.Mill.ReconnectGrace, 409 409 - }) 410 410 - engines = map[string]models.Engine{ 411 411 - "nixery": mill.NewEngine("nixery", m), 412 412 - "microvm": mill.NewEngine("microvm", m), 413 413 - "dummy": mill.NewEngine("dummy", m), 414 414 - } 415 415 - } else { 416 416 - // standalone and executor both run real engines locally. 417 417 - nixeryEng, err := nixery.New(ctx, cfg) 418 418 - if err != nil { 419 419 - return err 420 420 - } 421 421 - microvmEng, err := newMicrovmEngine(ctx, cfg, d) 422 422 - if err != nil { 423 423 - return err 424 424 - } 425 425 - engines = map[string]models.Engine{ 426 426 - "nixery": nixeryEng, 427 427 - "microvm": microvmEng, 428 428 - "dummy": dummy.New(logger), 429 429 - } 365 365 + nixeryEng, err := nixery.New(ctx, cfg) 366 366 + if err != nil { 367 367 + return err 430 368 } 431 369 432 432 - s, err := New(ctx, cfg, d, engines) 370 370 + microvmEng, err := newMicrovmEngine(ctx, cfg, d) 433 371 if err != nil { 434 372 return err 435 373 } 436 374 437 437 - if m != nil { 438 438 - // resolve the chicken-and-egg: the engines (built above) hold the mill, 439 439 - // but the mill's db/notifier are created inside New 440 440 - m.Attach(s.DB(), s.Notifier()) 441 441 - s.mill = m 442 442 - if err := m.RestoreState(); err != nil { 443 443 - return fmt.Errorf("restoring mill state: %w", err) 444 444 - } 375 375 + s, err := New(ctx, cfg, d, map[string]models.Engine{ 376 376 + "nixery": nixeryEng, 377 377 + "microvm": microvmEng, 378 378 + "dummy": dummy.New(log.FromContext(ctx)), 379 379 + }) 380 380 + if err != nil { 381 381 + return err 445 382 } 446 446 - if cfg.Role == config.RoleExecutor { 447 447 - s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store) 448 448 - if err != nil { 449 449 - return err 450 450 - } 451 451 - } 452 383 453 384 return s.Start(ctx) 454 385 ··· 459 390 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 460 391 w.Write(s.GetMotdContent()) 461 392 }) 462 462 - if s.cfg.Role == config.RoleExecutor { 463 463 - return mux 464 464 - } 465 465 - 466 393 mux.HandleFunc("/events", s.Events) 467 394 mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) 468 395 469 469 - // mill host: executors dial in here (plain ws, shared-secret auth) 470 470 - if s.mill != nil { 471 471 - mux.HandleFunc("/mill", s.mill.HandleExecutorConn) 472 472 - } 473 473 - 474 396 mux.Mount("/xrpc", s.XrpcRouter()) 475 397 return mux 476 398 } ··· 530 452 // NOTE: we are blindly trusting the knot that it will return only repos it own 531 453 repoCloneUri := s.newRepoCloneUrl(src.Host, repoDid) 532 454 repoPath := s.newRepoPath(repoDid) 533 533 - if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, event.NewSha); err != nil { 534 534 - return fmt.Errorf("sync git repo: %w", err) 535 535 - } 536 536 - l.Info("synced git repo") 537 455 538 456 triggerRepo, err := s.buildTriggerRepo(ctx, repo) 539 457 if err != nil { ··· 846 764 return rawPipeline, nil 847 765 } 848 766 849 849 - // newRepoPath creates a path to store repository by its did and rkey. 850 850 - // The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey 851 851 - func (s *Spindle) newRepoPath(repo syntax.DID) string { 852 852 - return filepath.Join(s.cfg.Server.RepoDir, repo.String()) 853 853 - } 854 854 - 855 855 - func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { 856 856 - scheme := "https://" 857 857 - if s.cfg.Server.Dev { 858 858 - scheme = "http://" 859 859 - } 860 860 - return fmt.Sprintf("%s%s/%s", scheme, knot, did) 861 861 - } 862 862 - 863 863 - const RequiredVersion = "2.49.0" 864 864 - 865 865 - func ensureGitVersion() error { 866 866 - v, err := git.Version() 867 867 - if err != nil { 868 868 - return fmt.Errorf("fetching git version: %w", err) 869 869 - } 870 870 - if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { 871 871 - return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) 872 872 - } 873 873 - return nil 874 874 - } 875 875 - 876 876 - func (s *Spindle) configureOwner() error { 877 877 - cfgOwner := s.cfg.Server.Owner 878 878 - 879 879 - existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) 880 880 - if err != nil { 881 881 - return err 882 882 - } 883 883 - 884 884 - switch len(existing) { 885 885 - case 0: 886 886 - // no owner configured, continue 887 887 - case 1: 888 888 - // find existing owner 889 889 - existingOwner := existing[0] 890 890 - 891 891 - // no ownership change, this is okay 892 892 - if existingOwner == s.cfg.Server.Owner { 893 893 - break 894 894 - } 895 895 - 896 896 - // remove existing owner 897 897 - err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) 898 898 - if err != nil { 899 899 - return nil 900 900 - } 901 901 - default: 902 902 - return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) 903 903 - } 904 904 - 905 905 - return s.e.AddSpindleOwner(rbacDomain, cfgOwner) 906 906 - } 907 907 - 908 767 func (s *Spindle) StartJobWorkers(ctx context.Context) { 909 768 for range s.cfg.Server.MaxJobCount { 910 769 go func() { ··· 1012 871 1013 872 } 1014 873 return nil 874 874 + } 875 875 + 876 876 + // newRepoPath creates a path to store repository by its did and rkey. 877 877 + // The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey 878 878 + func (s *Spindle) newRepoPath(repo syntax.DID) string { 879 879 + return filepath.Join(s.cfg.Server.RepoDir, repo.String()) 880 880 + } 881 881 + 882 882 + func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { 883 883 + scheme := "https://" 884 884 + if s.cfg.Server.Dev { 885 885 + scheme = "http://" 886 886 + } 887 887 + return fmt.Sprintf("%s%s/%s", scheme, knot, did) 888 888 + } 889 889 + 890 890 + const RequiredVersion = "2.49.0" 891 891 + 892 892 + func ensureGitVersion() error { 893 893 + v, err := git.Version() 894 894 + if err != nil { 895 895 + return fmt.Errorf("fetching git version: %w", err) 896 896 + } 897 897 + if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { 898 898 + return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) 899 899 + } 900 900 + return nil 901 901 + } 902 902 + 903 903 + func (s *Spindle) resolvePipelineRepoDid(repo *tangled.Pipeline_TriggerRepo) (syntax.DID, error) { 904 904 + if repo.RepoDid == nil || *repo.RepoDid == "" { 905 905 + return "", fmt.Errorf("pipeline trigger missing repoDid") 906 906 + } 907 907 + repoDid, err := syntax.ParseDID(*repo.RepoDid) 908 908 + if err != nil { 909 909 + return "", fmt.Errorf("parse repoDid %s: %w", *repo.RepoDid, err) 910 910 + } 911 911 + if _, err := s.db.GetRepoByDid(repoDid); err != nil { 912 912 + return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 913 913 + } 914 914 + return repoDid, nil 915 915 + } 916 916 + 917 917 + func (s *Spindle) configureOwner() error { 918 918 + cfgOwner := s.cfg.Server.Owner 919 919 + 920 920 + existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) 921 921 + if err != nil { 922 922 + return err 923 923 + } 924 924 + 925 925 + switch len(existing) { 926 926 + case 0: 927 927 + // no owner configured, continue 928 928 + case 1: 929 929 + // find existing owner 930 930 + existingOwner := existing[0] 931 931 + 932 932 + // no ownership change, this is okay 933 933 + if existingOwner == s.cfg.Server.Owner { 934 934 + break 935 935 + } 936 936 + 937 937 + // remove existing owner 938 938 + err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) 939 939 + if err != nil { 940 940 + return nil 941 941 + } 942 942 + default: 943 943 + return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) 944 944 + } 945 945 + 946 946 + return s.e.AddSpindleOwner(rbacDomain, cfgOwner) 1015 947 }

-930

spindle/server.go.master

Reviewed

··· 1 1 - package spindle 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "database/sql" 6 6 - _ "embed" 7 7 - "encoding/json" 8 8 - "errors" 9 9 - "fmt" 10 10 - "log/slog" 11 11 - "maps" 12 12 - "net/http" 13 13 - "path/filepath" 14 14 - "sync" 15 15 - "time" 16 16 - 17 17 - "github.com/bluesky-social/indigo/atproto/syntax" 18 18 - indigoxrpc "github.com/bluesky-social/indigo/xrpc" 19 19 - "github.com/go-chi/chi/v5" 20 20 - "github.com/go-git/go-git/v5/plumbing/object" 21 21 - "github.com/hashicorp/go-version" 22 22 - "tangled.org/core/api/tangled" 23 23 - "tangled.org/core/eventconsumer" 24 24 - "tangled.org/core/eventconsumer/cursor" 25 25 - "tangled.org/core/eventstream" 26 26 - "tangled.org/core/idresolver" 27 27 - "tangled.org/core/jetstream" 28 28 - knotdb "tangled.org/core/knotserver/db" 29 29 - kgit "tangled.org/core/knotserver/git" 30 30 - "tangled.org/core/log" 31 31 - "tangled.org/core/notifier" 32 32 - "tangled.org/core/rbac" 33 33 - "tangled.org/core/repoident" 34 34 - "tangled.org/core/repoverify" 35 35 - "tangled.org/core/spindle/config" 36 36 - "tangled.org/core/spindle/db" 37 37 - "tangled.org/core/spindle/engine" 38 38 - "tangled.org/core/spindle/engines/dummy" 39 39 - "tangled.org/core/spindle/engines/nixery" 40 40 - "tangled.org/core/spindle/git" 41 41 - "tangled.org/core/spindle/models" 42 42 - "tangled.org/core/spindle/secrets" 43 43 - "tangled.org/core/spindle/xrpc" 44 44 - "tangled.org/core/tid" 45 45 - "tangled.org/core/workflow" 46 46 - "tangled.org/core/xrpc/serviceauth" 47 47 - ) 48 48 - 49 49 - //go:embed motd 50 50 - var defaultMotd []byte 51 51 - 52 52 - const ( 53 53 - rbacDomain = "thisserver" 54 54 - ) 55 55 - 56 56 - type Spindle struct { 57 57 - jc *jetstream.JetstreamClient 58 58 - tap *Tap 59 59 - embedTap *embeddedTap 60 60 - db *db.DB 61 61 - e *rbac.Enforcer 62 62 - l *slog.Logger 63 63 - n *notifier.Notifier 64 64 - engs map[string]models.Engine 65 65 - cfg *config.Config 66 66 - ks *eventconsumer.Consumer 67 67 - res *idresolver.Resolver 68 68 - verify repoverify.Verifier 69 69 - vault secrets.Manager 70 70 - motd []byte 71 71 - motdMu sync.RWMutex 72 72 - rootCtx context.Context 73 73 - jobWake chan struct{} 74 74 - } 75 75 - 76 76 - // New creates a new Spindle server with the provided configuration and engines. 77 77 - func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { 78 78 - logger := log.FromContext(ctx) 79 79 - 80 80 - e, err := rbac.NewEnforcer(cfg.Server.DBPath) 81 81 - if err != nil { 82 82 - return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) 83 83 - } 84 84 - e.E.EnableAutoSave(true) 85 85 - 86 86 - n := notifier.New() 87 87 - 88 88 - var vault secrets.Manager 89 89 - switch cfg.Server.Secrets.Provider { 90 90 - case "openbao": 91 91 - if cfg.Server.Secrets.OpenBao.ProxyAddr == "" { 92 92 - return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider") 93 93 - } 94 94 - vault, err = secrets.NewOpenBaoManager( 95 95 - cfg.Server.Secrets.OpenBao.ProxyAddr, 96 96 - logger, 97 97 - secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount), 98 98 - ) 99 99 - if err != nil { 100 100 - return nil, fmt.Errorf("failed to setup openbao secrets provider: %w", err) 101 101 - } 102 102 - logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount) 103 103 - case "sqlite", "": 104 104 - vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) 105 105 - if err != nil { 106 106 - return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err) 107 107 - } 108 108 - logger.Info("using sqlite secrets provider", "path", cfg.Server.DBPath) 109 109 - default: 110 110 - return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) 111 111 - } 112 112 - 113 113 - if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { 114 114 - return nil, fmt.Errorf("failed to run startup migrations: %w", err) 115 115 - } 116 116 - 117 117 - collections := []string{ 118 118 - tangled.SpindleMemberNSID, 119 119 - tangled.RepoNSID, 120 120 - tangled.RepoCollaboratorNSID, 121 121 - tangled.RepoPullNSID, 122 122 - } 123 123 - jc, err := jetstream.NewJetstreamClient(cfg.Server.JetstreamEndpoint, "spindle", collections, nil, log.SubLogger(logger, "jetstream"), d, true, true) 124 124 - if err != nil { 125 125 - return nil, fmt.Errorf("failed to setup jetstream client: %w", err) 126 126 - } 127 127 - jc.AddDid(cfg.Server.Owner) 128 128 - // pull records are created by arbitrary users too, same hack as in tap 129 129 - jc.ExemptCollection(tangled.RepoPullNSID) 130 130 - 131 131 - // Check if the spindle knows about any Dids; 132 132 - dids, err := d.GetAllDids() 133 133 - if err != nil { 134 134 - return nil, fmt.Errorf("failed to get all dids: %w", err) 135 135 - } 136 136 - for _, d := range dids { 137 137 - jc.AddDid(d) 138 138 - } 139 139 - 140 140 - knownRepos, err := d.AllRepos() 141 141 - if err != nil { 142 142 - return nil, fmt.Errorf("failed to get known repos: %w", err) 143 143 - } 144 144 - for _, r := range knownRepos { 145 145 - if r.Owner != "" { 146 146 - jc.AddDid(r.Owner.String()) 147 147 - } 148 148 - } 149 149 - 150 150 - resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl) 151 151 - 152 152 - spindle := &Spindle{ 153 153 - jc: jc, 154 154 - e: e, 155 155 - db: d, 156 156 - l: logger, 157 157 - n: &n, 158 158 - engs: engines, 159 159 - cfg: cfg, 160 160 - res: resolver, 161 161 - verify: repoverify.New(resolver, cfg.Server.Dev), 162 162 - vault: vault, 163 163 - motd: defaultMotd, 164 164 - rootCtx: ctx, 165 165 - jobWake: make(chan struct{}, 1), 166 166 - } 167 167 - 168 168 - err = e.AddSpindle(rbacDomain) 169 169 - if err != nil { 170 170 - return nil, fmt.Errorf("failed to set rbac domain: %w", err) 171 171 - } 172 172 - err = spindle.configureOwner() 173 173 - if err != nil { 174 174 - return nil, err 175 175 - } 176 176 - logger.Info("owner set", "did", cfg.Server.Owner) 177 177 - 178 178 - cursorStore, err := cursor.NewSQLiteStore(cfg.Server.DBPath) 179 179 - if err != nil { 180 180 - return nil, fmt.Errorf("failed to setup sqlite3 cursor store: %w", err) 181 181 - } 182 182 - 183 183 - err = jc.StartJetstream(ctx, spindle.ingest()) 184 184 - if err != nil { 185 185 - return nil, fmt.Errorf("failed to start jetstream consumer: %w", err) 186 186 - } 187 187 - 188 188 - // spindle listen to knot stream for sh.tangled.git.refUpdate 189 189 - // which will sync the local workflow files in spindle and enqueues the 190 190 - // pipeline job for on-push workflows 191 191 - ccfg := eventconsumer.NewConsumerConfig() 192 192 - ccfg.Logger = log.SubLogger(logger, "eventconsumer") 193 193 - ccfg.ProcessFunc = spindle.processKnotStream 194 194 - ccfg.CursorStore = cursorStore 195 195 - ccfg.WorkerCount = 16 196 196 - ccfg.QueueSize = 200 197 197 - if cfg.Server.Dev { 198 198 - ccfg.RetryInterval = 5 * time.Second 199 199 - ccfg.MaxRetryInterval = 10 * time.Second 200 200 - } else { 201 201 - ccfg.RetryInterval = 1 * time.Minute 202 202 - ccfg.MaxRetryInterval = 10 * time.Minute 203 203 - } 204 204 - knownKnots, err := d.Knots() 205 205 - if err != nil { 206 206 - return nil, err 207 207 - } 208 208 - for _, knot := range knownKnots { 209 209 - logger.Info("adding source start", "knot", knot) 210 210 - src := eventconsumer.NewKnotSource(knot) 211 211 - eventconsumer.MigrateLegacyCursor(cursorStore, src) 212 212 - ccfg.Sources[src] = struct{}{} 213 213 - } 214 214 - spindle.ks = eventconsumer.NewConsumer(*ccfg) 215 215 - 216 216 - if cfg.Server.Tap.Embed { 217 217 - pw, err := randomAdminPassword() 218 218 - if err != nil { 219 219 - return nil, err 220 220 - } 221 221 - cfg.Server.Tap.AdminPassword = pw 222 222 - logger.Info("embedded tap: using random admin password") 223 223 - } 224 224 - spindle.tap = NewTapClient(spindle) 225 225 - 226 226 - return spindle, nil 227 227 - } 228 228 - 229 229 - // DB returns the database instance. 230 230 - func (s *Spindle) DB() *db.DB { 231 231 - return s.db 232 232 - } 233 233 - 234 234 - // Engines returns the map of available engines. 235 235 - func (s *Spindle) Engines() map[string]models.Engine { 236 236 - return s.engs 237 237 - } 238 238 - 239 239 - // Vault returns the secrets manager instance. 240 240 - func (s *Spindle) Vault() secrets.Manager { 241 241 - return s.vault 242 242 - } 243 243 - 244 244 - // Notifier returns the notifier instance. 245 245 - func (s *Spindle) Notifier() *notifier.Notifier { 246 246 - return s.n 247 247 - } 248 248 - 249 249 - // Enforcer returns the RBAC enforcer instance. 250 250 - func (s *Spindle) Enforcer() *rbac.Enforcer { 251 251 - return s.e 252 252 - } 253 253 - 254 254 - // SetMotdContent sets custom MOTD content, replacing the embedded default. 255 255 - func (s *Spindle) SetMotdContent(content []byte) { 256 256 - s.motdMu.Lock() 257 257 - defer s.motdMu.Unlock() 258 258 - s.motd = content 259 259 - } 260 260 - 261 261 - // GetMotdContent returns the current MOTD content. 262 262 - func (s *Spindle) GetMotdContent() []byte { 263 263 - s.motdMu.RLock() 264 264 - defer s.motdMu.RUnlock() 265 265 - return s.motd 266 266 - } 267 267 - 268 268 - // Start starts the Spindle server (blocking). 269 269 - func (s *Spindle) Start(ctx context.Context) error { 270 270 - // starts a job queue runner in the background 271 271 - s.StartJobWorkers(ctx) 272 272 - 273 273 - // Stop vault token renewal if it implements Stopper 274 274 - if stopper, ok := s.vault.(secrets.Stopper); ok { 275 275 - defer stopper.Stop() 276 276 - } 277 277 - 278 278 - tapCtx, tapCancel := context.WithCancel(ctx) 279 279 - 280 280 - if s.cfg.Server.Tap.Embed { 281 281 - emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) 282 282 - if err != nil { 283 283 - tapCancel() 284 284 - return fmt.Errorf("starting embedded tap: %w", err) 285 285 - } 286 286 - s.embedTap = emb 287 287 - defer func() { 288 288 - tapCancel() 289 289 - s.embedTap.Shutdown() 290 290 - }() 291 291 - 292 292 - go s.watchTapDrain(tapCtx, tapCancel) 293 293 - } else { 294 294 - defer tapCancel() 295 295 - } 296 296 - 297 297 - go func() { 298 298 - s.l.Info("starting knot event consumer") 299 299 - s.ks.Start(ctx) 300 300 - }() 301 301 - 302 302 - s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) 303 303 - s.tap.Start(tapCtx) 304 304 - 305 305 - s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr) 306 306 - return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router()) 307 307 - } 308 308 - 309 309 - func (s *Spindle) declareTapInterest(ctx context.Context) { 310 310 - repos, err := s.db.AllRepos() 311 311 - if err != nil { 312 312 - s.l.Warn("tap declare: failed to load known repos", "err", err) 313 313 - return 314 314 - } 315 315 - seen := make(map[syntax.DID]struct{}, len(repos)) 316 316 - dids := make([]syntax.DID, 0, len(repos)) 317 317 - for _, r := range repos { 318 318 - if r.Owner == "" { 319 319 - continue 320 320 - } 321 321 - if _, ok := seen[r.Owner]; ok { 322 322 - continue 323 323 - } 324 324 - seen[r.Owner] = struct{}{} 325 325 - dids = append(dids, r.Owner) 326 326 - } 327 327 - if err := s.tap.AddOwnerDIDs(ctx, dids); err != nil { 328 328 - s.l.Warn("tap declare: AddRepos rejected", "count", len(dids), "err", err) 329 329 - return 330 330 - } 331 331 - s.l.Info("tap declare: known owner DIDs registered", "count", len(dids)) 332 332 - } 333 333 - 334 334 - func Run(ctx context.Context) error { 335 335 - cfg, err := config.Load(ctx) 336 336 - if err != nil { 337 337 - return fmt.Errorf("failed to load config: %w", err) 338 338 - } 339 339 - 340 340 - if err := ensureGitVersion(); err != nil { 341 341 - return fmt.Errorf("ensuring git version: %w", err) 342 342 - } 343 343 - 344 344 - d, err := db.Make(ctx, cfg.Server.DBPath) 345 345 - if err != nil { 346 346 - return fmt.Errorf("failed to setup db: %w", err) 347 347 - } 348 348 - 349 349 - nixeryEng, err := nixery.New(ctx, cfg) 350 350 - if err != nil { 351 351 - return err 352 352 - } 353 353 - 354 354 - microvmEng, err := newMicrovmEngine(ctx, cfg, d) 355 355 - if err != nil { 356 356 - return err 357 357 - } 358 358 - 359 359 - s, err := New(ctx, cfg, d, map[string]models.Engine{ 360 360 - "nixery": nixeryEng, 361 361 - "microvm": microvmEng, 362 362 - "dummy": dummy.New(log.FromContext(ctx)), 363 363 - }) 364 364 - if err != nil { 365 365 - return err 366 366 - } 367 367 - 368 368 - return s.Start(ctx) 369 369 - } 370 370 - 371 371 - func (s *Spindle) Router() http.Handler { 372 372 - mux := chi.NewRouter() 373 373 - 374 374 - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 375 375 - w.Write(s.GetMotdContent()) 376 376 - }) 377 377 - mux.HandleFunc("/events", s.Events) 378 378 - mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) 379 379 - 380 380 - mux.Mount("/xrpc", s.XrpcRouter()) 381 381 - return mux 382 382 - } 383 383 - 384 384 - func (s *Spindle) XrpcRouter() http.Handler { 385 385 - serviceAuth := serviceauth.NewServiceAuth(s.l, s.res.Directory(), s.cfg.Server.Did().String()) 386 386 - 387 387 - l := log.SubLogger(s.l, "xrpc") 388 388 - 389 389 - x := xrpc.Xrpc{ 390 390 - Logger: l, 391 391 - Db: s.db, 392 392 - Enforcer: s.e, 393 393 - Engines: s.engs, 394 394 - Config: s.cfg, 395 395 - Resolver: s.res, 396 396 - Vault: s.vault, 397 397 - Notifier: s.Notifier(), 398 398 - ServiceAuth: serviceAuth, 399 399 - Trigger: s, 400 400 - } 401 401 - 402 402 - return x.Router() 403 403 - } 404 404 - 405 405 - func (s *Spindle) processKnotStream(ctx context.Context, src eventconsumer.Source, msg eventstream.Event) error { 406 406 - l := log.FromContext(ctx).With("handler", "processKnotStream") 407 407 - l = l.With("src", src.Key(), "msg.Nsid", msg.Nsid, "msg.Rkey", msg.Rkey) 408 408 - if msg.Nsid == knotdb.RepoCollaboratorUpdateNSID { 409 409 - return s.ingestKnotCollaborator(ctx, l, src, msg) 410 410 - } 411 411 - if msg.Nsid == tangled.GitRefUpdateNSID { 412 412 - event := tangled.GitRefUpdate{} 413 413 - if err := json.Unmarshal(msg.EventJson, &event); err != nil { 414 414 - l.Error("error unmarshalling", "err", err) 415 415 - return err 416 416 - } 417 417 - l = l.With("repo", event.Repo, "ref", event.Ref, "newSha", event.NewSha) 418 418 - l.Debug("debug") 419 419 - 420 420 - repoDid := syntax.DID(event.Repo) 421 421 - repo, err := s.db.GetRepoByDid(repoDid) 422 422 - if err != nil { 423 423 - return fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 424 424 - } 425 425 - 426 426 - if src.Host != repo.Knot { 427 427 - return fmt.Errorf("repo knot does not match event source: %s != %s", src.Host, repo.Knot) 428 428 - } 429 429 - 430 430 - if kgit.HasSkipCIPushOption(event.PushOptions) { 431 431 - l.Info("push event requested ci skip, skipping the event") 432 432 - return nil 433 433 - } 434 434 - 435 435 - // NOTE: we are blindly trusting the knot that it will return only repos it own 436 436 - repoCloneUri := s.newRepoCloneUrl(src.Host, repoDid) 437 437 - repoPath := s.newRepoPath(repoDid) 438 438 - 439 439 - triggerRepo, err := s.buildTriggerRepo(ctx, repo) 440 440 - if err != nil { 441 441 - return fmt.Errorf("building trigger repo: %w", err) 442 442 - } 443 443 - 444 444 - trigger := tangled.Pipeline_TriggerMetadata{ 445 445 - Kind: string(workflow.TriggerKindPush), 446 446 - Push: &tangled.Pipeline_PushTriggerData{ 447 447 - Ref: event.Ref, 448 448 - OldSha: event.OldSha, 449 449 - NewSha: event.NewSha, 450 450 - }, 451 451 - Repo: triggerRepo, 452 452 - } 453 453 - 454 454 - pipelineId, err := s.runPipeline(ctx, repoDid, trigger, event.ChangedFiles, repoCloneUri, repoPath, event.NewSha, nil, triggerRepo) 455 455 - if err != nil { 456 456 - return err 457 457 - } 458 458 - if pipelineId.Rkey == "" { 459 459 - l.Info("no workflow matched 'push' trigger, skipping the event") 460 460 - return nil 461 461 - } 462 462 - l.Info("pipeline triggered", "pipeline", pipelineId.AtUri()) 463 463 - } 464 464 - 465 465 - return nil 466 466 - } 467 467 - 468 468 - func (s *Spindle) ingestKnotCollaborator(ctx context.Context, l *slog.Logger, src eventconsumer.Source, msg eventstream.Event) error { 469 469 - var rec knotdb.RepoCollaboratorUpdate 470 470 - if err := json.Unmarshal(msg.EventJson, &rec); err != nil { 471 471 - l.Error("error unmarshalling collaboratorUpdate", "err", err) 472 472 - return err 473 473 - } 474 474 - 475 475 - subject, err := syntax.ParseDID(rec.Subject) 476 476 - if err != nil { 477 477 - l.Info("skipping collaboratorUpdate with malformed subject", "subject", rec.Subject, "err", err) 478 478 - return nil 479 479 - } 480 480 - repoDid, err := syntax.ParseDID(rec.Repo) 481 481 - if err != nil { 482 482 - l.Info("skipping collaboratorUpdate with malformed repo", "repo", rec.Repo, "err", err) 483 483 - return nil 484 484 - } 485 485 - 486 486 - repo, err := s.db.GetRepoByDid(repoDid) 487 487 - if errors.Is(err, sql.ErrNoRows) { 488 488 - l.Info("skipping collaboratorUpdate for unknown repo", "repo", repoDid) 489 489 - return nil 490 490 - } 491 491 - if err != nil { 492 492 - return fmt.Errorf("lookup repo %s: %w", repoDid, err) 493 493 - } 494 494 - if src.Host != repo.Knot { 495 495 - l.Warn("dropping collaboratorUpdate from non-owning knot", "src", src.Host, "repoKnot", repo.Knot) 496 496 - return nil 497 497 - } 498 498 - 499 499 - switch rec.Op { 500 500 - case knotdb.AclOpAdd: 501 501 - if err := s.e.AddCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { 502 502 - return fmt.Errorf("add collaborator policy: %w", err) 503 503 - } 504 504 - if err := s.db.AddKnotCollaborator(repoDid, subject); err != nil { 505 505 - return fmt.Errorf("track collaborator: %w", err) 506 506 - } 507 507 - l.Info("added knot-managed collaborator", "subject", subject, "repo", repoDid) 508 508 - case knotdb.AclOpRemove: 509 509 - if err := s.e.RemoveCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { 510 510 - return fmt.Errorf("remove collaborator policy: %w", err) 511 511 - } 512 512 - if err := s.db.DeleteRepoCollaboratorBySubjectRepo(subject, repoDid); err != nil { 513 513 - return fmt.Errorf("delete collaborator row: %w", err) 514 514 - } 515 515 - l.Info("removed knot-managed collaborator", "subject", subject, "repo", repoDid) 516 516 - default: 517 517 - return fmt.Errorf("collaboratorUpdate unknown op %q", rec.Op) 518 518 - } 519 519 - return nil 520 520 - } 521 521 - 522 522 - // buildTriggerRepo gathers trigger metadata, resolving default branch from the knot 523 523 - func (s *Spindle) buildTriggerRepo(ctx context.Context, repo *db.Repo) (*tangled.Pipeline_TriggerRepo, error) { 524 524 - rkey := string(repo.Rkey) 525 525 - repoDid := repo.RepoDid.String() 526 526 - return s.buildTriggerRepoFrom(ctx, repo.Knot, repo.Owner.String(), rkey, repoDid), nil 527 527 - } 528 528 - 529 529 - func (s *Spindle) buildTriggerRepoFrom(ctx context.Context, knot, did, rkey, repoDid string) *tangled.Pipeline_TriggerRepo { 530 530 - scheme := "https" 531 531 - if s.cfg.Server.Dev { 532 532 - scheme = "http" 533 533 - } 534 534 - client := &indigoxrpc.Client{Host: fmt.Sprintf("%s://%s", scheme, knot)} 535 535 - 536 536 - // this should maybe (?) be in the refUpdate event itself to save a roundtrip 537 537 - defaultBranch := "" 538 538 - if out, err := tangled.RepoGetDefaultBranch(ctx, client, repoDid); err == nil { 539 539 - defaultBranch = out.Name 540 540 - } 541 541 - 542 542 - var rkeyPtr *string 543 543 - if rkey != "" { 544 544 - rkeyPtr = &rkey 545 545 - } 546 546 - return &tangled.Pipeline_TriggerRepo{ 547 547 - Did: did, 548 548 - Knot: knot, 549 549 - Repo: rkeyPtr, 550 550 - RepoDid: &repoDid, 551 551 - DefaultBranch: defaultBranch, 552 552 - } 553 553 - } 554 554 - 555 555 - func (s *Spindle) resolvePipelineSourceRepo(ctx context.Context, trigger *tangled.Pipeline_TriggerMetadata) (*tangled.Pipeline_TriggerRepo, error) { 556 556 - if trigger == nil { 557 557 - return nil, nil 558 558 - } 559 559 - if trigger.SourceRepo == nil || *trigger.SourceRepo == "" { 560 560 - return trigger.Repo, nil 561 561 - } 562 562 - repoDid, err := syntax.ParseDID(*trigger.SourceRepo) 563 563 - if err != nil { 564 564 - return nil, fmt.Errorf("parse sourceRepo %s: %w", *trigger.SourceRepo, err) 565 565 - } 566 566 - return s.resolveSourceRepoInfo(ctx, repoDid) 567 567 - } 568 568 - 569 569 - // resolveSourceRepoInfo resolves trigger-repo metadata for a source repo DID. 570 570 - func (s *Spindle) resolveSourceRepoInfo(ctx context.Context, repoDid syntax.DID) (*tangled.Pipeline_TriggerRepo, error) { 571 571 - repo, err := s.db.GetRepoByDid(repoDid) 572 572 - if err == nil { 573 573 - return s.buildTriggerRepo(ctx, repo) 574 574 - } 575 575 - 576 576 - // verify repo, we don't want git sync to point to arbitrary endpoints 577 577 - res, err := s.verify(ctx, repoident.RepoDid(repoDid)) 578 578 - if err != nil { 579 579 - return nil, fmt.Errorf("verify sourceRepo %s: %w", repoDid, err) 580 580 - } 581 581 - return s.buildTriggerRepoFrom(ctx, res.KnotURL.Host, res.OwnerDid.String(), res.Rkey, repoDid.String()), nil 582 582 - } 583 583 - 584 584 - // runPipeline compiles and enqueues the pipeline for the given revision. 585 585 - // sourceRepo is the resolved repo the code was checked out from, forwarded to 586 586 - // processPipeline for env vars. 587 587 - func (s *Spindle) runPipeline(ctx context.Context, repoDid syntax.DID, trigger tangled.Pipeline_TriggerMetadata, changedFiles []string, repoCloneUri, repoPath, rev string, only []string, sourceRepo *tangled.Pipeline_TriggerRepo) (models.PipelineId, error) { 588 588 - l := log.FromContext(ctx) 589 589 - 590 590 - compiler := workflow.Compiler{ 591 591 - ChangedFiles: changedFiles, 592 592 - Trigger: trigger, 593 593 - } 594 594 - 595 595 - rawPipeline, err := s.loadPipeline(ctx, repoCloneUri, repoPath, rev) 596 596 - if err != nil { 597 597 - return models.PipelineId{}, fmt.Errorf("loading pipeline: %w", err) 598 598 - } 599 599 - if len(rawPipeline) == 0 { 600 600 - return models.PipelineId{}, nil 601 601 - } 602 602 - 603 603 - tpl := compiler.Compile(compiler.Parse(rawPipeline)) 604 604 - // todo(dawn): pass compile error to workflow log 605 605 - for _, w := range compiler.Diagnostics.Errors { 606 606 - l.Error(w.String()) 607 607 - } 608 608 - for _, w := range compiler.Diagnostics.Warnings { 609 609 - l.Warn(w.String()) 610 610 - } 611 611 - 612 612 - if len(only) > 0 { 613 613 - tpl.Workflows = filterWorkflows(tpl.Workflows, only) 614 614 - } 615 615 - if len(tpl.Workflows) == 0 { 616 616 - return models.PipelineId{}, nil 617 617 - } 618 618 - 619 619 - pipelineId := models.PipelineId{ 620 620 - Knot: trigger.Repo.Knot, 621 621 - Rkey: tid.TID(), 622 622 - } 623 623 - if err := s.db.CreatePipelineEvent(pipelineId.Rkey, tpl, s.n); err != nil { 624 624 - return models.PipelineId{}, fmt.Errorf("creating pipeline event: %w", err) 625 625 - } 626 626 - err = s.processPipeline(repoDid, tpl, pipelineId, sourceRepo) 627 627 - return pipelineId, err 628 628 - } 629 629 - 630 630 - // filterWorkflows filters workflows to the requested names 631 631 - func filterWorkflows(workflows []*tangled.Pipeline_Workflow, only []string) []*tangled.Pipeline_Workflow { 632 632 - allowed := make(map[string]struct{}, len(only)) 633 633 - for _, n := range only { 634 634 - allowed[n] = struct{}{} 635 635 - } 636 636 - var filtered []*tangled.Pipeline_Workflow 637 637 - for _, w := range workflows { 638 638 - if w == nil { 639 639 - continue 640 640 - } 641 641 - if _, ok := allowed[w.Name]; ok { 642 642 - filtered = append(filtered, w) 643 643 - } 644 644 - } 645 645 - return filtered 646 646 - } 647 647 - 648 648 - // TriggerManual dispatches a pipeline at sha, authorized against and recorded 649 649 - // under repoDid. sourceRepo, pull, and inputs are optional trigger payload. 650 650 - func (s *Spindle) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull xrpc.PullContext, inputs []*tangled.Pipeline_Pair) (syntax.ATURI, error) { 651 651 - repo, err := s.db.GetRepoByDid(repoDid) 652 652 - if err != nil { 653 653 - return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 654 654 - } 655 655 - 656 656 - triggerRepo, err := s.buildTriggerRepo(ctx, repo) 657 657 - if err != nil { 658 658 - return "", fmt.Errorf("building trigger repo: %w", err) 659 659 - } 660 660 - 661 661 - trigger := tangled.Pipeline_TriggerMetadata{Repo: triggerRepo} 662 662 - if pull.IsPullRequest { 663 663 - var pullAt *string 664 664 - if pull.Pull != "" { 665 665 - pullAtStr := pull.Pull.String() 666 666 - pullAt = &pullAtStr 667 667 - } 668 668 - trigger.Kind = string(workflow.TriggerKindPullRequest) 669 669 - trigger.PullRequest = &tangled.Pipeline_PullRequestTriggerData{ 670 670 - SourceBranch: pull.SourceBranch, 671 671 - TargetBranch: pull.TargetBranch, 672 672 - SourceSha: sha, 673 673 - Pull: pullAt, 674 674 - } 675 675 - } else { 676 676 - var refPtr *string 677 677 - if ref != "" { 678 678 - refPtr = &ref 679 679 - } 680 680 - trigger.Kind = string(workflow.TriggerKindManual) 681 681 - trigger.Manual = &tangled.Pipeline_ManualTriggerData{ 682 682 - Sha: sha, 683 683 - Ref: refPtr, 684 684 - Inputs: inputs, 685 685 - } 686 686 - } 687 687 - 688 688 - repoCloneUri := s.newRepoCloneUrl(repo.Knot, repoDid) 689 689 - repoPath := s.newRepoPath(repoDid) 690 690 - sourceInfo := triggerRepo // default: code comes from the repo itself 691 691 - if sourceRepo != "" && sourceRepo != repoDid { 692 692 - sourceInfo, err = s.resolveSourceRepoInfo(ctx, sourceRepo) 693 693 - if err != nil { 694 694 - return "", err 695 695 - } 696 696 - sourceRepoStr := sourceRepo.String() 697 697 - trigger.SourceRepo = &sourceRepoStr 698 698 - repoCloneUri = models.BuildRepoURL(sourceInfo) 699 699 - repoPath = s.newRepoPath(sourceRepo) 700 700 - } 701 701 - 702 702 - pipelineId, err := s.runPipeline(ctx, repoDid, trigger, nil, repoCloneUri, repoPath, sha, workflows, sourceInfo) 703 703 - if err != nil { 704 704 - return "", err 705 705 - } 706 706 - if pipelineId.Rkey == "" { 707 707 - return "", xrpc.ErrNoMatchingWorkflows 708 708 - } 709 709 - return pipelineId.AtUri(), nil 710 710 - } 711 711 - 712 712 - func (s *Spindle) loadPipeline(ctx context.Context, repoUri, repoPath, rev string) (workflow.RawPipeline, error) { 713 713 - if err := git.SparseSyncGitRepo(ctx, repoUri, repoPath, rev); err != nil { 714 714 - return nil, fmt.Errorf("syncing git repo: %w", err) 715 715 - } 716 716 - gr, err := kgit.Open(repoPath, rev) 717 717 - if err != nil { 718 718 - return nil, fmt.Errorf("opening git repo: %w", err) 719 719 - } 720 720 - 721 721 - workflowDir, err := gr.FileTree(ctx, workflow.WorkflowDir) 722 722 - if errors.Is(err, object.ErrDirectoryNotFound) { 723 723 - // return empty RawPipeline when directory doesn't exist 724 724 - return nil, nil 725 725 - } else if err != nil { 726 726 - return nil, fmt.Errorf("loading file tree: %w", err) 727 727 - } 728 728 - 729 729 - var rawPipeline workflow.RawPipeline 730 730 - for _, e := range workflowDir { 731 731 - if !e.IsFile() { 732 732 - continue 733 733 - } 734 734 - 735 735 - fpath := filepath.Join(workflow.WorkflowDir, e.Name) 736 736 - contents, err := gr.RawContent(fpath) 737 737 - if err != nil { 738 738 - return nil, fmt.Errorf("reading raw content of '%s': %w", fpath, err) 739 739 - } 740 740 - 741 741 - rawPipeline = append(rawPipeline, workflow.RawWorkflow{ 742 742 - Name: e.Name, 743 743 - Contents: contents, 744 744 - }) 745 745 - } 746 746 - 747 747 - return rawPipeline, nil 748 748 - } 749 749 - 750 750 - func (s *Spindle) StartJobWorkers(ctx context.Context) { 751 751 - for range s.cfg.Server.MaxJobCount { 752 752 - go func() { 753 753 - for { 754 754 - job, err := s.db.DequeueJob(ctx) 755 755 - if err != nil { 756 756 - s.l.Error("failed to dequeue job", "error", err) 757 757 - } 758 758 - if job == nil { 759 759 - // sleep until a new job wakes us 760 760 - select { 761 761 - case <-ctx.Done(): 762 762 - return 763 763 - case <-s.jobWake: 764 764 - } 765 765 - continue 766 766 - } 767 767 - s.runJob(ctx, job) 768 768 - } 769 769 - }() 770 770 - } 771 771 - } 772 772 - 773 773 - func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { 774 774 - pipelineId := models.PipelineId{ 775 775 - Knot: job.PipelineIdKnot, 776 776 - Rkey: job.PipelineIdRkey, 777 777 - } 778 778 - 779 779 - pipelineEnv := models.PipelineEnvVarsForSource(job.Tpl.TriggerMetadata, pipelineId, job.SourceRepo) 780 780 - trustedSource := true 781 781 - if tm := job.Tpl.TriggerMetadata; tm != nil && tm.SourceRepo != nil && 782 782 - *tm.SourceRepo != "" && *tm.SourceRepo != job.RepoDid { 783 783 - trustedSource = false 784 784 - } 785 785 - 786 786 - initTpl := job.Tpl 787 787 - if job.SourceRepo != nil && job.Tpl.TriggerMetadata != nil { 788 788 - tm := *job.Tpl.TriggerMetadata 789 789 - tm.Repo = job.SourceRepo 790 790 - initTpl.TriggerMetadata = &tm 791 791 - } 792 792 - 793 793 - workflows := make(map[models.Engine][]models.Workflow) 794 794 - for _, w := range job.Tpl.Workflows { 795 795 - if w == nil { 796 796 - continue 797 797 - } 798 798 - eng, ok := s.engs[w.Engine] 799 799 - if !ok { 800 800 - _ = s.db.StatusFailed(models.WorkflowId{ 801 801 - PipelineId: pipelineId, 802 802 - Name: w.Name, 803 803 - }, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n) 804 804 - continue 805 805 - } 806 806 - 807 807 - ewf, err := eng.InitWorkflow(*w, initTpl) 808 808 - if err != nil { 809 809 - _ = s.db.StatusFailed(models.WorkflowId{ 810 810 - PipelineId: pipelineId, 811 811 - Name: w.Name, 812 812 - }, fmt.Sprintf("init workflow: %s", err), -1, s.n) 813 813 - continue 814 814 - } 815 815 - 816 816 - if ewf.Environment == nil { 817 817 - ewf.Environment = make(map[string]string) 818 818 - } 819 819 - maps.Copy(ewf.Environment, pipelineEnv) 820 820 - workflows[eng] = append(workflows[eng], *ewf) 821 821 - } 822 822 - 823 823 - engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, &models.Pipeline{ 824 824 - RepoDid: syntax.DID(job.RepoDid), 825 825 - Workflows: workflows, 826 826 - TrustedSource: trustedSource, 827 827 - }, pipelineId) 828 828 - } 829 829 - 830 830 - // enqueues the workflows in tpl. 831 831 - func (s *Spindle) processPipeline(repoDid syntax.DID, tpl tangled.Pipeline, pipelineId models.PipelineId, sourceRepo *tangled.Pipeline_TriggerRepo) error { 832 832 - err := s.db.EnqueueJob(s.rootCtx, repoDid.String(), pipelineId, sourceRepo, tpl) 833 833 - if err != nil { 834 834 - return fmt.Errorf("failed to enqueue durable job: %w", err) 835 835 - } 836 836 - s.l.Info("pipeline enqueued successfully to db", "id", pipelineId) 837 837 - 838 838 - // wake up an idle worker to pick up more jobs if any 839 839 - select { 840 840 - case s.jobWake <- struct{}{}: 841 841 - default: 842 842 - } 843 843 - 844 844 - // pipelines visible from now on, they are sitting in queue 845 845 - for _, w := range tpl.Workflows { 846 846 - if w == nil { 847 847 - continue 848 848 - } 849 849 - if err := s.db.StatusPending(models.WorkflowId{ 850 850 - PipelineId: pipelineId, 851 851 - Name: w.Name, 852 852 - }, s.n); err != nil { 853 853 - return fmt.Errorf("db.StatusPending: %w", err) 854 854 - } 855 855 - } 856 856 - return nil 857 857 - } 858 858 - 859 859 - // newRepoPath creates a path to store repository by its did and rkey. 860 860 - // The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey 861 861 - func (s *Spindle) newRepoPath(repo syntax.DID) string { 862 862 - return filepath.Join(s.cfg.Server.RepoDir, repo.String()) 863 863 - } 864 864 - 865 865 - func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { 866 866 - scheme := "https://" 867 867 - if s.cfg.Server.Dev { 868 868 - scheme = "http://" 869 869 - } 870 870 - return fmt.Sprintf("%s%s/%s", scheme, knot, did) 871 871 - } 872 872 - 873 873 - const RequiredVersion = "2.49.0" 874 874 - 875 875 - func ensureGitVersion() error { 876 876 - v, err := git.Version() 877 877 - if err != nil { 878 878 - return fmt.Errorf("fetching git version: %w", err) 879 879 - } 880 880 - if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { 881 881 - return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) 882 882 - } 883 883 - return nil 884 884 - } 885 885 - 886 886 - func (s *Spindle) resolvePipelineRepoDid(repo *tangled.Pipeline_TriggerRepo) (syntax.DID, error) { 887 887 - if repo.RepoDid == nil || *repo.RepoDid == "" { 888 888 - return "", fmt.Errorf("pipeline trigger missing repoDid") 889 889 - } 890 890 - repoDid, err := syntax.ParseDID(*repo.RepoDid) 891 891 - if err != nil { 892 892 - return "", fmt.Errorf("parse repoDid %s: %w", *repo.RepoDid, err) 893 893 - } 894 894 - if _, err := s.db.GetRepoByDid(repoDid); err != nil { 895 895 - return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 896 896 - } 897 897 - return repoDid, nil 898 898 - } 899 899 - 900 900 - func (s *Spindle) configureOwner() error { 901 901 - cfgOwner := s.cfg.Server.Owner 902 902 - 903 903 - existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) 904 904 - if err != nil { 905 905 - return err 906 906 - } 907 907 - 908 908 - switch len(existing) { 909 909 - case 0: 910 910 - // no owner configured, continue 911 911 - case 1: 912 912 - // find existing owner 913 913 - existingOwner := existing[0] 914 914 - 915 915 - // no ownership change, this is okay 916 916 - if existingOwner == s.cfg.Server.Owner { 917 917 - break 918 918 - } 919 919 - 920 920 - // remove existing owner 921 921 - err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) 922 922 - if err != nil { 923 923 - return nil 924 924 - } 925 925 - default: 926 926 - return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) 927 927 - } 928 928 - 929 929 - return s.e.AddSpindleOwner(rbacDomain, cfgOwner) 930 930 - }

-1028

spindle/server.go.mill

Reviewed

··· 1 1 - package spindle 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "database/sql" 6 6 - _ "embed" 7 7 - "encoding/json" 8 8 - "errors" 9 9 - "fmt" 10 10 - "log/slog" 11 11 - "maps" 12 12 - "net/http" 13 13 - "path/filepath" 14 14 - "sync" 15 15 - "time" 16 16 - 17 17 - "github.com/bluesky-social/indigo/atproto/syntax" 18 18 - indigoxrpc "github.com/bluesky-social/indigo/xrpc" 19 19 - "github.com/go-chi/chi/v5" 20 20 - "github.com/go-git/go-git/v5/plumbing/object" 21 21 - "github.com/hashicorp/go-version" 22 22 - "tangled.org/core/api/tangled" 23 23 - "tangled.org/core/eventconsumer" 24 24 - "tangled.org/core/eventconsumer/cursor" 25 25 - "tangled.org/core/eventstream" 26 26 - "tangled.org/core/idresolver" 27 27 - "tangled.org/core/jetstream" 28 28 - knotdb "tangled.org/core/knotserver/db" 29 29 - kgit "tangled.org/core/knotserver/git" 30 30 - "tangled.org/core/log" 31 31 - "tangled.org/core/notifier" 32 32 - "tangled.org/core/rbac" 33 33 - "tangled.org/core/repoident" 34 34 - "tangled.org/core/repoverify" 35 35 - "tangled.org/core/spindle/artifactstore" 36 36 - "tangled.org/core/spindle/config" 37 37 - "tangled.org/core/spindle/db" 38 38 - "tangled.org/core/spindle/engine" 39 39 - "tangled.org/core/spindle/engines/dummy" 40 40 - "tangled.org/core/spindle/engines/nixery" 41 41 - "tangled.org/core/spindle/git" 42 42 - "tangled.org/core/spindle/mill" 43 43 - "tangled.org/core/spindle/mill/executor" 44 44 - "tangled.org/core/spindle/models" 45 45 - "tangled.org/core/spindle/queue" 46 46 - "tangled.org/core/spindle/secrets" 47 47 - "tangled.org/core/spindle/xrpc" 48 48 - "tangled.org/core/tid" 49 49 - "tangled.org/core/workflow" 50 50 - "tangled.org/core/xrpc/serviceauth" 51 51 - ) 52 52 - 53 53 - //go:embed motd 54 54 - var defaultMotd []byte 55 55 - 56 56 - const ( 57 57 - rbacDomain = "thisserver" 58 58 - ) 59 59 - 60 60 - type Spindle struct { 61 61 - jc *jetstream.JetstreamClient 62 62 - tap *Tap 63 63 - embedTap *embeddedTap 64 64 - db *db.DB 65 65 - e *rbac.Enforcer 66 66 - l *slog.Logger 67 67 - n *notifier.Notifier 68 68 - engs map[string]models.Engine 69 69 - jq *queue.Queue 70 70 - cfg *config.Config 71 71 - ks *eventconsumer.Consumer 72 72 - res *idresolver.Resolver 73 73 - verify repoverify.Verifier 74 74 - vault secrets.Manager 75 75 - motd []byte 76 76 - motdMu sync.RWMutex 77 77 - rootCtx context.Context 78 78 - store artifactstore.Store 79 79 - stores *artifactstore.Stores 80 80 - reader artifactstore.Reader 81 81 - // set only when this spindle hosts the mill or joins one as an executor 82 82 - mill *mill.Mill 83 83 - exec *executor.Executor 84 84 - } 85 85 - 86 86 - // New creates a new Spindle server with the provided configuration and engines. 87 87 - func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { 88 88 - logger := log.FromContext(ctx) 89 89 - n := notifier.New() 90 90 - 91 91 - if cfg.Role == config.RoleExecutor { 92 92 - if err := cleanupOrphanRepos(ctx, d, logger); err != nil { 93 93 - return nil, fmt.Errorf("failed to run startup cleanup: %w", err) 94 94 - } 95 95 - } else if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { 96 96 - return nil, fmt.Errorf("failed to run startup migrations: %w", err) 97 97 - } 98 98 - 99 99 - spindle := &Spindle{ 100 100 - db: d, 101 101 - l: logger, 102 102 - n: &n, 103 103 - engs: engines, 104 104 - cfg: cfg, 105 105 - motd: defaultMotd, 106 106 - rootCtx: ctx, 107 107 - } 108 108 - diskFallback := "" 109 109 - if cfg.Role == config.RoleStandalone { 110 110 - diskFallback = cfg.Server.LogDir 111 111 - if cfg.ArtifactStores.Disk.Dir == "" { 112 112 - logger.Warn("using SPINDLE_SERVER_LOG_DIR as the implicit disk artifact store; configure SPINDLE_ARTIFACT_STORES_DISK_DIR explicitly") 113 113 - } 114 114 - } 115 115 - stores, err := artifactstore.NewStores(cfg.ArtifactStores, diskFallback, cfg.LegacyS3.LogBucket) 116 116 - if err != nil { 117 117 - return nil, fmt.Errorf("failed to setup artifact stores: %w", err) 118 118 - } 119 119 - spindle.stores = stores 120 120 - if cfg.LegacyS3.LogBucket != "" { 121 121 - logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") 122 122 - } 123 123 - if cfg.Role == config.RoleStandalone { 124 124 - spindle.reader = stores 125 125 - } else { 126 126 - name := cfg.Mill.ArtifactStore 127 127 - if name == "" { 128 128 - names := stores.Names() 129 129 - if len(names) != 1 { 130 130 - return nil, fmt.Errorf("%s requires SPINDLE_MILL_ARTIFACT_STORE when %d artifact stores are configured", cfg.Role, len(names)) 131 131 - } 132 132 - name = names[0] 133 133 - logger.Warn("SPINDLE_MILL_ARTIFACT_STORE is not set; inferred the only configured store", "store", name) 134 134 - } 135 135 - store, ok := stores.Store(name) 136 136 - if !ok { 137 137 - return nil, fmt.Errorf("SPINDLE_MILL_ARTIFACT_STORE=%q is not configured", name) 138 138 - } 139 139 - spindle.store = store 140 140 - spindle.reader = store 141 141 - } 142 142 - if cfg.Role == config.RoleExecutor { 143 143 - return spindle, nil 144 144 - } 145 145 - 146 146 - e, err := rbac.NewEnforcer(cfg.Server.DBPath) 147 147 - if err != nil { 148 148 - return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) 149 149 - } 150 150 - e.E.EnableAutoSave(true) 151 151 - spindle.e = e 152 152 - 153 153 - switch cfg.Server.Secrets.Provider { 154 154 - case "openbao": 155 155 - if cfg.Server.Secrets.OpenBao.ProxyAddr == "" { 156 156 - return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider") 157 157 - } 158 158 - spindle.vault, err = secrets.NewOpenBaoManager( 159 159 - cfg.Server.Secrets.OpenBao.ProxyAddr, 160 160 - logger, 161 161 - secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount), 162 162 - ) 163 163 - if err != nil { 164 164 - return nil, fmt.Errorf("failed to setup openbao secrets provider: %w", err) 165 165 - } 166 166 - logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount) 167 167 - case "sqlite", "": 168 168 - spindle.vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) 169 169 - if err != nil { 170 170 - return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err) 171 171 - } 172 172 - logger.Info("using sqlite secrets provider", "path", cfg.Server.DBPath) 173 173 - default: 174 174 - return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) 175 175 - } 176 176 - 177 177 - if cfg.Role == config.RoleStandalone { 178 178 - spindle.jq = queue.NewQueue(cfg.Server.QueueSize, cfg.Server.MaxJobCount) 179 179 - logger.Info("initialized queue", "queueSize", cfg.Server.QueueSize, "numWorkers", cfg.Server.MaxJobCount) 180 180 - } 181 181 - 182 182 - collections := []string{ 183 183 - tangled.SpindleMemberNSID, 184 184 - tangled.RepoNSID, 185 185 - tangled.RepoCollaboratorNSID, 186 186 - tangled.RepoPullNSID, 187 187 - } 188 188 - jc, err := jetstream.NewJetstreamClient(cfg.Server.JetstreamEndpoint, "spindle", collections, nil, log.SubLogger(logger, "jetstream"), d, true, true) 189 189 - if err != nil { 190 190 - return nil, fmt.Errorf("failed to setup jetstream client: %w", err) 191 191 - } 192 192 - spindle.jc = jc 193 193 - jc.AddDid(cfg.Server.Owner) 194 194 - // pull records are created by arbitrary users too, same hack as in tap 195 195 - jc.ExemptCollection(tangled.RepoPullNSID) 196 196 - 197 197 - // Check if the spindle knows about any Dids; 198 198 - dids, err := d.GetAllDids() 199 199 - if err != nil { 200 200 - return nil, fmt.Errorf("failed to get all dids: %w", err) 201 201 - } 202 202 - for _, d := range dids { 203 203 - jc.AddDid(d) 204 204 - } 205 205 - 206 206 - knownRepos, err := d.AllRepos() 207 207 - if err != nil { 208 208 - return nil, fmt.Errorf("failed to get known repos: %w", err) 209 209 - } 210 210 - for _, r := range knownRepos { 211 211 - if r.Owner != "" { 212 212 - jc.AddDid(r.Owner.String()) 213 213 - } 214 214 - } 215 215 - 216 216 - spindle.res = idresolver.DefaultResolver(cfg.Server.PlcUrl) 217 217 - spindle.verify = repoverify.New(spindle.res, cfg.Server.Dev) 218 218 - 219 219 - err = e.AddSpindle(rbacDomain) 220 220 - if err != nil { 221 221 - return nil, fmt.Errorf("failed to set rbac domain: %w", err) 222 222 - } 223 223 - err = spindle.configureOwner() 224 224 - if err != nil { 225 225 - return nil, err 226 226 - } 227 227 - logger.Info("owner set", "did", cfg.Server.Owner) 228 228 - 229 229 - cursorStore, err := cursor.NewSQLiteStore(cfg.Server.DBPath) 230 230 - if err != nil { 231 231 - return nil, fmt.Errorf("failed to setup sqlite3 cursor store: %w", err) 232 232 - } 233 233 - 234 234 - err = jc.StartJetstream(ctx, spindle.ingest()) 235 235 - if err != nil { 236 236 - return nil, fmt.Errorf("failed to start jetstream consumer: %w", err) 237 237 - } 238 238 - 239 239 - // spindle listen to knot stream for sh.tangled.git.refUpdate 240 240 - // which will sync the local workflow files in spindle and enqueues the 241 241 - // pipeline job for on-push workflows 242 242 - ccfg := eventconsumer.NewConsumerConfig() 243 243 - ccfg.Logger = log.SubLogger(logger, "eventconsumer") 244 244 - ccfg.ProcessFunc = spindle.processKnotStream 245 245 - ccfg.CursorStore = cursorStore 246 246 - if cfg.Server.Dev { 247 247 - ccfg.RetryInterval = 5 * time.Second 248 248 - ccfg.MaxRetryInterval = 10 * time.Second 249 249 - } else { 250 250 - ccfg.RetryInterval = 1 * time.Minute 251 251 - ccfg.MaxRetryInterval = 10 * time.Minute 252 252 - } 253 253 - knownKnots, err := d.Knots() 254 254 - if err != nil { 255 255 - return nil, err 256 256 - } 257 257 - for _, knot := range knownKnots { 258 258 - logger.Info("adding source start", "knot", knot) 259 259 - src := eventconsumer.NewKnotSource(knot) 260 260 - eventconsumer.MigrateLegacyCursor(cursorStore, src) 261 261 - ccfg.Sources[src] = struct{}{} 262 262 - } 263 263 - spindle.ks = eventconsumer.NewConsumer(*ccfg) 264 264 - if cfg.Server.Tap.Embed { 265 265 - pw, err := randomAdminPassword() 266 266 - if err != nil { 267 267 - return nil, err 268 268 - } 269 269 - cfg.Server.Tap.AdminPassword = pw 270 270 - logger.Info("embedded tap: using random admin password") 271 271 - } 272 272 - spindle.tap = NewTapClient(spindle) 273 273 - 274 274 - return spindle, nil 275 275 - } 276 276 - func (s *Spindle) DB() *db.DB { 277 277 - return s.db 278 278 - } 279 279 - func (s *Spindle) Queue() *queue.Queue { 280 280 - return s.jq 281 281 - } 282 282 - 283 283 - // Engines returns the map of available engines. 284 284 - func (s *Spindle) Engines() map[string]models.Engine { 285 285 - return s.engs 286 286 - } 287 287 - 288 288 - // Vault returns the secrets manager instance. 289 289 - func (s *Spindle) Vault() secrets.Manager { 290 290 - return s.vault 291 291 - } 292 292 - 293 293 - // Notifier returns the notifier instance. 294 294 - func (s *Spindle) Notifier() *notifier.Notifier { 295 295 - return s.n 296 296 - } 297 297 - 298 298 - // Enforcer returns the RBAC enforcer instance. 299 299 - func (s *Spindle) Enforcer() *rbac.Enforcer { 300 300 - return s.e 301 301 - } 302 302 - 303 303 - // SetMotdContent sets custom MOTD content, replacing the embedded default. 304 304 - func (s *Spindle) SetMotdContent(content []byte) { 305 305 - s.motdMu.Lock() 306 306 - defer s.motdMu.Unlock() 307 307 - s.motd = content 308 308 - } 309 309 - 310 310 - // GetMotdContent returns the current MOTD content. 311 311 - func (s *Spindle) GetMotdContent() []byte { 312 312 - s.motdMu.RLock() 313 313 - defer s.motdMu.RUnlock() 314 314 - return s.motd 315 315 - } 316 316 - 317 317 - // runs the server. blocks 318 318 - func (s *Spindle) Start(ctx context.Context) error { 319 319 - // only standalone runs the local queue. mill hosts place directly onto 320 320 - // executors, and executors only run jobs explicitly assigned by a mill 321 321 - if s.cfg.Role == config.RoleStandalone { 322 322 - if s.jq != nil { 323 323 - s.jq.Start() 324 324 - defer s.jq.Stop() 325 325 - } 326 326 - } 327 327 - 328 328 - // an executor dials out to its mill and takes work from it 329 329 - if s.exec != nil { 330 330 - go s.exec.Connect(ctx) 331 331 - } 332 332 - 333 333 - if stopper, ok := s.vault.(secrets.Stopper); ok { 334 334 - defer stopper.Stop() 335 335 - } 336 336 - 337 337 - if s.cfg.Role != config.RoleExecutor { 338 338 - tapCtx, tapCancel := context.WithCancel(ctx) 339 339 - 340 340 - if s.cfg.Server.Tap.Embed { 341 341 - emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) 342 342 - if err != nil { 343 343 - tapCancel() 344 344 - return fmt.Errorf("starting embedded tap: %w", err) 345 345 - } 346 346 - s.embedTap = emb 347 347 - defer func() { 348 348 - tapCancel() 349 349 - s.embedTap.Shutdown() 350 350 - }() 351 351 - 352 352 - go s.watchTapDrain(tapCtx, tapCancel) 353 353 - } else { 354 354 - defer tapCancel() 355 355 - } 356 356 - 357 357 - go func() { 358 358 - s.l.Info("starting knot event consumer") 359 359 - s.ks.Start(ctx) 360 360 - }() 361 361 - 362 362 - s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) 363 363 - s.tap.Start(tapCtx) 364 364 - } 365 365 - 366 366 - s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr) 367 367 - return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router()) 368 368 - } 369 369 - 370 370 - func (s *Spindle) declareTapInterest(ctx context.Context) { 371 371 - repos, err := s.db.AllRepos() 372 372 - if err != nil { 373 373 - s.l.Warn("tap declare: failed to load known repos", "err", err) 374 374 - return 375 375 - } 376 376 - seen := make(map[syntax.DID]struct{}, len(repos)) 377 377 - dids := make([]syntax.DID, 0, len(repos)) 378 378 - for _, r := range repos { 379 379 - if r.Owner == "" { 380 380 - continue 381 381 - } 382 382 - if _, ok := seen[r.Owner]; ok { 383 383 - continue 384 384 - } 385 385 - seen[r.Owner] = struct{}{} 386 386 - dids = append(dids, r.Owner) 387 387 - } 388 388 - if err := s.tap.AddOwnerDIDs(ctx, dids); err != nil { 389 389 - s.l.Warn("tap declare: AddRepos rejected", "count", len(dids), "err", err) 390 390 - return 391 391 - } 392 392 - s.l.Info("tap declare: known owner DIDs registered", "count", len(dids)) 393 393 - } 394 394 - 395 395 - func Run(ctx context.Context) error { 396 396 - cfg, err := config.Load(ctx) 397 397 - if err != nil { 398 398 - return fmt.Errorf("failed to load config: %w", err) 399 399 - } 400 400 - 401 401 - if err := ensureGitVersion(); err != nil { 402 402 - return fmt.Errorf("ensuring git version: %w", err) 403 403 - } 404 404 - 405 405 - d, err := db.Make(ctx, cfg.Server.DBPath) 406 406 - if err != nil { 407 407 - return fmt.Errorf("failed to setup db: %w", err) 408 408 - } 409 409 - 410 410 - logger := log.FromContext(ctx) 411 411 - 412 412 - var engines map[string]models.Engine 413 413 - var m *mill.Mill 414 414 - 415 415 - if cfg.Role == config.RoleMill { 416 416 - // mill host: register engines that place jobs on executors instead of 417 417 - // running them. all names share one Mill 418 418 - m = mill.New(log.SubLogger(logger, "mill"), mill.Config{ 419 419 - LogDir: cfg.Server.LogDir, 420 420 - MaxPending: cfg.Mill.MaxPending, 421 421 - ReconnectGrace: cfg.Mill.ReconnectGrace, 422 422 - }) 423 423 - engines = map[string]models.Engine{ 424 424 - "nixery": mill.NewEngine("nixery", m), 425 425 - "microvm": mill.NewEngine("microvm", m), 426 426 - "dummy": mill.NewEngine("dummy", m), 427 427 - } 428 428 - } else { 429 429 - // standalone and executor both run real engines locally. 430 430 - nixeryEng, err := nixery.New(ctx, cfg) 431 431 - if err != nil { 432 432 - return err 433 433 - } 434 434 - microvmEng, err := newMicrovmEngine(ctx, cfg, d) 435 435 - if err != nil { 436 436 - return err 437 437 - } 438 438 - engines = map[string]models.Engine{ 439 439 - "nixery": nixeryEng, 440 440 - "microvm": microvmEng, 441 441 - "dummy": dummy.New(logger), 442 442 - } 443 443 - } 444 444 - 445 445 - s, err := New(ctx, cfg, d, engines) 446 446 - if err != nil { 447 447 - return err 448 448 - } 449 449 - 450 450 - if m != nil { 451 451 - // resolve the chicken-and-egg: the engines (built above) hold the mill, 452 452 - // but the mill's db/notifier are created inside New 453 453 - m.Attach(s.DB(), s.Notifier()) 454 454 - s.mill = m 455 455 - if err := m.RestoreState(); err != nil { 456 456 - return fmt.Errorf("restoring mill state: %w", err) 457 457 - } 458 458 - } 459 459 - if cfg.Role == config.RoleExecutor { 460 460 - s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store) 461 461 - if err != nil { 462 462 - return err 463 463 - } 464 464 - } 465 465 - 466 466 - return s.Start(ctx) 467 467 - } 468 468 - 469 469 - func (s *Spindle) Router() http.Handler { 470 470 - mux := chi.NewRouter() 471 471 - 472 472 - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 473 473 - w.Write(s.GetMotdContent()) 474 474 - }) 475 475 - if s.cfg.Role == config.RoleExecutor { 476 476 - return mux 477 477 - } 478 478 - 479 479 - mux.HandleFunc("/events", s.Events) 480 480 - mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) 481 481 - 482 482 - // mill host: executors dial in here (plain ws, shared-secret auth) 483 483 - if s.mill != nil { 484 484 - mux.HandleFunc("/mill", s.mill.HandleExecutorConn) 485 485 - } 486 486 - 487 487 - mux.Mount("/xrpc", s.XrpcRouter()) 488 488 - return mux 489 489 - } 490 490 - 491 491 - func (s *Spindle) XrpcRouter() http.Handler { 492 492 - serviceAuth := serviceauth.NewServiceAuth(s.l, s.res.Directory(), s.cfg.Server.Did().String()) 493 493 - 494 494 - l := log.SubLogger(s.l, "xrpc") 495 495 - 496 496 - x := xrpc.Xrpc{ 497 497 - Logger: l, 498 498 - Db: s.db, 499 499 - Enforcer: s.e, 500 500 - Engines: s.engs, 501 501 - Config: s.cfg, 502 502 - ArtifactReader: s.reader, 503 503 - Resolver: s.res, 504 504 - Vault: s.vault, 505 505 - Notifier: s.Notifier(), 506 506 - ServiceAuth: serviceAuth, 507 507 - Trigger: s, 508 508 - } 509 509 - 510 510 - return x.Router() 511 511 - } 512 512 - 513 513 - func (s *Spindle) processKnotStream(ctx context.Context, src eventconsumer.Source, msg eventstream.Event) error { 514 514 - l := log.FromContext(ctx).With("handler", "processKnotStream") 515 515 - l = l.With("src", src.Key(), "msg.Nsid", msg.Nsid, "msg.Rkey", msg.Rkey) 516 516 - if msg.Nsid == knotdb.RepoCollaboratorUpdateNSID { 517 517 - return s.ingestKnotCollaborator(ctx, l, src, msg) 518 518 - } 519 519 - if msg.Nsid == tangled.GitRefUpdateNSID { 520 520 - event := tangled.GitRefUpdate{} 521 521 - if err := json.Unmarshal(msg.EventJson, &event); err != nil { 522 522 - l.Error("error unmarshalling", "err", err) 523 523 - return err 524 524 - } 525 525 - l = l.With("repo", event.Repo, "ref", event.Ref, "newSha", event.NewSha) 526 526 - l.Debug("debug") 527 527 - 528 528 - repoDid := syntax.DID(event.Repo) 529 529 - repo, err := s.db.GetRepoByDid(repoDid) 530 530 - if err != nil { 531 531 - return fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 532 532 - } 533 533 - 534 534 - if src.Host != repo.Knot { 535 535 - return fmt.Errorf("repo knot does not match event source: %s != %s", src.Host, repo.Knot) 536 536 - } 537 537 - 538 538 - if kgit.HasSkipCIPushOption(event.PushOptions) { 539 539 - l.Info("push event requested ci skip, skipping the event") 540 540 - return nil 541 541 - } 542 542 - 543 543 - // NOTE: we are blindly trusting the knot that it will return only repos it own 544 544 - repoCloneUri := s.newRepoCloneUrl(src.Host, repoDid) 545 545 - repoPath := s.newRepoPath(repoDid) 546 546 - if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, event.NewSha); err != nil { 547 547 - return fmt.Errorf("sync git repo: %w", err) 548 548 - } 549 549 - l.Info("synced git repo") 550 550 - 551 551 - triggerRepo, err := s.buildTriggerRepo(ctx, repo) 552 552 - if err != nil { 553 553 - return fmt.Errorf("building trigger repo: %w", err) 554 554 - } 555 555 - 556 556 - trigger := tangled.Pipeline_TriggerMetadata{ 557 557 - Kind: string(workflow.TriggerKindPush), 558 558 - Push: &tangled.Pipeline_PushTriggerData{ 559 559 - Ref: event.Ref, 560 560 - OldSha: event.OldSha, 561 561 - NewSha: event.NewSha, 562 562 - }, 563 563 - Repo: triggerRepo, 564 564 - } 565 565 - 566 566 - pipelineId, err := s.runPipeline(ctx, repoDid, trigger, event.ChangedFiles, repoCloneUri, repoPath, event.NewSha, nil, triggerRepo) 567 567 - if err != nil { 568 568 - return err 569 569 - } 570 570 - if pipelineId.Rkey == "" { 571 571 - l.Info("no workflow matched 'push' trigger, skipping the event") 572 572 - return nil 573 573 - } 574 574 - l.Info("pipeline triggered", "pipeline", pipelineId.AtUri()) 575 575 - } 576 576 - 577 577 - return nil 578 578 - } 579 579 - 580 580 - func (s *Spindle) ingestKnotCollaborator(ctx context.Context, l *slog.Logger, src eventconsumer.Source, msg eventstream.Event) error { 581 581 - var rec knotdb.RepoCollaboratorUpdate 582 582 - if err := json.Unmarshal(msg.EventJson, &rec); err != nil { 583 583 - l.Error("error unmarshalling collaboratorUpdate", "err", err) 584 584 - return err 585 585 - } 586 586 - 587 587 - subject, err := syntax.ParseDID(rec.Subject) 588 588 - if err != nil { 589 589 - l.Info("skipping collaboratorUpdate with malformed subject", "subject", rec.Subject, "err", err) 590 590 - return nil 591 591 - } 592 592 - repoDid, err := syntax.ParseDID(rec.Repo) 593 593 - if err != nil { 594 594 - l.Info("skipping collaboratorUpdate with malformed repo", "repo", rec.Repo, "err", err) 595 595 - return nil 596 596 - } 597 597 - 598 598 - repo, err := s.db.GetRepoByDid(repoDid) 599 599 - if errors.Is(err, sql.ErrNoRows) { 600 600 - l.Info("skipping collaboratorUpdate for unknown repo", "repo", repoDid) 601 601 - return nil 602 602 - } 603 603 - if err != nil { 604 604 - return fmt.Errorf("lookup repo %s: %w", repoDid, err) 605 605 - } 606 606 - if src.Host != repo.Knot { 607 607 - l.Warn("dropping collaboratorUpdate from non-owning knot", "src", src.Host, "repoKnot", repo.Knot) 608 608 - return nil 609 609 - } 610 610 - 611 611 - switch rec.Op { 612 612 - case knotdb.AclOpAdd: 613 613 - if err := s.e.AddCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { 614 614 - return fmt.Errorf("add collaborator policy: %w", err) 615 615 - } 616 616 - if err := s.db.AddKnotCollaborator(repoDid, subject); err != nil { 617 617 - return fmt.Errorf("track collaborator: %w", err) 618 618 - } 619 619 - l.Info("added knot-managed collaborator", "subject", subject, "repo", repoDid) 620 620 - case knotdb.AclOpRemove: 621 621 - if err := s.e.RemoveCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { 622 622 - return fmt.Errorf("remove collaborator policy: %w", err) 623 623 - } 624 624 - if err := s.db.DeleteRepoCollaboratorBySubjectRepo(subject, repoDid); err != nil { 625 625 - return fmt.Errorf("delete collaborator row: %w", err) 626 626 - } 627 627 - l.Info("removed knot-managed collaborator", "subject", subject, "repo", repoDid) 628 628 - default: 629 629 - return fmt.Errorf("collaboratorUpdate unknown op %q", rec.Op) 630 630 - } 631 631 - return nil 632 632 - } 633 633 - 634 634 - // buildTriggerRepo gathers trigger metadata, resolving default branch from the knot 635 635 - func (s *Spindle) buildTriggerRepo(ctx context.Context, repo *db.Repo) (*tangled.Pipeline_TriggerRepo, error) { 636 636 - rkey := string(repo.Rkey) 637 637 - repoDid := repo.RepoDid.String() 638 638 - return s.buildTriggerRepoFrom(ctx, repo.Knot, repo.Owner.String(), rkey, repoDid), nil 639 639 - } 640 640 - 641 641 - func (s *Spindle) buildTriggerRepoFrom(ctx context.Context, knot, did, rkey, repoDid string) *tangled.Pipeline_TriggerRepo { 642 642 - scheme := "https" 643 643 - if s.cfg.Server.Dev { 644 644 - scheme = "http" 645 645 - } 646 646 - client := &indigoxrpc.Client{Host: fmt.Sprintf("%s://%s", scheme, knot)} 647 647 - 648 648 - // this should maybe (?) be in the refUpdate event itself to save a roundtrip 649 649 - defaultBranch := "" 650 650 - if out, err := tangled.RepoGetDefaultBranch(ctx, client, repoDid); err == nil { 651 651 - defaultBranch = out.Name 652 652 - } 653 653 - 654 654 - var rkeyPtr *string 655 655 - if rkey != "" { 656 656 - rkeyPtr = &rkey 657 657 - } 658 658 - return &tangled.Pipeline_TriggerRepo{ 659 659 - Did: did, 660 660 - Knot: knot, 661 661 - Repo: rkeyPtr, 662 662 - RepoDid: &repoDid, 663 663 - DefaultBranch: defaultBranch, 664 664 - } 665 665 - } 666 666 - 667 667 - func (s *Spindle) resolvePipelineSourceRepo(ctx context.Context, trigger *tangled.Pipeline_TriggerMetadata) (*tangled.Pipeline_TriggerRepo, error) { 668 668 - if trigger == nil { 669 669 - return nil, nil 670 670 - } 671 671 - if trigger.SourceRepo == nil || *trigger.SourceRepo == "" { 672 672 - return trigger.Repo, nil 673 673 - } 674 674 - repoDid, err := syntax.ParseDID(*trigger.SourceRepo) 675 675 - if err != nil { 676 676 - return nil, fmt.Errorf("parse sourceRepo %s: %w", *trigger.SourceRepo, err) 677 677 - } 678 678 - return s.resolveSourceRepoInfo(ctx, repoDid) 679 679 - } 680 680 - 681 681 - // resolveSourceRepoInfo resolves trigger-repo metadata for a source repo DID. 682 682 - func (s *Spindle) resolveSourceRepoInfo(ctx context.Context, repoDid syntax.DID) (*tangled.Pipeline_TriggerRepo, error) { 683 683 - repo, err := s.db.GetRepoByDid(repoDid) 684 684 - if err == nil { 685 685 - return s.buildTriggerRepo(ctx, repo) 686 686 - } 687 687 - 688 688 - // verify repo, we don't want git sync to point to arbitrary endpoints 689 689 - res, err := s.verify(ctx, repoident.RepoDid(repoDid)) 690 690 - if err != nil { 691 691 - return nil, fmt.Errorf("verify sourceRepo %s: %w", repoDid, err) 692 692 - } 693 693 - return s.buildTriggerRepoFrom(ctx, res.KnotURL.Host, res.OwnerDid.String(), res.Rkey, repoDid.String()), nil 694 694 - } 695 695 - 696 696 - // runPipeline compiles and enqueues the pipeline for the given revision. 697 697 - // sourceRepo is the resolved repo the code was checked out from, forwarded to 698 698 - // processPipeline for env vars. 699 699 - func (s *Spindle) runPipeline(ctx context.Context, repoDid syntax.DID, trigger tangled.Pipeline_TriggerMetadata, changedFiles []string, repoCloneUri, repoPath, rev string, only []string, sourceRepo *tangled.Pipeline_TriggerRepo) (models.PipelineId, error) { 700 700 - l := log.FromContext(ctx) 701 701 - 702 702 - compiler := workflow.Compiler{ 703 703 - ChangedFiles: changedFiles, 704 704 - Trigger: trigger, 705 705 - } 706 706 - 707 707 - rawPipeline, err := s.loadPipeline(ctx, repoCloneUri, repoPath, rev) 708 708 - if err != nil { 709 709 - return models.PipelineId{}, fmt.Errorf("loading pipeline: %w", err) 710 710 - } 711 711 - if len(rawPipeline) == 0 { 712 712 - return models.PipelineId{}, nil 713 713 - } 714 714 - 715 715 - tpl := compiler.Compile(compiler.Parse(rawPipeline)) 716 716 - // todo(dawn): pass compile error to workflow log 717 717 - for _, w := range compiler.Diagnostics.Errors { 718 718 - l.Error(w.String()) 719 719 - } 720 720 - for _, w := range compiler.Diagnostics.Warnings { 721 721 - l.Warn(w.String()) 722 722 - } 723 723 - 724 724 - if len(only) > 0 { 725 725 - tpl.Workflows = filterWorkflows(tpl.Workflows, only) 726 726 - } 727 727 - if len(tpl.Workflows) == 0 { 728 728 - return models.PipelineId{}, nil 729 729 - } 730 730 - 731 731 - pipelineId := models.PipelineId{ 732 732 - Knot: trigger.Repo.Knot, 733 733 - Rkey: tid.TID(), 734 734 - } 735 735 - if err := s.db.CreatePipelineEvent(pipelineId.Rkey, tpl, s.n); err != nil { 736 736 - return models.PipelineId{}, fmt.Errorf("creating pipeline event: %w", err) 737 737 - } 738 738 - err = s.processPipeline(repoDid, tpl, pipelineId, sourceRepo) 739 739 - return pipelineId, err 740 740 - } 741 741 - 742 742 - // filterWorkflows filters workflows to the requested names 743 743 - func filterWorkflows(workflows []*tangled.Pipeline_Workflow, only []string) []*tangled.Pipeline_Workflow { 744 744 - allowed := make(map[string]struct{}, len(only)) 745 745 - for _, n := range only { 746 746 - allowed[n] = struct{}{} 747 747 - } 748 748 - var filtered []*tangled.Pipeline_Workflow 749 749 - for _, w := range workflows { 750 750 - if w == nil { 751 751 - continue 752 752 - } 753 753 - if _, ok := allowed[w.Name]; ok { 754 754 - filtered = append(filtered, w) 755 755 - } 756 756 - } 757 757 - return filtered 758 758 - } 759 759 - 760 760 - // TriggerManual dispatches a pipeline at sha, authorized against and recorded 761 761 - // under repoDid. sourceRepo, pull, and inputs are optional trigger payload. 762 762 - func (s *Spindle) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull xrpc.PullContext, inputs []*tangled.Pipeline_Pair) (syntax.ATURI, error) { 763 763 - repo, err := s.db.GetRepoByDid(repoDid) 764 764 - if err != nil { 765 765 - return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) 766 766 - } 767 767 - 768 768 - triggerRepo, err := s.buildTriggerRepo(ctx, repo) 769 769 - if err != nil { 770 770 - return "", fmt.Errorf("building trigger repo: %w", err) 771 771 - } 772 772 - 773 773 - trigger := tangled.Pipeline_TriggerMetadata{Repo: triggerRepo} 774 774 - if pull.IsPullRequest { 775 775 - var pullAt *string 776 776 - if pull.Pull != "" { 777 777 - pullAtStr := pull.Pull.String() 778 778 - pullAt = &pullAtStr 779 779 - } 780 780 - trigger.Kind = string(workflow.TriggerKindPullRequest) 781 781 - trigger.PullRequest = &tangled.Pipeline_PullRequestTriggerData{ 782 782 - SourceBranch: pull.SourceBranch, 783 783 - TargetBranch: pull.TargetBranch, 784 784 - SourceSha: sha, 785 785 - Pull: pullAt, 786 786 - } 787 787 - } else { 788 788 - var refPtr *string 789 789 - if ref != "" { 790 790 - refPtr = &ref 791 791 - } 792 792 - trigger.Kind = string(workflow.TriggerKindManual) 793 793 - trigger.Manual = &tangled.Pipeline_ManualTriggerData{ 794 794 - Sha: sha, 795 795 - Ref: refPtr, 796 796 - Inputs: inputs, 797 797 - } 798 798 - } 799 799 - 800 800 - repoCloneUri := s.newRepoCloneUrl(repo.Knot, repoDid) 801 801 - repoPath := s.newRepoPath(repoDid) 802 802 - sourceInfo := triggerRepo // default: code comes from the repo itself 803 803 - if sourceRepo != "" && sourceRepo != repoDid { 804 804 - sourceInfo, err = s.resolveSourceRepoInfo(ctx, sourceRepo) 805 805 - if err != nil { 806 806 - return "", err 807 807 - } 808 808 - sourceRepoStr := sourceRepo.String() 809 809 - trigger.SourceRepo = &sourceRepoStr 810 810 - repoCloneUri = models.BuildRepoURL(sourceInfo) 811 811 - repoPath = s.newRepoPath(sourceRepo) 812 812 - } 813 813 - 814 814 - pipelineId, err := s.runPipeline(ctx, repoDid, trigger, nil, repoCloneUri, repoPath, sha, workflows, sourceInfo) 815 815 - if err != nil { 816 816 - return "", err 817 817 - } 818 818 - if pipelineId.Rkey == "" { 819 819 - return "", xrpc.ErrNoMatchingWorkflows 820 820 - } 821 821 - return pipelineId.AtUri(), nil 822 822 - } 823 823 - 824 824 - func (s *Spindle) loadPipeline(ctx context.Context, repoUri, repoPath, rev string) (workflow.RawPipeline, error) { 825 825 - if err := git.SparseSyncGitRepo(ctx, repoUri, repoPath, rev); err != nil { 826 826 - return nil, fmt.Errorf("syncing git repo: %w", err) 827 827 - } 828 828 - gr, err := kgit.Open(repoPath, rev) 829 829 - if err != nil { 830 830 - return nil, fmt.Errorf("opening git repo: %w", err) 831 831 - } 832 832 - 833 833 - workflowDir, err := gr.FileTree(ctx, workflow.WorkflowDir) 834 834 - if errors.Is(err, object.ErrDirectoryNotFound) { 835 835 - // return empty RawPipeline when directory doesn't exist 836 836 - return nil, nil 837 837 - } else if err != nil { 838 838 - return nil, fmt.Errorf("loading file tree: %w", err) 839 839 - } 840 840 - 841 841 - var rawPipeline workflow.RawPipeline 842 842 - for _, e := range workflowDir { 843 843 - if !e.IsFile() { 844 844 - continue 845 845 - } 846 846 - 847 847 - fpath := filepath.Join(workflow.WorkflowDir, e.Name) 848 848 - contents, err := gr.RawContent(fpath) 849 849 - if err != nil { 850 850 - return nil, fmt.Errorf("reading raw content of '%s': %w", fpath, err) 851 851 - } 852 852 - 853 853 - rawPipeline = append(rawPipeline, workflow.RawWorkflow{ 854 854 - Name: e.Name, 855 855 - Contents: contents, 856 856 - }) 857 857 - } 858 858 - 859 859 - return rawPipeline, nil 860 860 - } 861 861 - 862 862 - // processPipeline enqueues the workflows in tpl. 863 863 - func (s *Spindle) processPipeline(repoDid syntax.DID, tpl tangled.Pipeline, pipelineId models.PipelineId, sourceRepo *tangled.Pipeline_TriggerRepo) error { 864 864 - // derive security-relevant things like whether this run is trusted and can be passed 865 865 - // secrets to from the original metadata. 866 866 - pipelineEnv := models.PipelineEnvVarsForSource(tpl.TriggerMetadata, pipelineId, sourceRepo) 867 867 - trustedSource := true 868 868 - if tm := tpl.TriggerMetadata; tm != nil && tm.SourceRepo != nil && 869 869 - *tm.SourceRepo != "" && *tm.SourceRepo != repoDid.String() { 870 870 - trustedSource = false 871 871 - } 872 872 - 873 873 - // swap the repo with our sourceRepo if we are running a pipeline on a fork. 874 874 - // the metadata stays the same. we check whether the repo is trusted above, 875 875 - // so this only affects the clone URL. 876 876 - initTpl := tpl 877 877 - if sourceRepo != nil && tpl.TriggerMetadata != nil { 878 878 - tm := *tpl.TriggerMetadata 879 879 - tm.Repo = sourceRepo 880 880 - initTpl.TriggerMetadata = &tm 881 881 - } 882 882 - 883 883 - // filter & init workflows 884 884 - workflows := make(map[models.Engine][]models.Workflow) 885 885 - for _, w := range tpl.Workflows { 886 886 - if w == nil { 887 887 - continue 888 888 - } 889 889 - eng, ok := s.engs[w.Engine] 890 890 - if !ok { 891 891 - err := s.db.StatusFailed(models.WorkflowId{ 892 892 - PipelineId: pipelineId, 893 893 - Name: w.Name, 894 894 - }, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n) 895 895 - if err != nil { 896 896 - return fmt.Errorf("db.StatusFailed: %w", err) 897 897 - } 898 898 - 899 899 - continue 900 900 - } 901 901 - 902 902 - ewf, err := eng.InitWorkflow(*w, initTpl) 903 903 - if err != nil { 904 904 - err = s.db.StatusFailed(models.WorkflowId{ 905 905 - PipelineId: pipelineId, 906 906 - Name: w.Name, 907 907 - }, fmt.Sprintf("init workflow: %s", err), -1, s.n) 908 908 - if err != nil { 909 909 - return fmt.Errorf("db.StatusFailed: %w", err) 910 910 - } 911 911 - 912 912 - continue 913 913 - } 914 914 - 915 915 - // inject TANGLED_* env vars after InitWorkflow 916 916 - // This prevents user-defined env vars from overriding them 917 917 - if ewf.Environment == nil { 918 918 - ewf.Environment = make(map[string]string) 919 919 - } 920 920 - maps.Copy(ewf.Environment, pipelineEnv) 921 921 - 922 922 - workflows[eng] = append(workflows[eng], *ewf) 923 923 - } 924 924 - 925 925 - pipeline := &models.Pipeline{ 926 926 - RepoDid: repoDid, 927 927 - Workflows: workflows, 928 928 - TrustedSource: trustedSource, 929 929 - } 930 930 - 931 931 - if s.mill != nil { 932 932 - // mill host: no bounded pool. each job blocks in placement 933 933 - // (AcquireWorkflowSlot) which the user sees as pending. the only bound 934 934 - // is the mill's maxPending. rootCtx is the long-lived consumer context, 935 935 - // so the goroutine safely outlives this call 936 936 - go engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.rootCtx, pipeline, pipelineId) 937 937 - s.l.Info("pipeline handed to mill placement", "id", pipelineId) 938 938 - } else if s.jq != nil { 939 939 - ok := s.jq.Enqueue(repoDid, queue.Job{ 940 940 - Run: func() error { 941 941 - engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.rootCtx, pipeline, pipelineId) 942 942 - return nil 943 943 - }, 944 944 - OnFail: func(jobError error) { 945 945 - s.l.Error("pipeline run failed", "error", jobError) 946 946 - }, 947 947 - }) 948 948 - if !ok { 949 949 - return fmt.Errorf("failed to enqueue pipeline: queue is full") 950 950 - } 951 951 - s.l.Info("pipeline enqueued successfully", "id", pipelineId) 952 952 - } else { 953 953 - return fmt.Errorf("no queue or mill available to process pipeline") 954 954 - } 955 955 - 956 956 - // after successful enqueue, emit StatusPending for all workflows 957 957 - for _, ewfs := range workflows { 958 958 - for _, ewf := range ewfs { 959 959 - err := s.db.StatusPending(models.WorkflowId{ 960 960 - PipelineId: pipelineId, 961 961 - Name: ewf.Name, 962 962 - }, s.n) 963 963 - if err != nil { 964 964 - return fmt.Errorf("db.StatusPending: %w", err) 965 965 - } 966 966 - } 967 967 - } 968 968 - return nil 969 969 - } 970 970 - 971 971 - // newRepoPath creates a path to store repository by its did and rkey. 972 972 - // The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey 973 973 - func (s *Spindle) newRepoPath(repo syntax.DID) string { 974 974 - return filepath.Join(s.cfg.Server.RepoDir, repo.String()) 975 975 - } 976 976 - 977 977 - func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { 978 978 - scheme := "https://" 979 979 - if s.cfg.Server.Dev { 980 980 - scheme = "http://" 981 981 - } 982 982 - return fmt.Sprintf("%s%s/%s", scheme, knot, did) 983 983 - } 984 984 - 985 985 - const RequiredVersion = "2.49.0" 986 986 - 987 987 - func ensureGitVersion() error { 988 988 - v, err := git.Version() 989 989 - if err != nil { 990 990 - return fmt.Errorf("fetching git version: %w", err) 991 991 - } 992 992 - if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { 993 993 - return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) 994 994 - } 995 995 - return nil 996 996 - } 997 997 - 998 998 - func (s *Spindle) configureOwner() error { 999 999 - cfgOwner := s.cfg.Server.Owner 1000 1000 - 1001 1001 - existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) 1002 1002 - if err != nil { 1003 1003 - return err 1004 1004 - } 1005 1005 - 1006 1006 - switch len(existing) { 1007 1007 - case 0: 1008 1008 - // no owner configured, continue 1009 1009 - case 1: 1010 1010 - // find existing owner 1011 1011 - existingOwner := existing[0] 1012 1012 - 1013 1013 - // no ownership change, this is okay 1014 1014 - if existingOwner == s.cfg.Server.Owner { 1015 1015 - break 1016 1016 - } 1017 1017 - 1018 1018 - // remove existing owner 1019 1019 - err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) 1020 1020 - if err != nil { 1021 1021 - return nil 1022 1022 - } 1023 1023 - default: 1024 1024 - return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) 1025 1025 - } 1026 1026 - 1027 1027 - return s.e.AddSpindleOwner(rbacDomain, cfgOwner) 1028 1028 - }

-46

spindle/server_test.go

Reviewed

··· 1 1 package spindle 2 2 3 3 import ( 4 4 - "context" 5 5 - "net/http" 6 6 - "net/http/httptest" 7 7 - "path/filepath" 8 4 "testing" 9 5 10 6 kgit "tangled.org/core/knotserver/git" 11 11 - "tangled.org/core/spindle/config" 12 12 - "tangled.org/core/spindle/db" 13 13 - "tangled.org/core/spindle/models" 14 7 ) 15 8 16 9 func TestHasSkipCIPushOption(t *testing.T) { ··· 60 53 }) 61 54 } 62 55 } 63 63 - 64 64 - func TestExecutorRoleBuildsMinimalSpindle(t *testing.T) { 65 65 - ctx := context.Background() 66 66 - dbPath := filepath.Join(t.TempDir(), "spindle.db") 67 67 - d, err := db.Make(ctx, dbPath) 68 68 - if err != nil { 69 69 - t.Fatalf("db.Make() error = %v", err) 70 70 - } 71 71 - 72 72 - cfg := &config.Config{Role: config.RoleExecutor} 73 73 - cfg.Server.DBPath = dbPath 74 74 - cfg.Server.Hostname = "executor.test" 75 75 - cfg.Server.Tap.Embed = true 76 76 - cfg.ArtifactStores.Disk.Dir = t.TempDir() 77 77 - cfg.Mill.ArtifactStore = "disk" 78 78 - 79 79 - s, err := New(ctx, cfg, d, map[string]models.Engine{}) 80 80 - if err != nil { 81 81 - t.Fatalf("New() error = %v", err) 82 82 - } 83 83 - 84 84 - if s.jc != nil || s.tap != nil || s.e != nil || s.ks != nil || s.res != nil || s.vault != nil { 85 85 - t.Fatal("executor role built coordinator-only spindle dependencies") 86 86 - } 87 87 - 88 88 - rr := httptest.NewRecorder() 89 89 - req := httptest.NewRequest(http.MethodGet, "/", nil) 90 90 - s.Router().ServeHTTP(rr, req) 91 91 - if rr.Code != http.StatusOK { 92 92 - t.Fatalf("root status = %d, want %d", rr.Code, http.StatusOK) 93 93 - } 94 94 - 95 95 - rr = httptest.NewRecorder() 96 96 - req = httptest.NewRequest(http.MethodGet, "/xrpc/_health", nil) 97 97 - s.Router().ServeHTTP(rr, req) 98 98 - if rr.Code != http.StatusNotFound { 99 99 - t.Fatalf("executor xrpc status = %d, want %d", rr.Code, http.StatusNotFound) 100 100 - } 101 101 - }

spindle/stream.go

Reviewed

This file has not been changed.

spindle/xrpc/ci_pipeline_subscribe_logs.go

Reviewed

This file has not been changed.

spindle/xrpc/xrpc.go

Reviewed

This file has not been changed.

+8 -7

workflow/compile.go

Reviewed

··· 112 112 } 113 113 114 114 func (compiler *Compiler) compileWorkflow(w Workflow) *tangled.Pipeline_Workflow { 115 115 + cw := &tangled.Pipeline_Workflow{} 116 116 + 115 117 matched, err := w.Match(compiler.Trigger, compiler.ChangedFiles) 116 118 if err != nil { 117 119 compiler.Diagnostics.AddError( ··· 132 134 // validate clone options 133 135 compiler.analyzeCloneOptions(w) 134 136 137 137 + cw.Name = w.Name 138 138 + 135 139 if w.Engine == "" { 136 140 compiler.Diagnostics.AddError(w.Name, MissingEngine) 137 141 return nil 138 142 } 139 143 144 144 + cw.Engine = w.Engine 145 145 + cw.Raw = w.Raw 146 146 + 140 147 o := w.CloneOpts.AsRecord() 148 148 + cw.Clone = &o 141 141 - cw := &tangled.Pipeline_Workflow{ 142 142 - Clone: &o, 143 143 - Engine: w.Engine, 144 144 - Name: w.Name, 145 145 - Raw: w.Raw, 146 146 - RunsOn: w.RunsOn, 147 147 - } 148 149 149 150 return cw 150 151 }

-45

workflow/compile_test.go

Reviewed

··· 1 1 package workflow 2 2 3 3 import ( 4 4 - "encoding/json" 5 4 "strings" 6 5 "testing" 7 6 ··· 42 41 assert.False(t, c.Diagnostics.IsErr()) 43 42 } 44 43 45 45 - func TestCompileWorkflow_RunsOnLabelsPersistWithoutFanout(t *testing.T) { 46 46 - wf := Workflow{ 47 47 - Name: ".tangled/workflows/arm64.yml", 48 48 - Engine: "microvm", 49 49 - When: when, 50 50 - RunsOn: []string{"linux/arm64", "kvm"}, 51 51 - } 52 52 - 53 53 - c := Compiler{Trigger: trigger} 54 54 - cp := c.Compile([]Workflow{wf}) 55 55 - 56 56 - assert.Len(t, cp.Workflows, 1) 57 57 - assert.Equal(t, []string{"linux/arm64", "kvm"}, cp.Workflows[0].RunsOn) 58 58 - 59 59 - raw, err := json.Marshal(cp.Workflows[0]) 60 60 - assert.NoError(t, err) 61 61 - 62 62 - var persisted map[string]any 63 63 - assert.NoError(t, json.Unmarshal(raw, &persisted)) 64 64 - assert.Equal(t, []any{"linux/arm64", "kvm"}, persisted["runsOn"]) 65 65 - } 66 66 - 67 67 - func TestCompileWorkflow_LegacyWorkflowOmitsRunsOn(t *testing.T) { 68 68 - wf := Workflow{ 69 69 - Name: ".tangled/workflows/legacy.yml", 70 70 - Engine: "microvm", 71 71 - When: when, 72 72 - } 73 73 - 74 74 - c := Compiler{Trigger: trigger} 75 75 - cp := c.Compile([]Workflow{wf}) 76 76 - 77 77 - assert.Len(t, cp.Workflows, 1) 78 78 - assert.Empty(t, cp.Workflows[0].RunsOn) 79 79 - 80 80 - raw, err := json.Marshal(cp.Workflows[0]) 81 81 - assert.NoError(t, err) 82 82 - 83 83 - var persisted map[string]any 84 84 - assert.NoError(t, json.Unmarshal(raw, &persisted)) 85 85 - _, ok := persisted["runsOn"] 86 86 - assert.False(t, ok, "legacy workflow JSON should not gain runsOn") 87 87 - } 88 88 - 89 44 func TestCompileWorkflow_TriggerMismatch(t *testing.T) { 90 45 wf := Workflow{ 91 46 Name: ".tangled/workflows/mismatch.yml",

+7 -14

workflow/def.go

Reviewed

··· 27 27 Workflow struct { 28 28 Name string `yaml:"-"` // name of the workflow file 29 29 Engine string `yaml:"engine"` 30 30 - RunsOn []string `yaml:"runs_on"` 31 30 When []Constraint `yaml:"when"` 32 31 CloneOpts CloneOpts `yaml:"clone"` 33 32 Raw string `yaml:"-"` ··· 122 121 } 123 122 124 123 func (c *Constraint) Match(trigger tangled.Pipeline_TriggerMetadata, changedFiles []string) (bool, error) { 124 124 + match := true 125 125 + 125 126 // manual triggers always pass this constraint 126 127 if trigger.Manual != nil { 127 128 return true, nil 128 129 } 129 130 130 131 // apply event constraints 132 132 + match = match && c.MatchEvent(trigger.Kind) 131 131 - if !c.MatchEvent(trigger.Kind) { 132 132 - return false, nil 133 133 - } 134 133 135 134 // apply branch constraints for PRs 136 135 if trigger.PullRequest != nil { ··· 138 137 if err != nil { 139 138 return false, err 140 139 } 140 140 + match = match && matched 141 141 - if !matched { 142 142 - return false, nil 143 143 - } 144 141 } 145 142 146 143 // apply ref constraints for pushes ··· 149 146 if err != nil { 150 147 return false, err 151 148 } 149 149 + match = match && matched 152 152 - if !matched { 153 153 - return false, nil 154 154 - } 155 150 } 156 151 157 152 // apply paths filter: if specified, at least one changed file must match ··· 160 155 if err != nil { 161 156 return false, err 162 157 } 158 158 + match = match && matched 163 163 - if !matched { 164 164 - return false, nil 165 165 - } 166 159 } 167 160 161 161 + return match, nil 168 168 - return true, nil 169 162 } 170 163 171 164 // matchesAnyFile returns true if any file in files matches any of the glob patterns.

-13

workflow/def_test.go

Reviewed

··· 23 23 assert.False(t, wf.CloneOpts.Skip, "Skip should default to false") 24 24 } 25 25 26 26 - func TestUnmarshalWorkflowWithRunsOnLabels(t *testing.T) { 27 27 - yamlData := ` 28 28 - engine: microvm 29 29 - runs_on: [linux/arm64, kvm] 30 30 - when: 31 31 - - event: push` 32 32 - 33 33 - wf, err := FromFile("test.yml", []byte(yamlData)) 34 34 - assert.NoError(t, err) 35 35 - 36 36 - assert.Equal(t, []string{"linux/arm64", "kvm"}, wf.RunsOn) 37 37 - } 38 38 - 39 26 func TestUnmarshalCloneFalse(t *testing.T) { 40 27 yamlData := ` 41 28 when:

+32

spindle/db/artifacts.go

Reviewed

··· 1 1 + package db 2 2 + 3 3 + type FinishedLog struct { 4 4 + LeaseID string 5 5 + Workflow string 6 6 + Ref string 7 7 + Hash string 8 8 + } 9 9 + 10 10 + func (d *DB) GetFinishedLog(workflow string) (*FinishedLog, error) { 11 11 + var fl FinishedLog 12 12 + err := d.QueryRow( 13 13 + `select lease_id, workflow, ref, hash 14 14 + from mill_artifacts 15 15 + where workflow = ? 16 16 + order by id desc limit 1`, 17 17 + workflow, 18 18 + ).Scan(&fl.LeaseID, &fl.Workflow, &fl.Ref, &fl.Hash) 19 19 + if err != nil { 20 20 + return nil, err 21 21 + } 22 22 + return &fl, nil 23 23 + } 24 24 + 25 25 + func (d *DB) SaveArtifactRef(leaseID, workflow, ref, hash string) error { 26 26 + _, err := d.Exec( 27 27 + `insert into mill_artifacts (lease_id, workflow, ref, hash) 28 28 + values (?, ?, ?, ?)`, 29 29 + leaseID, workflow, ref, hash, 30 30 + ) 31 31 + return err 32 32 + }