tangled.org
Monorepo for Tangled
tangled.org
spindle: implement mill
Signed-off-by: dawn <dawn@tangled.org>
author
dawn
date
(Jul 21, 2026, 3:17 AM +0300)
commit
66022d43
66022d4367d3a0dccd7e9c44ef7b6cb9578dc485
parent
f6d37afe
f6d37afeda5a265b1362aa580c1f573ce71e9e43
change-id
kxzqyrzk
kxzqyrzkntnrvotunruwrwyxkontoxvo
+12028
-284
+82
-1
api/tangled/cbor_gen.go
+82
-1
api/tangled/cbor_gen.go
···
9749
9749
}
9750
9750
9751
9751
cw := cbg.NewCborWriter(w)
9752
+
fieldCount := 5
9752
9753
9753
-
if _, err := cw.Write([]byte{164}); err != nil {
9754
+
if t.RunsOn == nil {
9755
+
fieldCount--
9756
+
}
9757
+
9758
+
if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil {
9754
9759
return err
9755
9760
}
9756
9761
···
9838
9843
if _, err := cw.WriteString(string(t.Engine)); err != nil {
9839
9844
return err
9840
9845
}
9846
+
9847
+
// t.RunsOn ([]string) (slice)
9848
+
if t.RunsOn != nil {
9849
+
9850
+
if len("runsOn") > 1000000 {
9851
+
return xerrors.Errorf("Value in field \"runsOn\" was too long")
9852
+
}
9853
+
9854
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("runsOn"))); err != nil {
9855
+
return err
9856
+
}
9857
+
if _, err := cw.WriteString(string("runsOn")); err != nil {
9858
+
return err
9859
+
}
9860
+
9861
+
if len(t.RunsOn) > 8192 {
9862
+
return xerrors.Errorf("Slice value in field t.RunsOn was too long")
9863
+
}
9864
+
9865
+
if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.RunsOn))); err != nil {
9866
+
return err
9867
+
}
9868
+
for _, v := range t.RunsOn {
9869
+
if len(v) > 1000000 {
9870
+
return xerrors.Errorf("Value in field v was too long")
9871
+
}
9872
+
9873
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil {
9874
+
return err
9875
+
}
9876
+
if _, err := cw.WriteString(string(v)); err != nil {
9877
+
return err
9878
+
}
9879
+
9880
+
}
9881
+
}
9841
9882
return nil
9842
9883
}
9843
9884
···
9934
9975
}
9935
9976
9936
9977
t.Engine = string(sval)
9978
+
}
9979
+
// t.RunsOn ([]string) (slice)
9980
+
case "runsOn":
9981
+
9982
+
maj, extra, err = cr.ReadHeader()
9983
+
if err != nil {
9984
+
return err
9985
+
}
9986
+
9987
+
if extra > 8192 {
9988
+
return fmt.Errorf("t.RunsOn: array too large (%d)", extra)
9989
+
}
9990
+
9991
+
if maj != cbg.MajArray {
9992
+
return fmt.Errorf("expected cbor array")
9993
+
}
9994
+
9995
+
if extra > 0 {
9996
+
t.RunsOn = make([]string, extra)
9997
+
}
9998
+
9999
+
for i := 0; i < int(extra); i++ {
10000
+
{
10001
+
var maj byte
10002
+
var extra uint64
10003
+
var err error
10004
+
_ = maj
10005
+
_ = extra
10006
+
_ = err
10007
+
10008
+
{
10009
+
sval, err := cbg.ReadStringWithMax(cr, 1000000)
10010
+
if err != nil {
10011
+
return err
10012
+
}
10013
+
10014
+
t.RunsOn[i] = string(sval)
10015
+
}
10016
+
10017
+
}
9937
10018
}
9938
10019
9939
10020
default:
+1
api/tangled/tangledpipeline.go
+1
api/tangled/tangledpipeline.go
+4
buf.gen.yaml
+4
buf.gen.yaml
+1
buf.yaml
+1
buf.yaml
+160
cmd/spindle/main.go
+160
cmd/spindle/main.go
···
2
2
3
3
import (
4
4
"context"
5
+
"fmt"
5
6
"log/slog"
6
7
"os"
8
+
"strings"
9
+
"text/tabwriter"
10
+
"time"
7
11
8
12
"github.com/urfave/cli/v3"
9
13
tlog "tangled.org/core/log"
10
14
"tangled.org/core/spindle"
15
+
"tangled.org/core/spindle/db"
16
+
"tangled.org/core/spindle/mill"
11
17
)
12
18
13
19
func main() {
···
16
22
Usage: "spindle continuous integration runner",
17
23
Commands: []*cli.Command{
18
24
Command(),
25
+
millCommand(),
19
26
},
20
27
DefaultCommand: "run",
21
28
}
···
41
48
},
42
49
}
43
50
}
51
+
52
+
// for mill host administration and executor management
53
+
func millCommand() *cli.Command {
54
+
dbFlag := &cli.StringFlag{
55
+
Name: "db",
56
+
Usage: "path to the spindle sqlite db",
57
+
Value: "spindle.db",
58
+
Sources: cli.EnvVars("SPINDLE_SERVER_DB_PATH"),
59
+
}
60
+
openDB := func(ctx context.Context, cmd *cli.Command) (*db.DB, error) {
61
+
return db.Make(ctx, cmd.String("db"))
62
+
}
63
+
return &cli.Command{
64
+
Name: "mill",
65
+
Usage: "mill host administration",
66
+
Commands: []*cli.Command{
67
+
{
68
+
Name: "executor",
69
+
Usage: "manage executors allowed to join this mill",
70
+
Commands: []*cli.Command{
71
+
{
72
+
Name: "add",
73
+
Usage: "register an executor and print its token",
74
+
ArgsUsage: "<name>",
75
+
Flags: []cli.Flag{
76
+
dbFlag,
77
+
&cli.DurationFlag{
78
+
Name: "ttl",
79
+
Usage: "token lifetime (e.g. 720h); omit for no expiry",
80
+
},
81
+
&cli.StringSliceFlag{
82
+
Name: "label",
83
+
Usage: "authorized labels for this executor",
84
+
},
85
+
},
86
+
Action: func(ctx context.Context, cmd *cli.Command) error {
87
+
name := cmd.Args().First()
88
+
if name == "" {
89
+
return fmt.Errorf("usage: spindle mill executor add <name>")
90
+
}
91
+
d, err := openDB(ctx, cmd)
92
+
if err != nil {
93
+
return err
94
+
}
95
+
token, err := mill.GenerateToken()
96
+
if err != nil {
97
+
return err
98
+
}
99
+
var expiresAt *time.Time
100
+
if ttl := cmd.Duration("ttl"); ttl > 0 {
101
+
exp := time.Now().UTC().Add(ttl)
102
+
expiresAt = &exp
103
+
}
104
+
labels := cmd.StringSlice("label")
105
+
if err := d.AddExecutorToken(name, mill.HashToken(token), expiresAt, labels); err != nil {
106
+
return fmt.Errorf("registering executor %q: %w", name, err)
107
+
}
108
+
fmt.Println(token)
109
+
return nil
110
+
},
111
+
},
112
+
{
113
+
Name: "list",
114
+
Usage: "list registered executors",
115
+
Flags: []cli.Flag{dbFlag},
116
+
Action: func(ctx context.Context, cmd *cli.Command) error {
117
+
d, err := openDB(ctx, cmd)
118
+
if err != nil {
119
+
return err
120
+
}
121
+
tokens, err := d.ListExecutorTokens()
122
+
if err != nil {
123
+
return err
124
+
}
125
+
w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
126
+
fmt.Fprintln(w, "NAME\tCREATED\tEXPIRES\tLABELS\tQUARANTINE")
127
+
for _, t := range tokens {
128
+
expires := "never"
129
+
if t.ExpiresAt != nil {
130
+
expires = t.ExpiresAt.Format(time.RFC3339)
131
+
if time.Now().After(*t.ExpiresAt) {
132
+
expires += " (expired)"
133
+
}
134
+
}
135
+
labels := strings.Join(t.Labels, ",")
136
+
if labels == "" {
137
+
labels = "-"
138
+
}
139
+
quarantine := "-"
140
+
if t.QuarantineReason != nil {
141
+
quarantine = *t.QuarantineReason
142
+
if t.QuarantinedAt != nil {
143
+
quarantine = *t.QuarantinedAt + ": " + quarantine
144
+
}
145
+
}
146
+
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", t.Name, t.CreatedAt, expires, labels, quarantine)
147
+
}
148
+
return w.Flush()
149
+
},
150
+
},
151
+
{
152
+
Name: "unquarantine",
153
+
Usage: "allow a quarantined executor to reconnect",
154
+
ArgsUsage: "<name>",
155
+
Flags: []cli.Flag{dbFlag},
156
+
Action: func(ctx context.Context, cmd *cli.Command) error {
157
+
name := cmd.Args().First()
158
+
if name == "" {
159
+
return fmt.Errorf("usage: spindle mill executor unquarantine <name>")
160
+
}
161
+
d, err := openDB(ctx, cmd)
162
+
if err != nil {
163
+
return err
164
+
}
165
+
ok, err := d.ClearExecutorQuarantine(name)
166
+
if err != nil {
167
+
return err
168
+
}
169
+
if !ok {
170
+
return fmt.Errorf("no such executor identity %q", name)
171
+
}
172
+
return nil
173
+
},
174
+
},
175
+
{
176
+
Name: "revoke",
177
+
Usage: "revoke an executor's token",
178
+
ArgsUsage: "<name>",
179
+
Flags: []cli.Flag{dbFlag},
180
+
Action: func(ctx context.Context, cmd *cli.Command) error {
181
+
name := cmd.Args().First()
182
+
if name == "" {
183
+
return fmt.Errorf("usage: spindle mill executor revoke <name>")
184
+
}
185
+
d, err := openDB(ctx, cmd)
186
+
if err != nil {
187
+
return err
188
+
}
189
+
ok, err := d.RevokeExecutorToken(name)
190
+
if err != nil {
191
+
return err
192
+
}
193
+
if !ok {
194
+
return fmt.Errorf("no such executor identity %q", name)
195
+
}
196
+
return nil
197
+
},
198
+
},
199
+
},
200
+
},
201
+
},
202
+
}
203
+
}
+148
docker-compose.mill.yml
+148
docker-compose.mill.yml
···
1
+
# turns the primary spindle into a mill host and runs a three-executor fleet
2
+
# against it. the executors differ on two axes so placement is testable:
3
+
#
4
+
# executor labels images can run
5
+
# --------- ------------- ----------- ------------------------
6
+
# executor-a linux, fast full image/alpine, image/nixos
7
+
# executor-b linux, slow full image/alpine, image/nixos
8
+
# executor-c linux, gpu alpine only image/alpine
9
+
10
+
x-mill-executor: &mill-executor
11
+
profiles: ["linux"]
12
+
build:
13
+
context: .
14
+
dockerfile: localinfra/spindle.Dockerfile
15
+
restart: unless-stopped
16
+
environment: &mill-executor-env
17
+
SPINDLE_ROLE: executor
18
+
SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555
19
+
SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db
20
+
SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev
21
+
SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe
22
+
SPINDLE_SERVER_DEV: "true"
23
+
SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev
24
+
SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db
25
+
SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev
26
+
SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images
27
+
SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays
28
+
SPINDLE_S3_LOG_BUCKET: ""
29
+
SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false"
30
+
SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501
31
+
SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU=
32
+
SPINDLE_NIX_CACHE_UPLOAD_URL: http://ncps:8501/upload
33
+
# dials the mill container directly using ws
34
+
SPINDLE_MILL_URL: ws://spindle:6555/mill
35
+
devices:
36
+
- /dev/vsock:/dev/vsock
37
+
- /dev/kvm:/dev/kvm
38
+
- /dev/vhost-vsock:/dev/vhost-vsock
39
+
- /dev/net/tun:/dev/net/tun
40
+
cap_add:
41
+
- NET_ADMIN
42
+
- SYS_ADMIN
43
+
security_opt:
44
+
- label=disable
45
+
- seccomp=unconfined
46
+
healthcheck:
47
+
test: ["CMD", "wget", "-qO-", "http://localhost:6555/"]
48
+
interval: 2s
49
+
timeout: 2s
50
+
retries: 30
51
+
start_period: 5s
52
+
depends_on: &mill-executor-deps
53
+
plc:
54
+
condition: service_started
55
+
jetstream:
56
+
condition: service_started
57
+
init-accounts:
58
+
condition: service_completed_successfully
59
+
ncps:
60
+
condition: service_started
61
+
spindle:
62
+
condition: service_healthy
63
+
mill-tokens:
64
+
condition: service_completed_successfully
65
+
networks: [tngl]
66
+
67
+
services:
68
+
# configures the primary spindle container as a mill host
69
+
spindle:
70
+
environment:
71
+
SPINDLE_ROLE: mill
72
+
73
+
mill-tokens:
74
+
profiles: ["linux"]
75
+
build:
76
+
context: .
77
+
dockerfile: localinfra/spindle.Dockerfile
78
+
restart: "no"
79
+
entrypoint: ["/bin/sh", "-c"]
80
+
environment:
81
+
SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db
82
+
command:
83
+
- |
84
+
set -eu
85
+
seed() {
86
+
spindle mill executor revoke "$$1" >/dev/null 2>&1 || true
87
+
args=""
88
+
for l in $$(echo "$$2" | tr ',' ' '); do args="$$args --label $$l"; done
89
+
spindle mill executor add "$$1" $$args > "/shared/$$1.mill-token"
90
+
}
91
+
seed executor-a linux,fast
92
+
seed executor-b linux,slow
93
+
seed executor-c linux,gpu
94
+
echo "seeded executor tokens"
95
+
volumes:
96
+
- spindle-data:/var/lib/spindle
97
+
- init-state:/shared
98
+
depends_on:
99
+
spindle:
100
+
condition: service_healthy
101
+
networks: [tngl]
102
+
103
+
spindle-executor-a:
104
+
<<: *mill-executor
105
+
environment:
106
+
<<: *mill-executor-env
107
+
SPINDLE_SERVER_HOSTNAME: executor-a.tngl.boltless.dev
108
+
SPINDLE_MILL_LABELS: linux,fast
109
+
SPINDLE_MILL_TOKEN_FILE: /shared/executor-a.mill-token
110
+
SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11241"
111
+
volumes:
112
+
- spindle-executor-a-data:/var/lib/spindle
113
+
- ./out/localinfra-spindle-images:/var/lib/spindle/images:ro
114
+
- init-state:/shared:ro
115
+
- ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro
116
+
117
+
spindle-executor-b:
118
+
<<: *mill-executor
119
+
environment:
120
+
<<: *mill-executor-env
121
+
SPINDLE_SERVER_HOSTNAME: executor-b.tngl.boltless.dev
122
+
SPINDLE_MILL_LABELS: linux,slow
123
+
SPINDLE_MILL_TOKEN_FILE: /shared/executor-b.mill-token
124
+
SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11242"
125
+
volumes:
126
+
- spindle-executor-b-data:/var/lib/spindle
127
+
- ./out/localinfra-spindle-images:/var/lib/spindle/images:ro
128
+
- init-state:/shared:ro
129
+
- ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro
130
+
131
+
spindle-executor-c:
132
+
<<: *mill-executor
133
+
environment:
134
+
<<: *mill-executor-env
135
+
SPINDLE_SERVER_HOSTNAME: executor-c.tngl.boltless.dev
136
+
SPINDLE_MILL_LABELS: linux,gpu
137
+
SPINDLE_MILL_TOKEN_FILE: /shared/executor-c.mill-token
138
+
SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11243"
139
+
volumes:
140
+
- spindle-executor-c-data:/var/lib/spindle
141
+
- ./out/localinfra-spindle-images-alpine:/var/lib/spindle/images:ro
142
+
- init-state:/shared:ro
143
+
- ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro
144
+
145
+
volumes:
146
+
spindle-executor-a-data:
147
+
spindle-executor-b-data:
148
+
spindle-executor-c-data:
+49
eventstream/eventstream_test.go
+49
eventstream/eventstream_test.go
···
293
293
}
294
294
}
295
295
296
+
func TestHighWaterSeedsClockFromStoredEvents(t *testing.T) {
297
+
db, err := sql.Open("sqlite3", t.TempDir()+"/events.db")
298
+
if err != nil {
299
+
t.Fatalf("open: %v", err)
300
+
}
301
+
t.Cleanup(func() { db.Close() })
302
+
if _, err := db.Exec(`create table events (
303
+
rkey text not null,
304
+
nsid text not null,
305
+
event text not null,
306
+
created integer not null,
307
+
primary key (rkey, nsid)
308
+
)`); err != nil {
309
+
t.Fatalf("schema: %v", err)
310
+
}
311
+
312
+
stored := time.Now().Add(time.Hour).UnixNano()
313
+
if _, err := db.Exec(
314
+
`insert into events (rkey, nsid, event, created) values (?, ?, ?, ?)`,
315
+
"stored", "sh.tangled.test", "{}", stored,
316
+
); err != nil {
317
+
t.Fatalf("seed event: %v", err)
318
+
}
319
+
320
+
cut, err := HighWater(db)
321
+
if err != nil {
322
+
t.Fatalf("HighWater() error = %v", err)
323
+
}
324
+
if cut < stored {
325
+
t.Fatalf("HighWater() = %d, want at least stored cursor %d", cut, stored)
326
+
}
327
+
328
+
n := notifier.New()
329
+
if err := Insert(db, Event{
330
+
Rkey: "new",
331
+
Nsid: "sh.tangled.test",
332
+
EventJson: json.RawMessage("{}"),
333
+
}, &n); err != nil {
334
+
t.Fatalf("Insert() error = %v", err)
335
+
}
336
+
events, err := List(db, cut, 10)
337
+
if err != nil {
338
+
t.Fatalf("List() error = %v", err)
339
+
}
340
+
if len(events) != 1 || events[0].Rkey != "new" || events[0].Created <= cut {
341
+
t.Fatalf("events after cut = %+v, want only new event above %d", events, cut)
342
+
}
343
+
}
344
+
296
345
func isCloseErr(err error) bool {
297
346
if err == nil {
298
347
return false
+29
eventstream/store.go
+29
eventstream/store.go
···
19
19
lastNanos int64
20
20
)
21
21
22
+
func HighWater(s Store) (int64, error) {
23
+
clockMu.Lock()
24
+
defer clockMu.Unlock()
25
+
26
+
rows, err := s.Query(`select coalesce(max(created), 0) from events`)
27
+
if err != nil {
28
+
return 0, err
29
+
}
30
+
defer rows.Close()
31
+
32
+
var created int64
33
+
if !rows.Next() {
34
+
if err := rows.Err(); err != nil {
35
+
return 0, err
36
+
}
37
+
return 0, sql.ErrNoRows
38
+
}
39
+
if err := rows.Scan(&created); err != nil {
40
+
return 0, err
41
+
}
42
+
if err := rows.Err(); err != nil {
43
+
return 0, err
44
+
}
45
+
if created > lastNanos {
46
+
lastNanos = created
47
+
}
48
+
return lastNanos, nil
49
+
}
50
+
22
51
func Insert(s Store, ev Event, n *notifier.Notifier) error {
23
52
clockMu.Lock()
24
53
defer clockMu.Unlock()
+6
lexicons/pipeline/pipeline.json
+6
lexicons/pipeline/pipeline.json
+20
localinfra/readme.md
+20
localinfra/readme.md
···
51
51
This writes the image directory under `out/localinfra-spindle-images`.
52
52
5. `docker compose up`
53
53
6. AppView will be running on `127.0.0.1:3000` with two test users: `alice.pds.tngl.boltless.dev` and `bob.pds.tngl.boltless.dev`. Both with password `password`.
54
+
55
+
## Mill mode
56
+
57
+
The default stack runs one standalone spindle. `docker-compose.mill.yml` is an overlay that splits that into the distributed arch: the primary spindle becomes the mill host (`role=mill`, it only places jobs) and a three-executor fleet (`role=executor`) runs the engines.
58
+
59
+
```bash
60
+
docker compose -f docker-compose.yml -f docker-compose.mill.yml --profile linux up
61
+
```
62
+
63
+
The executors differ on the two axes placement cares about, so you can watch candidates get filtered and ranked:
64
+
65
+
| executor | labels | images | runs |
66
+
|------------|---------------|-------------|-----------------------------|
67
+
| executor-a | `linux, fast` | full | `image/alpine, image/nixos` |
68
+
| executor-b | `linux, slow` | full | `image/alpine, image/nixos` |
69
+
| executor-c | `linux, gpu` | alpine only | `image/alpine` |
70
+
71
+
So `image: nixos` has two candidates, `image: alpine` has three, and `runs_on: [gpu]` pins to executor-c. The alpine-only image set is staged by `prepare-spindle-images.sh` (step 4) alongside the full one, no extra step.
72
+
73
+
Each executor needs its own identity (one live session per token), so the `mill-tokens` service registers a token per executor in the mill db and drops it into the shared volume for the executor to read.
+10
localinfra/scripts/prepare-spindle-images.sh
+10
localinfra/scripts/prepare-spindle-images.sh
···
29
29
extract_image spindle-nixos-image-tarball nixos-x86_64 nixos
30
30
extract_image spindle-alpine-image-tarball alpine-x86_64 alpine
31
31
32
+
# a reduced image set (alpine only) for the mill overlay: an executor mounting
33
+
# this advertises image/alpine but not image/nixos, so capability-based
34
+
# placement is testable across a mixed fleet
35
+
alpine_only="$repo/out/localinfra-spindle-images-alpine"
36
+
[ -d "$alpine_only" ] && chmod -R +w "$alpine_only" || true
37
+
rm -rf "$alpine_only"
38
+
mkdir -p "$alpine_only"
39
+
cp -r "$image_root/alpine-x86_64" "$alpine_only/alpine-x86_64"
40
+
ln -s alpine-x86_64 "$alpine_only/alpine"
41
+
32
42
echo "prepared spindle microVM images in $image_root"
+5
localinfra/spindle.Dockerfile
+5
localinfra/spindle.Dockerfile
···
49
49
export SPINDLE_SERVER_OWNER="$(cat /shared/owner-did)"
50
50
: "${SPINDLE_SERVER_OWNER:?set via env or /shared/owner-did}"
51
51
52
+
# executors read their mill token from a file seeded by the mill-tokens
53
+
# service (command substitution strips the trailing newline)
54
+
[ -z "${SPINDLE_MILL_SHARED_SECRET:-}" ] && [ -n "${SPINDLE_MILL_TOKEN_FILE:-}" ] && [ -r "${SPINDLE_MILL_TOKEN_FILE}" ] && \
55
+
export SPINDLE_MILL_SHARED_SECRET="$(cat "${SPINDLE_MILL_TOKEN_FILE}")"
56
+
52
57
mkdir -p /var/lib/spindle /var/lib/spindle/overlays /var/log/spindle
53
58
54
59
if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then
+3
-3
netutil/ssrf.go
+3
-3
netutil/ssrf.go
···
10
10
"github.com/gorilla/websocket"
11
11
)
12
12
13
-
// SSRFDialer returns a net.Dialer that refuses non-public IPs.
13
+
// refuses non-public ips to prevent ssrf
14
14
func SSRFDialer(dev bool) *net.Dialer {
15
15
if dev {
16
16
return &net.Dialer{}
···
18
18
return ssrf.PublicOnlyDialer()
19
19
}
20
20
21
-
// SSRFTransport returns an http.Transport that refuses non-public IPs.
21
+
// refuses non-public ips to prevent ssrf
22
22
func SSRFTransport(dev bool) *http.Transport {
23
23
if dev {
24
24
return &http.Transport{}
···
26
26
return ssrf.PublicOnlyTransport()
27
27
}
28
28
29
-
// SSRFWebsocketDialer returns a websocket.Dialer that refuses non-public IPs.
29
+
// refuses non-public ips to prevent ssrf
30
30
func SSRFWebsocketDialer(dev bool) *websocket.Dialer {
31
31
dialer := *websocket.DefaultDialer
32
32
dialer.NetDialContext = SSRFDialer(dev).DialContext
+44
spindle/config/config.go
+44
spindle/config/config.go
···
2
2
3
3
import (
4
4
"context"
5
+
"fmt"
5
6
"time"
6
7
7
8
"github.com/bluesky-social/indigo/atproto/syntax"
···
93
94
UploadURL string `env:"UPLOAD_URL"`
94
95
}
95
96
97
+
// governs how spindle places and runs jobs
98
+
type Role string
99
+
100
+
const (
101
+
RoleStandalone Role = "standalone"
102
+
RoleMill Role = "mill"
103
+
RoleExecutor Role = "executor"
104
+
)
105
+
106
+
// fields are selectively active depending on the role
107
+
type Mill struct {
108
+
URL string `env:"URL"` // mill websocket endpoint dialled by the executor
109
+
SharedSecret string `env:"SHARED_SECRET"` // the executor's token for dialing the mill
110
+
MaxPending int `env:"MAX_PENDING, default=100"` // mill pending job queue limit
111
+
ReconnectGrace time.Duration `env:"RECONNECT_GRACE, default=45s"` // reconnect window before leases are failed
112
+
Seats int `env:"SEATS, default=4"` // executor seats advertised to the mill
113
+
Labels []string `env:"LABELS"` // executor capability labels
114
+
}
115
+
96
116
type Config struct {
117
+
Role Role `env:"SPINDLE_ROLE, default=standalone"`
97
118
Server Server `env:",prefix=SPINDLE_SERVER_"`
98
119
NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"`
99
120
MicroVMPipelines MicroVMPipelines `env:",prefix=SPINDLE_MICROVM_PIPELINES_"`
100
121
NixCache NixCache `env:",prefix=SPINDLE_NIX_CACHE_"`
101
122
S3 S3 `env:",prefix=SPINDLE_S3_"`
123
+
Mill Mill `env:",prefix=SPINDLE_MILL_"`
124
+
}
125
+
126
+
func (c *Config) validate() error {
127
+
switch c.Role {
128
+
case RoleStandalone, RoleMill:
129
+
if c.Mill.URL != "" {
130
+
return fmt.Errorf("SPINDLE_MILL_URL is set but SPINDLE_ROLE=%s; only an executor dials a mill", c.Role)
131
+
}
132
+
case RoleExecutor:
133
+
if c.Mill.URL == "" {
134
+
return fmt.Errorf("SPINDLE_ROLE=executor requires SPINDLE_MILL_URL (the mill to dial)")
135
+
}
136
+
if c.Mill.SharedSecret == "" {
137
+
return fmt.Errorf("SPINDLE_ROLE=executor requires SPINDLE_MILL_SHARED_SECRET (its executor token)")
138
+
}
139
+
default:
140
+
return fmt.Errorf("unknown SPINDLE_ROLE %q (want standalone, mill, or executor)", c.Role)
141
+
}
142
+
return nil
102
143
}
103
144
104
145
func Load(ctx context.Context) (*Config, error) {
105
146
var cfg Config
106
147
err := envconfig.Process(ctx, &cfg)
107
148
if err != nil {
149
+
return nil, err
150
+
}
151
+
if err := cfg.validate(); err != nil {
108
152
return nil, err
109
153
}
110
154
+56
-12
spindle/db/db.go
+56
-12
spindle/db/db.go
···
79
79
);
80
80
81
81
create table if not exists spindle_members (
82
-
-- identifiers for the record
83
82
id integer primary key autoincrement,
84
83
did text not null,
85
84
rkey text not null,
86
-
87
-
-- data
88
85
instance text not null,
89
86
subject text not null,
90
87
created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
91
-
92
-
-- constraints
93
88
unique (did, rkey)
94
89
);
95
90
96
-
-- status event for a single workflow
97
91
create table if not exists events (
98
92
rkey text not null,
99
93
nsid text not null,
100
-
event text not null, -- json
101
-
created integer not null -- unix nanos
94
+
event text not null,
95
+
created integer not null
102
96
);
103
97
104
98
create table if not exists nixos_toplevel_cache (
···
123
117
foreign key (pipeline_id) references pipelines(id) on delete cascade
124
118
);
125
119
120
+
create table if not exists mill_executors (
121
+
name text primary key,
122
+
token_hash text not null unique,
123
+
created_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
124
+
expires_at text,
125
+
labels text,
126
+
quarantine_reason text,
127
+
quarantined_at text
128
+
);
129
+
130
+
create table if not exists mill_leases (
131
+
lease_id text primary key,
132
+
node_id text not null,
133
+
epoch text not null,
134
+
engine text not null,
135
+
knot text not null,
136
+
rkey text not null,
137
+
workflow text not null,
138
+
state text not null,
139
+
created_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))
140
+
);
141
+
142
+
create table if not exists mill_executor_cursors (
143
+
node_id text not null,
144
+
epoch text not null,
145
+
acked_seqno integer not null,
146
+
primary key (node_id, epoch)
147
+
);
148
+
149
+
create table if not exists mill_outbox_state (
150
+
epoch text not null,
151
+
next_seqno integer not null,
152
+
primary key (epoch)
153
+
);
154
+
155
+
create table if not exists mill_outbox_rows (
156
+
epoch text not null,
157
+
seqno integer not null,
158
+
payload blob not null,
159
+
byte_size integer not null,
160
+
control integer not null,
161
+
primary key (epoch, seqno),
162
+
foreign key (epoch) references mill_outbox_state(epoch) on delete cascade
163
+
);
164
+
165
+
create table if not exists mill_canonical_logs (
166
+
node_id text not null,
167
+
epoch text not null,
168
+
seqno integer not null,
169
+
workflow text not null,
170
+
payload blob not null,
171
+
primary key (node_id, epoch, seqno)
172
+
);
173
+
126
174
create table if not exists migrations (
127
175
id integer primary key autoincrement,
128
176
name text unique
129
177
);
130
178
`)
131
-
if err != nil {
132
-
return nil, err
133
-
}
134
-
135
179
if err := runMigrations(ctx, conn, logger); err != nil {
136
180
return nil, err
137
181
}
+59
-15
spindle/db/events.go
+59
-15
spindle/db/events.go
···
19
19
return eventstream.List(d, cursor, limit)
20
20
}
21
21
22
+
func (d *DB) EventHighWater() (int64, error) {
23
+
return eventstream.HighWater(d)
24
+
}
25
+
22
26
func (d *DB) CreatePipelineEvent(rkey string, pipeline tangled.Pipeline, n *notifier.Notifier) error {
23
27
eventJson, err := json.Marshal(pipeline)
24
28
if err != nil {
···
32
36
return d.insertEvent(event, n)
33
37
}
34
38
35
-
func (d *DB) createStatusEvent(
36
-
workflowId models.WorkflowId,
37
-
statusKind models.StatusKind,
38
-
workflowError *string,
39
-
exitCode *int64,
40
-
n *notifier.Notifier,
41
-
) error {
42
-
now := time.Now()
43
-
pipelineAtUri := workflowId.PipelineId.AtUri()
39
+
// leaves created at zero so insertevent stamps the local clock
40
+
func statusEvent(pipelineAtUri, workflow, status string, workflowError *string, exitCode *int64) (eventstream.Event, error) {
44
41
s := tangled.PipelineStatus{
45
-
CreatedAt: now.Format(time.RFC3339),
42
+
CreatedAt: time.Now().Format(time.RFC3339),
46
43
Error: workflowError,
47
44
ExitCode: exitCode,
48
-
Pipeline: string(pipelineAtUri),
49
-
Workflow: workflowId.Name,
50
-
Status: string(statusKind),
45
+
Pipeline: pipelineAtUri,
46
+
Workflow: workflow,
47
+
Status: status,
51
48
}
52
49
53
50
eventJson, err := json.Marshal(s)
54
51
if err != nil {
55
-
return err
52
+
return eventstream.Event{}, err
56
53
}
57
54
58
-
event := eventstream.Event{
55
+
return eventstream.Event{
59
56
Rkey: tid.TID(),
60
57
Nsid: tangled.PipelineStatusNSID,
61
58
EventJson: eventJson,
59
+
}, nil
60
+
}
61
+
62
+
func (d *DB) createStatusEvent(
63
+
workflowId models.WorkflowId,
64
+
statusKind models.StatusKind,
65
+
workflowError *string,
66
+
exitCode *int64,
67
+
n *notifier.Notifier,
68
+
) error {
69
+
event, err := statusEvent(string(workflowId.PipelineId.AtUri()), workflowId.Name, string(statusKind), workflowError, exitCode)
70
+
if err != nil {
71
+
return err
62
72
}
73
+
return d.insertEvent(event, n)
74
+
}
63
75
76
+
// stamps the mill's own clock so it orders against the cursor like a local write
77
+
func (d *DB) InsertEventStatus(
78
+
pipelineAtUri string,
79
+
workflow string,
80
+
status string,
81
+
workflowError *string,
82
+
exitCode *int64,
83
+
n *notifier.Notifier,
84
+
) error {
85
+
event, err := statusEvent(pipelineAtUri, workflow, status, workflowError, exitCode)
86
+
if err != nil {
87
+
return err
88
+
}
64
89
return d.insertEvent(event, n)
90
+
}
91
+
92
+
// deleting the lease in the same transaction prevents the terminal event
93
+
// from replaying
94
+
func (d *DB) CompleteMillLease(
95
+
leaseID string,
96
+
pipelineAtUri string,
97
+
workflow string,
98
+
status string,
99
+
workflowError *string,
100
+
exitCode *int64,
101
+
n *notifier.Notifier,
102
+
) error {
103
+
return d.ApplyEventBatch(n, func(tx *EventBatchTx) error {
104
+
if err := tx.InsertStatusEvent(pipelineAtUri, workflow, status, workflowError, exitCode); err != nil {
105
+
return err
106
+
}
107
+
return tx.DeleteLease(leaseID)
108
+
})
65
109
}
66
110
67
111
func (d *DB) GetStatus(workflowId models.WorkflowId) (*tangled.PipelineStatus, error) {
+434
spindle/db/mill_state.go
+434
spindle/db/mill_state.go
···
1
+
package db
2
+
3
+
import (
4
+
"context"
5
+
"database/sql"
6
+
"fmt"
7
+
"time"
8
+
9
+
"tangled.org/core/eventstream"
10
+
"tangled.org/core/notifier"
11
+
)
12
+
13
+
// enough to rebuild the fencing token and workflow identity after a restart
14
+
type MillLease struct {
15
+
LeaseID string
16
+
NodeID string
17
+
Epoch string
18
+
Engine string
19
+
Knot string
20
+
Rkey string
21
+
Workflow string
22
+
State string
23
+
}
24
+
25
+
type ExecutorCursor struct {
26
+
NodeID string
27
+
Epoch string
28
+
AckedSeqno uint64
29
+
}
30
+
31
+
type OutboxRow struct {
32
+
Epoch string
33
+
Seqno uint64
34
+
Payload []byte
35
+
ByteSize int64
36
+
Control bool
37
+
}
38
+
type OutboxDeletion struct {
39
+
Rows int64
40
+
LogBytes int64
41
+
ControlBytes int64
42
+
}
43
+
44
+
type CanonicalLog struct {
45
+
NodeID string
46
+
Epoch string
47
+
Seqno uint64
48
+
Workflow string
49
+
Payload []byte
50
+
}
51
+
52
+
func (d *DB) SaveMillLease(l MillLease) error {
53
+
_, err := d.Exec(
54
+
`insert into mill_leases (
55
+
lease_id, node_id, epoch, engine, knot, rkey, workflow, state
56
+
) values (?, ?, ?, ?, ?, ?, ?, ?)
57
+
on conflict(lease_id) do update set state = excluded.state`,
58
+
l.LeaseID, l.NodeID, l.Epoch, l.Engine, l.Knot, l.Rkey, l.Workflow, l.State,
59
+
)
60
+
return err
61
+
}
62
+
63
+
func (d *DB) DeleteMillLease(leaseID string) error {
64
+
_, err := d.Exec(`delete from mill_leases where lease_id = ?`, leaseID)
65
+
return err
66
+
}
67
+
68
+
func (d *DB) ListMillLeases() ([]MillLease, error) {
69
+
rows, err := d.Query(`
70
+
select lease_id, node_id, epoch, engine, knot, rkey, workflow, state
71
+
from mill_leases
72
+
`)
73
+
if err != nil {
74
+
return nil, err
75
+
}
76
+
defer rows.Close()
77
+
78
+
var leases []MillLease
79
+
for rows.Next() {
80
+
var l MillLease
81
+
if err := rows.Scan(
82
+
&l.LeaseID, &l.NodeID, &l.Epoch, &l.Engine, &l.Knot, &l.Rkey, &l.Workflow, &l.State,
83
+
); err != nil {
84
+
return nil, err
85
+
}
86
+
leases = append(leases, l)
87
+
}
88
+
return leases, rows.Err()
89
+
}
90
+
91
+
func (d *DB) SetExecutorCursor(nodeID, epoch string, seqno uint64) error {
92
+
_, err := d.Exec(
93
+
`insert into mill_executor_cursors (node_id, epoch, acked_seqno) values (?, ?, ?)
94
+
on conflict(node_id, epoch) do update set acked_seqno = excluded.acked_seqno`,
95
+
nodeID, epoch, seqno,
96
+
)
97
+
return err
98
+
}
99
+
100
+
func (d *DB) DeleteExecutorCursor(nodeID, epoch string) error {
101
+
_, err := d.Exec(`delete from mill_executor_cursors where node_id = ? and epoch = ?`, nodeID, epoch)
102
+
return err
103
+
}
104
+
105
+
func (d *DB) ListExecutorCursors() ([]ExecutorCursor, error) {
106
+
rows, err := d.Query(`select node_id, epoch, acked_seqno from mill_executor_cursors`)
107
+
if err != nil {
108
+
return nil, err
109
+
}
110
+
defer rows.Close()
111
+
112
+
var cursors []ExecutorCursor
113
+
for rows.Next() {
114
+
var c ExecutorCursor
115
+
if err := rows.Scan(&c.NodeID, &c.Epoch, &c.AckedSeqno); err != nil {
116
+
return nil, err
117
+
}
118
+
cursors = append(cursors, c)
119
+
}
120
+
return cursors, rows.Err()
121
+
}
122
+
123
+
func (d *DB) SetOutboxEpoch(epoch string) error {
124
+
tx, err := d.Begin()
125
+
if err != nil {
126
+
return err
127
+
}
128
+
defer tx.Rollback()
129
+
130
+
if _, err := tx.Exec(`delete from mill_outbox_rows`); err != nil {
131
+
return err
132
+
}
133
+
if _, err := tx.Exec(`delete from mill_outbox_state`); err != nil {
134
+
return err
135
+
}
136
+
if _, err := tx.Exec(`insert into mill_outbox_state (epoch, next_seqno) values (?, 1)`, epoch); err != nil {
137
+
return err
138
+
}
139
+
return tx.Commit()
140
+
}
141
+
142
+
func (d *DB) AppendOutboxRow(payload []byte, control bool) (uint64, error) {
143
+
tx, err := d.Begin()
144
+
if err != nil {
145
+
return 0, err
146
+
}
147
+
defer tx.Rollback()
148
+
149
+
var epoch string
150
+
var nextSeqno uint64
151
+
err = tx.QueryRow(`select epoch, next_seqno from mill_outbox_state limit 1`).Scan(&epoch, &nextSeqno)
152
+
if err == sql.ErrNoRows {
153
+
return 0, fmt.Errorf("no outbox epoch set")
154
+
} else if err != nil {
155
+
return 0, err
156
+
}
157
+
158
+
byteSize := int64(len(payload))
159
+
controlVal := 0
160
+
if control {
161
+
controlVal = 1
162
+
}
163
+
164
+
if _, err := tx.Exec(
165
+
`insert into mill_outbox_rows (epoch, seqno, payload, byte_size, control) values (?, ?, ?, ?, ?)`,
166
+
epoch, nextSeqno, payload, byteSize, controlVal,
167
+
); err != nil {
168
+
return 0, err
169
+
}
170
+
171
+
if _, err := tx.Exec(
172
+
`update mill_outbox_state set next_seqno = ? where epoch = ?`,
173
+
nextSeqno+1, epoch,
174
+
); err != nil {
175
+
return 0, err
176
+
}
177
+
178
+
if err := tx.Commit(); err != nil {
179
+
return 0, err
180
+
}
181
+
return nextSeqno, nil
182
+
}
183
+
184
+
func (d *DB) DeleteOutboxPrefix(ackedSeqno uint64) (OutboxDeletion, error) {
185
+
tx, err := d.Begin()
186
+
if err != nil {
187
+
return OutboxDeletion{}, err
188
+
}
189
+
defer tx.Rollback()
190
+
191
+
var epoch string
192
+
err = tx.QueryRow(`select epoch from mill_outbox_state limit 1`).Scan(&epoch)
193
+
if err == sql.ErrNoRows {
194
+
return OutboxDeletion{}, nil
195
+
}
196
+
if err != nil {
197
+
return OutboxDeletion{}, err
198
+
}
199
+
200
+
var deleted OutboxDeletion
201
+
if err := tx.QueryRow(`
202
+
select count(*),
203
+
coalesce(sum(case when control = 0 then byte_size else 0 end), 0),
204
+
coalesce(sum(case when control != 0 then byte_size else 0 end), 0)
205
+
from mill_outbox_rows
206
+
where epoch = ? and seqno <= ?
207
+
`, epoch, ackedSeqno).Scan(&deleted.Rows, &deleted.LogBytes, &deleted.ControlBytes); err != nil {
208
+
return OutboxDeletion{}, err
209
+
}
210
+
if _, err := tx.Exec(
211
+
`delete from mill_outbox_rows where epoch = ? and seqno <= ?`,
212
+
epoch, ackedSeqno,
213
+
); err != nil {
214
+
return OutboxDeletion{}, err
215
+
}
216
+
if err := tx.Commit(); err != nil {
217
+
return OutboxDeletion{}, err
218
+
}
219
+
return deleted, nil
220
+
}
221
+
222
+
func (d *DB) ListOutboxRows() ([]OutboxRow, error) {
223
+
rows, err := d.Query(`select epoch, seqno, payload, byte_size, control from mill_outbox_rows order by seqno`)
224
+
if err != nil {
225
+
return nil, err
226
+
}
227
+
defer rows.Close()
228
+
229
+
var out []OutboxRow
230
+
for rows.Next() {
231
+
var r OutboxRow
232
+
var controlVal int
233
+
if err := rows.Scan(&r.Epoch, &r.Seqno, &r.Payload, &r.ByteSize, &controlVal); err != nil {
234
+
return nil, err
235
+
}
236
+
r.Control = (controlVal != 0)
237
+
out = append(out, r)
238
+
}
239
+
return out, rows.Err()
240
+
}
241
+
func (d *DB) ListOutboxRowsAfter(seqno uint64, limit int) ([]OutboxRow, error) {
242
+
rows, err := d.Query(`
243
+
select epoch, seqno, payload, byte_size, control
244
+
from mill_outbox_rows
245
+
where epoch = (select epoch from mill_outbox_state limit 1)
246
+
and seqno > ?
247
+
order by seqno
248
+
limit ?
249
+
`, seqno, limit)
250
+
if err != nil {
251
+
return nil, err
252
+
}
253
+
defer rows.Close()
254
+
255
+
var out []OutboxRow
256
+
for rows.Next() {
257
+
var row OutboxRow
258
+
var control int
259
+
if err := rows.Scan(&row.Epoch, &row.Seqno, &row.Payload, &row.ByteSize, &control); err != nil {
260
+
return nil, err
261
+
}
262
+
row.Control = control != 0
263
+
out = append(out, row)
264
+
}
265
+
return out, rows.Err()
266
+
}
267
+
268
+
func (d *DB) GetOutboxState() (string, uint64, error) {
269
+
var epoch string
270
+
var nextSeqno uint64
271
+
err := d.QueryRow(`select epoch, next_seqno from mill_outbox_state limit 1`).Scan(&epoch, &nextSeqno)
272
+
if err == sql.ErrNoRows {
273
+
return "", 0, nil
274
+
}
275
+
return epoch, nextSeqno, err
276
+
}
277
+
278
+
func (d *DB) listCanonicalLogs(where string, args ...any) ([]CanonicalLog, error) {
279
+
rows, err := d.Query(`
280
+
select node_id, epoch, seqno, workflow, payload
281
+
from mill_canonical_logs
282
+
`+where+`
283
+
order by workflow, node_id, epoch, seqno
284
+
`, args...)
285
+
if err != nil {
286
+
return nil, err
287
+
}
288
+
defer rows.Close()
289
+
290
+
var logs []CanonicalLog
291
+
for rows.Next() {
292
+
var log CanonicalLog
293
+
if err := rows.Scan(&log.NodeID, &log.Epoch, &log.Seqno, &log.Workflow, &log.Payload); err != nil {
294
+
return nil, err
295
+
}
296
+
logs = append(logs, log)
297
+
}
298
+
return logs, rows.Err()
299
+
}
300
+
301
+
func (d *DB) ListCanonicalLogs() ([]CanonicalLog, error) {
302
+
return d.listCanonicalLogs("")
303
+
}
304
+
305
+
func (d *DB) ListCanonicalLogsFor(workflow string) ([]CanonicalLog, error) {
306
+
return d.listCanonicalLogs(`where workflow = ?`, workflow)
307
+
}
308
+
309
+
// rows after a (node_id, epoch, seqno) position, so a follower can resume
310
+
// without replaying
311
+
func (d *DB) ListCanonicalLogsAfter(workflow, nodeID, epoch string, seqno uint64, limit int) ([]CanonicalLog, error) {
312
+
rows, err := d.Query(`
313
+
select node_id, epoch, seqno, workflow, payload
314
+
from mill_canonical_logs
315
+
where workflow = ? and (node_id, epoch, seqno) > (?, ?, ?)
316
+
order by node_id, epoch, seqno
317
+
limit ?
318
+
`, workflow, nodeID, epoch, seqno, limit)
319
+
if err != nil {
320
+
return nil, err
321
+
}
322
+
defer rows.Close()
323
+
324
+
var logs []CanonicalLog
325
+
for rows.Next() {
326
+
var log CanonicalLog
327
+
if err := rows.Scan(&log.NodeID, &log.Epoch, &log.Seqno, &log.Workflow, &log.Payload); err != nil {
328
+
return nil, err
329
+
}
330
+
logs = append(logs, log)
331
+
}
332
+
return logs, rows.Err()
333
+
}
334
+
335
+
// streams a workflow's canonical payloads as they arrive, until ctx
336
+
// ends. query errors retry on the next tick, the table is the live log
337
+
// while a job runs
338
+
func (d *DB) FollowCanonicalLog(ctx context.Context, workflow string) <-chan []byte {
339
+
out := make(chan []byte, 64)
340
+
go func() {
341
+
defer close(out)
342
+
var nodeID, epoch string
343
+
var seqno uint64
344
+
for {
345
+
logs, err := d.ListCanonicalLogsAfter(workflow, nodeID, epoch, seqno, 256)
346
+
if err == nil {
347
+
for _, log := range logs {
348
+
select {
349
+
case out <- log.Payload:
350
+
nodeID, epoch, seqno = log.NodeID, log.Epoch, log.Seqno
351
+
case <-ctx.Done():
352
+
return
353
+
}
354
+
}
355
+
if len(logs) == 256 {
356
+
continue
357
+
}
358
+
}
359
+
select {
360
+
case <-time.After(500 * time.Millisecond):
361
+
case <-ctx.Done():
362
+
return
363
+
}
364
+
}
365
+
}()
366
+
return out
367
+
}
368
+
369
+
func (d *DB) DeleteCanonicalLogs(workflow string) error {
370
+
_, err := d.Exec(`delete from mill_canonical_logs where workflow = ?`, workflow)
371
+
return err
372
+
}
373
+
374
+
type EventBatchTx struct {
375
+
tx *sql.Tx
376
+
db *DB
377
+
}
378
+
379
+
func (tx *EventBatchTx) InsertStatusEvent(pipelineAtUri, workflow, status string, workflowError *string, exitCode *int64) error {
380
+
event, err := statusEvent(pipelineAtUri, workflow, status, workflowError, exitCode)
381
+
if err != nil {
382
+
return err
383
+
}
384
+
return eventstream.Insert(tx.tx, event, nil)
385
+
}
386
+
387
+
func (tx *EventBatchTx) AddCanonicalLog(nodeID, epoch string, seqno uint64, workflow string, payload []byte) error {
388
+
_, err := tx.tx.Exec(
389
+
`insert into mill_canonical_logs (node_id, epoch, seqno, workflow, payload) values (?, ?, ?, ?, ?)
390
+
on conflict(node_id, epoch, seqno) do nothing`,
391
+
nodeID, epoch, seqno, workflow, payload,
392
+
)
393
+
return err
394
+
}
395
+
396
+
func (tx *EventBatchTx) DeleteLease(leaseID string) error {
397
+
_, err := tx.tx.Exec(`delete from mill_leases where lease_id = ?`, leaseID)
398
+
return err
399
+
}
400
+
401
+
func (tx *EventBatchTx) AdvanceCursor(nodeID, epoch string, seqno uint64) error {
402
+
_, err := tx.tx.Exec(
403
+
`insert into mill_executor_cursors (node_id, epoch, acked_seqno) values (?, ?, ?)
404
+
on conflict(node_id, epoch) do update set acked_seqno = excluded.acked_seqno`,
405
+
nodeID, epoch, seqno,
406
+
)
407
+
return err
408
+
}
409
+
410
+
func (d *DB) ApplyEventBatch(n *notifier.Notifier, fn func(tx *EventBatchTx) error) error {
411
+
tx, err := d.Begin()
412
+
if err != nil {
413
+
return err
414
+
}
415
+
defer tx.Rollback()
416
+
417
+
batchTx := &EventBatchTx{
418
+
tx: tx,
419
+
db: d,
420
+
}
421
+
422
+
if err := fn(batchTx); err != nil {
423
+
return err
424
+
}
425
+
426
+
if err := tx.Commit(); err != nil {
427
+
return err
428
+
}
429
+
430
+
if n != nil {
431
+
n.NotifyAll()
432
+
}
433
+
return nil
434
+
}
+448
spindle/db/mill_state_test.go
+448
spindle/db/mill_state_test.go
···
1
+
package db
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"path/filepath"
7
+
"testing"
8
+
9
+
"tangled.org/core/notifier"
10
+
)
11
+
12
+
func TestMillLeaseRoundTrip(t *testing.T) {
13
+
d := newTestDB(t)
14
+
15
+
lease := MillLease{
16
+
LeaseID: "lease-1",
17
+
NodeID: "node-1",
18
+
Epoch: "inc-1",
19
+
Engine: "dummy",
20
+
Knot: "knot.example",
21
+
Rkey: "rkey1",
22
+
Workflow: "build",
23
+
State: "reserved",
24
+
}
25
+
if err := d.SaveMillLease(lease); err != nil {
26
+
t.Fatalf("SaveMillLease: %v", err)
27
+
}
28
+
29
+
lease.State = "running"
30
+
if err := d.SaveMillLease(lease); err != nil {
31
+
t.Fatalf("SaveMillLease(transition): %v", err)
32
+
}
33
+
34
+
leases, err := d.ListMillLeases()
35
+
if err != nil {
36
+
t.Fatalf("ListMillLeases: %v", err)
37
+
}
38
+
if len(leases) != 1 {
39
+
t.Fatalf("ListMillLeases returned %d leases, want 1 (state transition must replace, not duplicate)", len(leases))
40
+
}
41
+
if leases[0].LeaseID != lease.LeaseID || leases[0].State != "running" {
42
+
t.Fatalf("ListMillLeases[0] = %+v, want %+v", leases[0], lease)
43
+
}
44
+
45
+
if err := d.DeleteMillLease("lease-1"); err != nil {
46
+
t.Fatalf("DeleteMillLease: %v", err)
47
+
}
48
+
if leases, _ = d.ListMillLeases(); len(leases) != 0 {
49
+
t.Fatalf("lease survived deletion: %+v", leases)
50
+
}
51
+
}
52
+
53
+
func TestExecutorCursors(t *testing.T) {
54
+
d := newTestDB(t)
55
+
56
+
if err := d.SetExecutorCursor("node-1", "inc-1", 5); err != nil {
57
+
t.Fatalf("SetExecutorCursor: %v", err)
58
+
}
59
+
if err := d.SetExecutorCursor("node-1", "inc-1", 9); err != nil {
60
+
t.Fatalf("SetExecutorCursor(advance): %v", err)
61
+
}
62
+
if err := d.SetExecutorCursor("node-2", "inc-1", 1); err != nil {
63
+
t.Fatalf("SetExecutorCursor(node-2): %v", err)
64
+
}
65
+
66
+
cursors, err := d.ListExecutorCursors()
67
+
if err != nil {
68
+
t.Fatalf("ListExecutorCursors: %v", err)
69
+
}
70
+
if len(cursors) != 2 {
71
+
t.Fatalf("ListExecutorCursors = %v, want 2 cursors", cursors)
72
+
}
73
+
74
+
cursorMap := make(map[string]uint64)
75
+
for _, c := range cursors {
76
+
cursorMap[c.NodeID+"/"+c.Epoch] = c.AckedSeqno
77
+
}
78
+
79
+
if cursorMap["node-1/inc-1"] != 9 || cursorMap["node-2/inc-1"] != 1 {
80
+
t.Fatalf("unexpected cursors: %v", cursorMap)
81
+
}
82
+
83
+
if err := d.DeleteExecutorCursor("node-1", "inc-1"); err != nil {
84
+
t.Fatalf("DeleteExecutorCursor: %v", err)
85
+
}
86
+
cursors, _ = d.ListExecutorCursors()
87
+
if len(cursors) != 1 || cursors[0].NodeID != "node-2" {
88
+
t.Fatalf("ListExecutorCursors after delete = %v, want only node-2", cursors)
89
+
}
90
+
}
91
+
92
+
func TestCompleteMillLeaseIsAtomic(t *testing.T) {
93
+
d := newTestDB(t)
94
+
lease := MillLease{
95
+
LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy",
96
+
Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: "running",
97
+
}
98
+
if err := d.SaveMillLease(lease); err != nil {
99
+
t.Fatalf("SaveMillLease: %v", err)
100
+
}
101
+
if _, err := d.Exec(`
102
+
create trigger reject_mill_lease_delete
103
+
before delete on mill_leases
104
+
begin
105
+
select raise(abort, 'forced delete failure');
106
+
end
107
+
`); err != nil {
108
+
t.Fatalf("create failure trigger: %v", err)
109
+
}
110
+
111
+
n := notifier.New()
112
+
notifications := n.Subscribe()
113
+
defer n.Unsubscribe(notifications)
114
+
err := d.CompleteMillLease(
115
+
"lease-1",
116
+
"at://knot.example/sh.tangled.pipeline/rkey1",
117
+
"build",
118
+
"failed",
119
+
nil,
120
+
nil,
121
+
&n,
122
+
)
123
+
if err == nil {
124
+
t.Fatal("CompleteMillLease succeeded despite forced lease deletion failure")
125
+
}
126
+
var eventCount int
127
+
if err := d.QueryRow(`select count(*) from events`).Scan(&eventCount); err != nil {
128
+
t.Fatalf("count events after rollback: %v", err)
129
+
}
130
+
if eventCount != 0 {
131
+
t.Fatalf("terminal event count after rollback = %d, want 0", eventCount)
132
+
}
133
+
if leases, listErr := d.ListMillLeases(); listErr != nil || len(leases) != 1 {
134
+
t.Fatalf("leases after rollback = %+v, err = %v; want original lease", leases, listErr)
135
+
}
136
+
select {
137
+
case <-notifications:
138
+
t.Fatal("rollback notified event subscribers")
139
+
default:
140
+
}
141
+
142
+
if _, err := d.Exec(`drop trigger reject_mill_lease_delete`); err != nil {
143
+
t.Fatalf("drop failure trigger: %v", err)
144
+
}
145
+
if err := d.CompleteMillLease(
146
+
"lease-1",
147
+
"at://knot.example/sh.tangled.pipeline/rkey1",
148
+
"build",
149
+
"failed",
150
+
nil,
151
+
nil,
152
+
&n,
153
+
); err != nil {
154
+
t.Fatalf("CompleteMillLease retry: %v", err)
155
+
}
156
+
if err := d.QueryRow(`select count(*) from events`).Scan(&eventCount); err != nil {
157
+
t.Fatalf("count committed events: %v", err)
158
+
}
159
+
if eventCount != 1 {
160
+
t.Fatalf("terminal event count after commit = %d, want 1", eventCount)
161
+
}
162
+
if leases, listErr := d.ListMillLeases(); listErr != nil || len(leases) != 0 {
163
+
t.Fatalf("leases after commit = %+v, err = %v; want none", leases, listErr)
164
+
}
165
+
select {
166
+
case <-notifications:
167
+
default:
168
+
t.Fatal("committed terminal event did not notify subscribers")
169
+
}
170
+
}
171
+
172
+
func TestRestartPersistence(t *testing.T) {
173
+
dbPath := filepath.Join(t.TempDir(), "persist.db")
174
+
ctx := context.Background()
175
+
d, err := Make(ctx, dbPath)
176
+
if err != nil {
177
+
t.Fatalf("Make: %v", err)
178
+
}
179
+
180
+
lease := MillLease{
181
+
LeaseID: "lease-p", NodeID: "node-p", Epoch: "inc-p", Engine: "dummy",
182
+
Knot: "k", Rkey: "r", Workflow: "w", State: "running",
183
+
}
184
+
if err := d.SaveMillLease(lease); err != nil {
185
+
t.Fatalf("SaveMillLease: %v", err)
186
+
}
187
+
188
+
if err := d.SetExecutorCursor("node-p", "inc-p", 42); err != nil {
189
+
t.Fatalf("SetExecutorCursor: %v", err)
190
+
}
191
+
192
+
if err := d.SetOutboxEpoch("inc-p"); err != nil {
193
+
t.Fatalf("SetOutboxEpoch: %v", err)
194
+
}
195
+
if _, err := d.AppendOutboxRow([]byte("hello world"), true); err != nil {
196
+
t.Fatalf("AppendOutboxRow: %v", err)
197
+
}
198
+
199
+
if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error {
200
+
return tx.AddCanonicalLog("node-p", "inc-p", 1, "w", []byte("log content"))
201
+
}); err != nil {
202
+
t.Fatalf("AddCanonicalLog: %v", err)
203
+
}
204
+
205
+
if err := d.Close(); err != nil {
206
+
t.Fatalf("Close: %v", err)
207
+
}
208
+
209
+
d2, err := Make(ctx, dbPath)
210
+
if err != nil {
211
+
t.Fatalf("Make reopen: %v", err)
212
+
}
213
+
defer d2.Close()
214
+
215
+
leases, err := d2.ListMillLeases()
216
+
if err != nil {
217
+
t.Fatalf("ListMillLeases: %v", err)
218
+
}
219
+
if len(leases) != 1 || leases[0].LeaseID != "lease-p" || leases[0].Epoch != "inc-p" {
220
+
t.Fatalf("unexpected leases: %+v", leases)
221
+
}
222
+
223
+
cursors, err := d2.ListExecutorCursors()
224
+
if err != nil {
225
+
t.Fatalf("ListExecutorCursors: %v", err)
226
+
}
227
+
if len(cursors) != 1 || cursors[0].NodeID != "node-p" || cursors[0].Epoch != "inc-p" || cursors[0].AckedSeqno != 42 {
228
+
t.Fatalf("unexpected cursors: %+v", cursors)
229
+
}
230
+
231
+
inc, nextSeqno, err := d2.GetOutboxState()
232
+
if err != nil {
233
+
t.Fatalf("GetOutboxState: %v", err)
234
+
}
235
+
if inc != "inc-p" || nextSeqno != 2 {
236
+
t.Fatalf("unexpected outbox state: inc=%q, next=%d", inc, nextSeqno)
237
+
}
238
+
rows, err := d2.ListOutboxRows()
239
+
if err != nil {
240
+
t.Fatalf("ListOutboxRows: %v", err)
241
+
}
242
+
if len(rows) != 1 || string(rows[0].Payload) != "hello world" || !rows[0].Control {
243
+
t.Fatalf("unexpected outbox rows: %+v", rows)
244
+
}
245
+
246
+
logs, err := d2.ListCanonicalLogs()
247
+
if err != nil {
248
+
t.Fatalf("ListCanonicalLogs: %v", err)
249
+
}
250
+
if len(logs) != 1 || string(logs[0].Payload) != "log content" {
251
+
t.Fatalf("unexpected canonical logs: %+v", logs)
252
+
}
253
+
}
254
+
255
+
func TestOutboxPrefixAck(t *testing.T) {
256
+
d := newTestDB(t)
257
+
258
+
if err := d.SetOutboxEpoch("inc-1"); err != nil {
259
+
t.Fatalf("SetOutboxEpoch: %v", err)
260
+
}
261
+
262
+
o1, err := d.AppendOutboxRow([]byte("msg1"), false)
263
+
if err != nil || o1 != 1 {
264
+
t.Fatalf("AppendOutboxRow 1: %v, seqno=%d", err, o1)
265
+
}
266
+
o2, err := d.AppendOutboxRow([]byte("msg2"), false)
267
+
if err != nil || o2 != 2 {
268
+
t.Fatalf("AppendOutboxRow 2: %v, seqno=%d", err, o2)
269
+
}
270
+
o3, err := d.AppendOutboxRow([]byte("msg3"), true)
271
+
if err != nil || o3 != 3 {
272
+
t.Fatalf("AppendOutboxRow 3: %v, seqno=%d", err, o3)
273
+
}
274
+
275
+
n, err := d.DeleteOutboxPrefix(2)
276
+
if err != nil {
277
+
t.Fatalf("DeleteOutboxPrefix: %v", err)
278
+
}
279
+
if n.Rows != 2 || n.LogBytes != 8 || n.ControlBytes != 0 {
280
+
t.Fatalf("deleted prefix = %+v, want 2 rows and 8 log bytes", n)
281
+
}
282
+
283
+
rows, err := d.ListOutboxRows()
284
+
if err != nil {
285
+
t.Fatalf("ListOutboxRows: %v", err)
286
+
}
287
+
if len(rows) != 1 || rows[0].Seqno != 3 || string(rows[0].Payload) != "msg3" {
288
+
t.Fatalf("expected only msg3 (seqno 3) to remain, got: %+v", rows)
289
+
}
290
+
}
291
+
292
+
func TestCompositeCursors(t *testing.T) {
293
+
d := newTestDB(t)
294
+
295
+
if err := d.SetExecutorCursor("node-1", "inc-1", 10); err != nil {
296
+
t.Fatalf("SetExecutorCursor: %v", err)
297
+
}
298
+
if err := d.SetExecutorCursor("node-1", "inc-2", 20); err != nil {
299
+
t.Fatalf("SetExecutorCursor: %v", err)
300
+
}
301
+
if err := d.SetExecutorCursor("node-2", "inc-1", 5); err != nil {
302
+
t.Fatalf("SetExecutorCursor: %v", err)
303
+
}
304
+
305
+
cursors, err := d.ListExecutorCursors()
306
+
if err != nil {
307
+
t.Fatalf("ListExecutorCursors: %v", err)
308
+
}
309
+
if len(cursors) != 3 {
310
+
t.Fatalf("expected 3 cursors, got %d", len(cursors))
311
+
}
312
+
313
+
cursorMap := make(map[string]uint64)
314
+
for _, c := range cursors {
315
+
key := c.NodeID + "/" + c.Epoch
316
+
cursorMap[key] = c.AckedSeqno
317
+
}
318
+
319
+
if cursorMap["node-1/inc-1"] != 10 || cursorMap["node-1/inc-2"] != 20 || cursorMap["node-2/inc-1"] != 5 {
320
+
t.Fatalf("unexpected cursor values: %v", cursorMap)
321
+
}
322
+
323
+
if err := d.DeleteExecutorCursor("node-1", "inc-1"); err != nil {
324
+
t.Fatalf("DeleteExecutorCursor: %v", err)
325
+
}
326
+
327
+
cursors, err = d.ListExecutorCursors()
328
+
if err != nil {
329
+
t.Fatalf("ListExecutorCursors: %v", err)
330
+
}
331
+
if len(cursors) != 2 {
332
+
t.Fatalf("expected 2 cursors, got %d", len(cursors))
333
+
}
334
+
}
335
+
336
+
func TestBatchRollback(t *testing.T) {
337
+
d := newTestDB(t)
338
+
339
+
lease := MillLease{
340
+
LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy",
341
+
Knot: "k", Rkey: "r", Workflow: "w", State: "running",
342
+
}
343
+
if err := d.SaveMillLease(lease); err != nil {
344
+
t.Fatalf("SaveMillLease: %v", err)
345
+
}
346
+
347
+
n := notifier.New()
348
+
err := d.ApplyEventBatch(&n, func(tx *EventBatchTx) error {
349
+
if err := tx.DeleteLease("lease-1"); err != nil {
350
+
return err
351
+
}
352
+
if err := tx.AdvanceCursor("node-1", "inc-1", 100); err != nil {
353
+
return err
354
+
}
355
+
return fmt.Errorf("forced batch failure")
356
+
})
357
+
358
+
if err == nil {
359
+
t.Fatal("expected ApplyEventBatch to return error")
360
+
}
361
+
362
+
leases, err := d.ListMillLeases()
363
+
if err != nil {
364
+
t.Fatalf("ListMillLeases: %v", err)
365
+
}
366
+
if len(leases) != 1 {
367
+
t.Fatalf("lease was deleted despite rollback: %+v", leases)
368
+
}
369
+
370
+
cursors, err := d.ListExecutorCursors()
371
+
if err != nil {
372
+
t.Fatalf("ListExecutorCursors: %v", err)
373
+
}
374
+
if len(cursors) != 0 {
375
+
t.Fatalf("cursor was advanced despite rollback: %+v", cursors)
376
+
}
377
+
}
378
+
379
+
func TestTerminalCursorAtomicity(t *testing.T) {
380
+
d := newTestDB(t)
381
+
382
+
lease := MillLease{
383
+
LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy",
384
+
Knot: "k", Rkey: "r", Workflow: "w", State: "running",
385
+
}
386
+
if err := d.SaveMillLease(lease); err != nil {
387
+
t.Fatalf("SaveMillLease: %v", err)
388
+
}
389
+
390
+
n := notifier.New()
391
+
notifications := n.Subscribe()
392
+
defer n.Unsubscribe(notifications)
393
+
394
+
err := d.ApplyEventBatch(&n, func(tx *EventBatchTx) error {
395
+
if err := tx.DeleteLease("lease-1"); err != nil {
396
+
return err
397
+
}
398
+
return tx.AdvanceCursor("node-1", "inc-1", 100)
399
+
})
400
+
401
+
if err != nil {
402
+
t.Fatalf("ApplyEventBatch: %v", err)
403
+
}
404
+
405
+
leases, err := d.ListMillLeases()
406
+
if err != nil {
407
+
t.Fatalf("ListMillLeases: %v", err)
408
+
}
409
+
if len(leases) != 0 {
410
+
t.Fatalf("lease not deleted: %+v", leases)
411
+
}
412
+
413
+
cursors, err := d.ListExecutorCursors()
414
+
if err != nil {
415
+
t.Fatalf("ListExecutorCursors: %v", err)
416
+
}
417
+
if len(cursors) != 1 || cursors[0].AckedSeqno != 100 {
418
+
t.Fatalf("cursor not advanced correctly: %+v", cursors)
419
+
}
420
+
421
+
select {
422
+
case <-notifications:
423
+
default:
424
+
t.Fatal("notifier was not fired after batch commit")
425
+
}
426
+
}
427
+
428
+
func TestCanonicalLogDedup(t *testing.T) {
429
+
d := newTestDB(t)
430
+
add := func(payload string) error {
431
+
return d.ApplyEventBatch(nil, func(tx *EventBatchTx) error {
432
+
return tx.AddCanonicalLog("node-1", "inc-1", 10, "workflow-1", []byte(payload))
433
+
})
434
+
}
435
+
if err := add("first content"); err != nil {
436
+
t.Fatalf("first AddCanonicalLog: %v", err)
437
+
}
438
+
if err := add("second content"); err != nil {
439
+
t.Fatalf("second AddCanonicalLog: %v", err)
440
+
}
441
+
logs, err := d.ListCanonicalLogs()
442
+
if err != nil {
443
+
t.Fatalf("ListCanonicalLogs: %v", err)
444
+
}
445
+
if len(logs) != 1 || string(logs[0].Payload) != "first content" {
446
+
t.Fatalf("canonical logs = %+v, want original payload", logs)
447
+
}
448
+
}
+167
spindle/db/mill_tokens.go
+167
spindle/db/mill_tokens.go
···
1
+
package db
2
+
3
+
import (
4
+
"database/sql"
5
+
"encoding/json"
6
+
"slices"
7
+
"strings"
8
+
"time"
9
+
)
10
+
11
+
// the raw token is never stored, only its hash
12
+
type ExecutorToken struct {
13
+
Name string
14
+
CreatedAt string
15
+
ExpiresAt *time.Time
16
+
Labels []string
17
+
QuarantineReason *string
18
+
QuarantinedAt *string
19
+
}
20
+
21
+
func (d *DB) AddExecutorToken(name, tokenHash string, expiresAt *time.Time, labels []string) error {
22
+
normalized := normalizeLabels(labels)
23
+
labelsBytes, err := json.Marshal(normalized)
24
+
if err != nil {
25
+
return err
26
+
}
27
+
_, err = d.Exec(
28
+
`insert into mill_executors (name, token_hash, expires_at, labels) values (?, ?, ?, ?)`,
29
+
name, tokenHash, expiryArg(expiresAt), string(labelsBytes),
30
+
)
31
+
return err
32
+
}
33
+
34
+
func (d *DB) ResolveExecutorToken(tokenHash string) (string, []string, bool, error) {
35
+
var name string
36
+
var expires sql.NullString
37
+
var labelsRaw, quarantineReason sql.NullString
38
+
err := d.QueryRow(
39
+
`select name, expires_at, labels, quarantine_reason from mill_executors where token_hash = ?`, tokenHash,
40
+
).Scan(&name, &expires, &labelsRaw, &quarantineReason)
41
+
if err == sql.ErrNoRows {
42
+
return "", nil, false, nil
43
+
}
44
+
if err != nil {
45
+
return "", nil, false, err
46
+
}
47
+
if quarantineReason.Valid {
48
+
return "", nil, false, nil
49
+
}
50
+
exp, hasExpiry, err := parseExpiry(expires)
51
+
if err != nil {
52
+
return "", nil, false, err
53
+
}
54
+
if hasExpiry && time.Now().After(exp) {
55
+
return "", nil, false, nil
56
+
}
57
+
var labels []string
58
+
if labelsRaw.Valid && labelsRaw.String != "" {
59
+
if err := json.Unmarshal([]byte(labelsRaw.String), &labels); err != nil {
60
+
return "", nil, false, err
61
+
}
62
+
}
63
+
return name, labels, true, nil
64
+
}
65
+
66
+
func (d *DB) QuarantineExecutor(name, reason string) error {
67
+
_, err := d.Exec(
68
+
`update mill_executors
69
+
set quarantine_reason = ?, quarantined_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now')
70
+
where name = ?`,
71
+
reason, name,
72
+
)
73
+
return err
74
+
}
75
+
76
+
func (d *DB) ClearExecutorQuarantine(name string) (bool, error) {
77
+
res, err := d.Exec(
78
+
`update mill_executors set quarantine_reason = null, quarantined_at = null where name = ?`,
79
+
name,
80
+
)
81
+
if err != nil {
82
+
return false, err
83
+
}
84
+
n, err := res.RowsAffected()
85
+
return n > 0, err
86
+
}
87
+
88
+
func (d *DB) RevokeExecutorToken(name string) (bool, error) {
89
+
res, err := d.Exec(`delete from mill_executors where name = ?`, name)
90
+
if err != nil {
91
+
return false, err
92
+
}
93
+
n, err := res.RowsAffected()
94
+
return n > 0, err
95
+
}
96
+
97
+
func (d *DB) ListExecutorTokens() ([]ExecutorToken, error) {
98
+
rows, err := d.Query(`select name, created_at, expires_at, labels, quarantine_reason, quarantined_at from mill_executors order by name`)
99
+
if err != nil {
100
+
return nil, err
101
+
}
102
+
defer rows.Close()
103
+
104
+
var out []ExecutorToken
105
+
for rows.Next() {
106
+
var t ExecutorToken
107
+
var expires, labelsRaw, quarantineReason, quarantinedAt sql.NullString
108
+
if err := rows.Scan(&t.Name, &t.CreatedAt, &expires, &labelsRaw, &quarantineReason, &quarantinedAt); err != nil {
109
+
return nil, err
110
+
}
111
+
exp, hasExpiry, err := parseExpiry(expires)
112
+
if err != nil {
113
+
return nil, err
114
+
}
115
+
if hasExpiry {
116
+
t.ExpiresAt = &exp
117
+
}
118
+
if labelsRaw.Valid && labelsRaw.String != "" {
119
+
if err := json.Unmarshal([]byte(labelsRaw.String), &t.Labels); err != nil {
120
+
return nil, err
121
+
}
122
+
}
123
+
if quarantineReason.Valid {
124
+
t.QuarantineReason = &quarantineReason.String
125
+
}
126
+
if quarantinedAt.Valid {
127
+
t.QuarantinedAt = &quarantinedAt.String
128
+
}
129
+
out = append(out, t)
130
+
}
131
+
return out, rows.Err()
132
+
}
133
+
134
+
func normalizeLabels(labels []string) []string {
135
+
var out []string
136
+
seen := make(map[string]bool)
137
+
for _, l := range labels {
138
+
trimmed := strings.TrimSpace(l)
139
+
if trimmed == "" {
140
+
continue
141
+
}
142
+
if !seen[trimmed] {
143
+
seen[trimmed] = true
144
+
out = append(out, trimmed)
145
+
}
146
+
}
147
+
slices.Sort(out)
148
+
return out
149
+
}
150
+
151
+
func expiryArg(t *time.Time) any {
152
+
if t == nil {
153
+
return nil
154
+
}
155
+
return t.UTC().Format(time.RFC3339)
156
+
}
157
+
158
+
func parseExpiry(s sql.NullString) (time.Time, bool, error) {
159
+
if !s.Valid {
160
+
return time.Time{}, false, nil
161
+
}
162
+
t, err := time.Parse(time.RFC3339, s.String)
163
+
if err != nil {
164
+
return time.Time{}, false, err
165
+
}
166
+
return t, true, nil
167
+
}
+272
spindle/db/mill_tokens_test.go
+272
spindle/db/mill_tokens_test.go
···
1
+
package db
2
+
3
+
import (
4
+
"slices"
5
+
"testing"
6
+
"time"
7
+
)
8
+
9
+
func TestAddExecutorTokenRejectsDuplicateName(t *testing.T) {
10
+
d := newTestDB(t)
11
+
12
+
if err := d.AddExecutorToken("exec-1", "hash-a", nil, nil); err != nil {
13
+
t.Fatalf("AddExecutorToken: %v", err)
14
+
}
15
+
if err := d.AddExecutorToken("exec-1", "hash-b", nil, nil); err == nil {
16
+
t.Fatal("AddExecutorToken re-registered an existing name; a duplicate must be rejected")
17
+
}
18
+
19
+
name, _, ok, err := d.ResolveExecutorToken("hash-a")
20
+
if err != nil {
21
+
t.Fatalf("ResolveExecutorToken(hash-a): %v", err)
22
+
}
23
+
if !ok || name != "exec-1" {
24
+
t.Fatalf("ResolveExecutorToken(hash-a) = (%q, %v), want (exec-1, true)", name, ok)
25
+
}
26
+
if _, _, ok, _ := d.ResolveExecutorToken("hash-b"); ok {
27
+
t.Fatal("rejected duplicate's token resolved; the failed insert leaked a credential")
28
+
}
29
+
}
30
+
31
+
func TestResolveExecutorTokenMissAndHit(t *testing.T) {
32
+
d := newTestDB(t)
33
+
34
+
name, _, ok, err := d.ResolveExecutorToken("no-such-hash")
35
+
if err != nil {
36
+
t.Fatalf("ResolveExecutorToken(miss): %v", err)
37
+
}
38
+
if ok || name != "" {
39
+
t.Fatalf("ResolveExecutorToken(miss) = (%q, %v), want (\"\", false)", name, ok)
40
+
}
41
+
42
+
if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil {
43
+
t.Fatalf("AddExecutorToken: %v", err)
44
+
}
45
+
46
+
name, _, ok, err = d.ResolveExecutorToken("hash-1")
47
+
if err != nil {
48
+
t.Fatalf("ResolveExecutorToken(hit): %v", err)
49
+
}
50
+
if !ok || name != "exec-1" {
51
+
t.Fatalf("ResolveExecutorToken(hash-1) = (%q, %v), want (exec-1, true)", name, ok)
52
+
}
53
+
54
+
if _, _, ok, _ := d.ResolveExecutorToken("hash-unregistered"); ok {
55
+
t.Fatal("ResolveExecutorToken matched an unregistered hash")
56
+
}
57
+
}
58
+
59
+
func TestRevokeExecutorToken(t *testing.T) {
60
+
d := newTestDB(t)
61
+
62
+
if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil {
63
+
t.Fatalf("AddExecutorToken: %v", err)
64
+
}
65
+
66
+
deleted, err := d.RevokeExecutorToken("exec-1")
67
+
if err != nil {
68
+
t.Fatalf("RevokeExecutorToken: %v", err)
69
+
}
70
+
if !deleted {
71
+
t.Fatal("RevokeExecutorToken reported no deletion for an existing identity")
72
+
}
73
+
74
+
if _, _, ok, _ := d.ResolveExecutorToken("hash-1"); ok {
75
+
t.Fatal("revoked token still resolves; revocation is not enforced")
76
+
}
77
+
78
+
if deleted, err := d.RevokeExecutorToken("exec-1"); err != nil || deleted {
79
+
t.Fatalf("RevokeExecutorToken(already-gone) = (%v, %v), want (false, nil)", deleted, err)
80
+
}
81
+
82
+
if deleted, err := d.RevokeExecutorToken("ghost"); err != nil || deleted {
83
+
t.Fatalf("RevokeExecutorToken(unknown) = (%v, %v), want (false, nil)", deleted, err)
84
+
}
85
+
}
86
+
87
+
func TestExecutorQuarantineIsVisibleAndReversible(t *testing.T) {
88
+
d := newTestDB(t)
89
+
if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil {
90
+
t.Fatalf("AddExecutorToken: %v", err)
91
+
}
92
+
if err := d.QuarantineExecutor("exec-1", "missed cancel deadline"); err != nil {
93
+
t.Fatalf("QuarantineExecutor: %v", err)
94
+
}
95
+
if _, _, ok, err := d.ResolveExecutorToken("hash-1"); err != nil || ok {
96
+
t.Fatalf("ResolveExecutorToken(quarantined) = (ok=%v, err=%v), want (false, nil)", ok, err)
97
+
}
98
+
tokens, err := d.ListExecutorTokens()
99
+
if err != nil {
100
+
t.Fatalf("ListExecutorTokens: %v", err)
101
+
}
102
+
if len(tokens) != 1 || tokens[0].QuarantineReason == nil || *tokens[0].QuarantineReason != "missed cancel deadline" || tokens[0].QuarantinedAt == nil {
103
+
t.Fatalf("quarantined token not surfaced: %+v", tokens)
104
+
}
105
+
if cleared, err := d.ClearExecutorQuarantine("exec-1"); err != nil || !cleared {
106
+
t.Fatalf("ClearExecutorQuarantine = (%v, %v), want (true, nil)", cleared, err)
107
+
}
108
+
if _, _, ok, err := d.ResolveExecutorToken("hash-1"); err != nil || !ok {
109
+
t.Fatalf("ResolveExecutorToken(cleared) = (ok=%v, err=%v), want (true, nil)", ok, err)
110
+
}
111
+
if cleared, err := d.ClearExecutorQuarantine("missing"); err != nil || cleared {
112
+
t.Fatalf("ClearExecutorQuarantine(missing) = (%v, %v), want (false, nil)", cleared, err)
113
+
}
114
+
}
115
+
116
+
func TestListExecutorTokens(t *testing.T) {
117
+
d := newTestDB(t)
118
+
119
+
tokens, err := d.ListExecutorTokens()
120
+
if err != nil {
121
+
t.Fatalf("ListExecutorTokens(empty): %v", err)
122
+
}
123
+
if len(tokens) != 0 {
124
+
t.Fatalf("ListExecutorTokens on empty table = %d rows, want 0", len(tokens))
125
+
}
126
+
127
+
// insert out of alphabetical order to test query sorting
128
+
for _, name := range []string{"charlie", "alice", "bob"} {
129
+
if err := d.AddExecutorToken(name, "hash-"+name, nil, nil); err != nil {
130
+
t.Fatalf("AddExecutorToken(%s): %v", name, err)
131
+
}
132
+
}
133
+
134
+
tokens, err = d.ListExecutorTokens()
135
+
if err != nil {
136
+
t.Fatalf("ListExecutorTokens: %v", err)
137
+
}
138
+
want := []string{"alice", "bob", "charlie"}
139
+
if len(tokens) != len(want) {
140
+
t.Fatalf("ListExecutorTokens = %d rows, want %d", len(tokens), len(want))
141
+
}
142
+
for i := range want {
143
+
if tokens[i].Name != want[i] {
144
+
t.Fatalf("ListExecutorTokens[%d].Name = %q, want %q (ordered by name)", i, tokens[i].Name, want[i])
145
+
}
146
+
}
147
+
}
148
+
149
+
// expired tokens must fail closed
150
+
func TestResolveExecutorTokenExpiry(t *testing.T) {
151
+
cases := []struct {
152
+
name string
153
+
seqno time.Duration
154
+
noExpiry bool
155
+
wantOK bool
156
+
}{
157
+
{"future expiry resolves", time.Hour, false, true},
158
+
{"past expiry fails closed", -time.Hour, false, false},
159
+
{"nil expiry never expires", 0, true, true},
160
+
}
161
+
for _, tc := range cases {
162
+
t.Run(tc.name, func(t *testing.T) {
163
+
d := newTestDB(t)
164
+
165
+
var expires *time.Time
166
+
if !tc.noExpiry {
167
+
exp := time.Now().Add(tc.seqno)
168
+
expires = &exp
169
+
}
170
+
if err := d.AddExecutorToken("exec-1", "hash-1", expires, nil); err != nil {
171
+
t.Fatalf("AddExecutorToken: %v", err)
172
+
}
173
+
174
+
name, _, ok, err := d.ResolveExecutorToken("hash-1")
175
+
if err != nil {
176
+
t.Fatalf("ResolveExecutorToken: %v", err)
177
+
}
178
+
if ok != tc.wantOK {
179
+
t.Fatalf("ResolveExecutorToken ok = %v, want %v", ok, tc.wantOK)
180
+
}
181
+
if tc.wantOK && name != "exec-1" {
182
+
t.Fatalf("ResolveExecutorToken name = %q, want exec-1", name)
183
+
}
184
+
if !tc.wantOK && name != "" {
185
+
t.Fatalf("ResolveExecutorToken name = %q, want \"\" when failing closed", name)
186
+
}
187
+
})
188
+
}
189
+
}
190
+
191
+
func TestResolveExecutorTokenRejectsMalformedExpiry(t *testing.T) {
192
+
d := newTestDB(t)
193
+
if _, err := d.Exec(
194
+
`insert into mill_executors (name, token_hash, expires_at) values (?, ?, ?)`,
195
+
"exec-1", "hash-1", "not-a-timestamp",
196
+
); err != nil {
197
+
t.Fatalf("insert malformed token: %v", err)
198
+
}
199
+
200
+
name, _, ok, err := d.ResolveExecutorToken("hash-1")
201
+
if err == nil {
202
+
t.Fatal("ResolveExecutorToken accepted a malformed non-NULL expiry")
203
+
}
204
+
if ok || name != "" {
205
+
t.Fatalf("ResolveExecutorToken = (%q, %v, %v), want (\"\", false, error)", name, ok, err)
206
+
}
207
+
}
208
+
209
+
// expiry storage is RFC3339 second precision UTC so
210
+
// round trips compare to the second
211
+
func TestListExecutorTokensSurfacesExpiry(t *testing.T) {
212
+
d := newTestDB(t)
213
+
214
+
exp := time.Now().Add(24 * time.Hour)
215
+
if err := d.AddExecutorToken("expiring", "hash-exp", &exp, nil); err != nil {
216
+
t.Fatalf("AddExecutorToken(expiring): %v", err)
217
+
}
218
+
if err := d.AddExecutorToken("forever", "hash-forever", nil, nil); err != nil {
219
+
t.Fatalf("AddExecutorToken(forever): %v", err)
220
+
}
221
+
222
+
tokens, err := d.ListExecutorTokens()
223
+
if err != nil {
224
+
t.Fatalf("ListExecutorTokens: %v", err)
225
+
}
226
+
227
+
got := make(map[string]*time.Time, len(tokens))
228
+
for _, tok := range tokens {
229
+
got[tok.Name] = tok.ExpiresAt
230
+
}
231
+
232
+
e, present := got["forever"]
233
+
if !present {
234
+
t.Fatal("ListExecutorTokens omitted the non-expiring identity")
235
+
}
236
+
if e != nil {
237
+
t.Fatalf("forever.ExpiresAt = %v, want nil (never expires)", e)
238
+
}
239
+
240
+
e, present = got["expiring"]
241
+
if !present {
242
+
t.Fatal("ListExecutorTokens omitted the expiring identity")
243
+
}
244
+
if e == nil {
245
+
t.Fatal("expiring.ExpiresAt = nil, want the stored expiry")
246
+
}
247
+
if e.Unix() != exp.Unix() {
248
+
t.Fatalf("expiring.ExpiresAt = %d (unix), want %d", e.Unix(), exp.Unix())
249
+
}
250
+
}
251
+
252
+
func TestExecutorTokenLabels(t *testing.T) {
253
+
d := newTestDB(t)
254
+
255
+
labels := []string{" foo ", "bar", " foo", ""}
256
+
if err := d.AddExecutorToken("exec-1", "hash-1", nil, labels); err != nil {
257
+
t.Fatalf("AddExecutorToken: %v", err)
258
+
}
259
+
260
+
name, resolvedLabels, ok, err := d.ResolveExecutorToken("hash-1")
261
+
if err != nil {
262
+
t.Fatalf("ResolveExecutorToken: %v", err)
263
+
}
264
+
if !ok || name != "exec-1" {
265
+
t.Fatalf("ResolveExecutorToken: ok=%v, name=%q, want true, exec-1", ok, name)
266
+
}
267
+
268
+
wantLabels := []string{"bar", "foo"}
269
+
if !slices.Equal(resolvedLabels, wantLabels) {
270
+
t.Fatalf("resolved labels = %v, want %v", resolvedLabels, wantLabels)
271
+
}
272
+
}
+53
-39
spindle/engine/engine.go
+53
-39
spindle/engine/engine.go
···
18
18
var (
19
19
ErrTimedOut = errors.New("timed out")
20
20
ErrWorkflowFailed = errors.New("workflow failed")
21
+
ErrCancelled = errors.New("workflow cancelled")
21
22
)
22
23
23
-
type workflowFinalizer interface {
24
-
FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error
24
+
// the mill streams the executor's real log file into place itself
25
+
// a local logger here would only write competing lines
26
+
type workflowLoggerProvider interface {
27
+
WorkflowLogger(wid models.WorkflowId) models.WorkflowLogger
28
+
}
29
+
30
+
// for engines that manage status updates outside StartWorkflows
31
+
type RemoteStatusEngine interface {
32
+
AuthorsRemoteStatus()
33
+
}
34
+
35
+
func reportWorkflowStatusError(l *slog.Logger, database *db.DB, n *notifier.Notifier, wid models.WorkflowId, err error) {
36
+
if errors.Is(err, ErrTimedOut) {
37
+
dbErr := database.StatusTimeout(wid, n)
38
+
if dbErr != nil {
39
+
l.Error("failed to set workflow status to timeout", "wid", wid, "err", dbErr)
40
+
}
41
+
} else if errors.Is(err, ErrCancelled) {
42
+
dbErr := database.StatusCancelled(wid, err.Error(), -1, n)
43
+
if dbErr != nil {
44
+
l.Error("failed to set workflow status to cancelled", "wid", wid, "err", dbErr)
45
+
}
46
+
} else {
47
+
dbErr := database.StatusFailed(wid, err.Error(), -1, n)
48
+
if dbErr != nil {
49
+
l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr)
50
+
}
51
+
}
25
52
}
26
53
27
54
func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) {
···
68
95
}
69
96
}()
70
97
71
-
wfLogger, err := models.NewFileWorkflowLogger(cfg.Server.LogDir, wid, secretValues)
72
-
if err != nil {
98
+
var wfLogger models.WorkflowLogger
99
+
if p, ok := eng.(workflowLoggerProvider); ok {
100
+
wfLogger = p.WorkflowLogger(wid)
101
+
} else if fileLogger, err := models.NewFileWorkflowLogger(cfg.Server.LogDir, wid, secretValues); err != nil {
73
102
l.Warn("failed to setup step logger; logs will not be persisted", "error", err)
74
103
wfLogger = models.NullLogger{}
75
104
} else {
76
105
l.Info("setup step logger; logs will be persisted", "logDir", cfg.Server.LogDir, "wid", wid)
77
-
defer wfLogger.Close()
106
+
wfLogger = fileLogger
107
+
defer fileLogger.Close()
78
108
}
79
109
80
110
l.Info("waiting for slot", "wid", wid)
81
111
slot := WorkflowSlot(NoopSlot{})
112
+
_, remoteStatus := eng.(RemoteStatusEngine)
113
+
82
114
if s, ok := eng.(WorkflowSlotter); ok {
83
115
var err error
84
-
slot, err = s.AcquireWorkflowSlot(ctx, wid, &w)
116
+
slot, err = s.AcquireWorkflowSlot(ctx, wid, &w, Wait)
85
117
if err != nil {
86
118
l.Error("failed to acquire slot", "wid", wid, "err", err)
87
-
dbErr := db.StatusFailed(wid, err.Error(), -1, n)
88
-
if dbErr != nil {
89
-
l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr)
90
-
}
119
+
reportWorkflowStatusError(l, db, n, wid, err)
91
120
return
92
121
}
93
122
}
94
123
defer slot.Release()
95
124
96
-
err = db.StatusRunning(wid, n)
97
-
if err != nil {
98
-
l.Error("failed to set workflow status to running", "wid", wid, "err", err)
99
-
return
125
+
if !remoteStatus {
126
+
err := db.StatusRunning(wid, n)
127
+
if err != nil {
128
+
l.Error("failed to set workflow status to running", "wid", wid, "err", err)
129
+
return
130
+
}
100
131
}
101
132
102
133
err = eng.SetupWorkflow(ctx, wid, &w, wfLogger)
···
110
141
l.Error("failed to destroy workflow after setup failure", "error", destroyErr)
111
142
}
112
143
113
-
dbErr := db.StatusFailed(wid, err.Error(), -1, n)
114
-
if dbErr != nil {
115
-
l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr)
144
+
if !remoteStatus {
145
+
reportWorkflowStatusError(l, db, n, wid, err)
116
146
}
117
147
return
118
148
}
···
139
169
}
140
170
141
171
if err != nil {
142
-
if errors.Is(err, ErrTimedOut) {
143
-
dbErr := db.StatusTimeout(wid, n)
144
-
if dbErr != nil {
145
-
l.Error("failed to set workflow status to timeout", "wid", wid, "err", dbErr)
146
-
}
147
-
} else {
148
-
dbErr := db.StatusFailed(wid, err.Error(), -1, n)
149
-
if dbErr != nil {
150
-
l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr)
151
-
}
172
+
if !remoteStatus {
173
+
reportWorkflowStatusError(l, db, n, wid, err)
152
174
}
153
175
return
154
176
}
155
177
}
156
178
157
-
if finalizer, ok := eng.(workflowFinalizer); ok {
158
-
if err := finalizer.FinalizeWorkflow(ctx, wid, &w, wfLogger); err != nil {
159
-
dbErr := db.StatusFailed(wid, err.Error(), -1, n)
160
-
if dbErr != nil {
161
-
l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr)
162
-
}
163
-
return
179
+
if !remoteStatus {
180
+
err = db.StatusSuccess(wid, n)
181
+
if err != nil {
182
+
l.Error("failed to set workflow status to success", "wid", wid, "err", err)
164
183
}
165
-
}
166
-
167
-
err = db.StatusSuccess(wid, n)
168
-
if err != nil {
169
-
l.Error("failed to set workflow status to success", "wid", wid, "err", err)
170
184
}
171
185
})
172
186
}
+345
spindle/engine/engine_test.go
+345
spindle/engine/engine_test.go
···
1
+
package engine
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"errors"
7
+
"io"
8
+
"log/slog"
9
+
"path/filepath"
10
+
"testing"
11
+
"time"
12
+
13
+
"github.com/bluesky-social/indigo/atproto/syntax"
14
+
"tangled.org/core/api/tangled"
15
+
"tangled.org/core/notifier"
16
+
"tangled.org/core/spindle/config"
17
+
"tangled.org/core/spindle/db"
18
+
"tangled.org/core/spindle/models"
19
+
"tangled.org/core/spindle/secrets"
20
+
)
21
+
22
+
type mockSecretsManager struct{}
23
+
24
+
func (m *mockSecretsManager) AddSecret(ctx context.Context, secret secrets.UnlockedSecret) error {
25
+
return nil
26
+
}
27
+
func (m *mockSecretsManager) RemoveSecret(ctx context.Context, secret secrets.Secret[any]) error {
28
+
return nil
29
+
}
30
+
func (m *mockSecretsManager) GetSecretsLocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.LockedSecret, error) {
31
+
return nil, nil
32
+
}
33
+
func (m *mockSecretsManager) GetSecretsUnlocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.UnlockedSecret, error) {
34
+
return nil, nil
35
+
}
36
+
37
+
type testStep struct {
38
+
name string
39
+
command string
40
+
}
41
+
42
+
func (s testStep) Name() string { return s.name }
43
+
func (s testStep) Command() string { return s.command }
44
+
func (s testStep) Kind() models.StepKind { return models.StepKindUser }
45
+
46
+
type mockEngine struct {
47
+
timeout time.Duration
48
+
setupErr error
49
+
runErr error
50
+
destroyErr error
51
+
}
52
+
53
+
func (m *mockEngine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) {
54
+
return &models.Workflow{
55
+
Name: twf.Name,
56
+
Steps: []models.Step{testStep{name: "step-1"}},
57
+
}, nil
58
+
}
59
+
60
+
func (m *mockEngine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error {
61
+
return m.setupErr
62
+
}
63
+
64
+
func (m *mockEngine) WorkflowTimeout() time.Duration {
65
+
if m.timeout == 0 {
66
+
return 5 * time.Second
67
+
}
68
+
return m.timeout
69
+
}
70
+
71
+
func (m *mockEngine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error {
72
+
return m.destroyErr
73
+
}
74
+
75
+
func (m *mockEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error {
76
+
return m.runErr
77
+
}
78
+
79
+
type mockRemoteStatusEngine struct {
80
+
mockEngine
81
+
}
82
+
83
+
func (m *mockRemoteStatusEngine) AuthorsRemoteStatus() {}
84
+
85
+
type mockRemoteSlotter struct {
86
+
mockRemoteStatusEngine
87
+
acquireErr error
88
+
}
89
+
90
+
func (m *mockRemoteSlotter) AcquireWorkflowSlot(context.Context, models.WorkflowId, *models.Workflow, AcquireMode) (WorkflowSlot, error) {
91
+
return nil, m.acquireErr
92
+
}
93
+
94
+
func newTestDB(t *testing.T) *db.DB {
95
+
t.Helper()
96
+
d, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "spindle.db"))
97
+
if err != nil {
98
+
t.Fatalf("Make: %v", err)
99
+
}
100
+
t.Cleanup(func() { d.Close() })
101
+
return d
102
+
}
103
+
104
+
func getWorkflowStatuses(t *testing.T, database *db.DB, wid models.WorkflowId) []string {
105
+
t.Helper()
106
+
rows, err := database.Query(`
107
+
select event from events
108
+
where nsid = ?
109
+
and json_extract(event, '$.pipeline') = ?
110
+
and json_extract(event, '$.workflow') = ?
111
+
order by created asc
112
+
`, tangled.PipelineStatusNSID, string(wid.PipelineId.AtUri()), wid.Name)
113
+
if err != nil {
114
+
t.Fatalf("Query: %v", err)
115
+
}
116
+
defer rows.Close()
117
+
118
+
var statuses []string
119
+
for rows.Next() {
120
+
var eventJson string
121
+
if err := rows.Scan(&eventJson); err != nil {
122
+
t.Fatalf("Scan: %v", err)
123
+
}
124
+
var status struct {
125
+
Status string `json:"status"`
126
+
}
127
+
if err := json.Unmarshal([]byte(eventJson), &status); err != nil {
128
+
t.Fatalf("Unmarshal: %v", err)
129
+
}
130
+
statuses = append(statuses, status.Status)
131
+
}
132
+
return statuses
133
+
}
134
+
135
+
func TestStartWorkflows_OrdinaryEngine(t *testing.T) {
136
+
database := newTestDB(t)
137
+
vault := &mockSecretsManager{}
138
+
cfg := &config.Config{}
139
+
cfg.Server.LogDir = t.TempDir()
140
+
n := notifier.New()
141
+
l := slog.New(slog.NewTextHandler(io.Discard, nil))
142
+
143
+
pipelineId := models.PipelineId{
144
+
Knot: "test-knot",
145
+
Rkey: "test-rkey",
146
+
}
147
+
148
+
t.Run("success path", func(t *testing.T) {
149
+
eng := &mockEngine{}
150
+
pipeline := &models.Pipeline{
151
+
RepoDid: syntax.DID("did:plc:testrepo"),
152
+
Workflows: map[models.Engine][]models.Workflow{
153
+
eng: {
154
+
{
155
+
Name: "ci",
156
+
Steps: []models.Step{testStep{name: "step-1"}},
157
+
},
158
+
},
159
+
},
160
+
TrustedSource: true,
161
+
}
162
+
163
+
StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId)
164
+
165
+
wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci"}
166
+
statuses := getWorkflowStatuses(t, database, wid)
167
+
want := []string{"running", "success"}
168
+
if len(statuses) != len(want) {
169
+
t.Fatalf("got statuses %v, want %v", statuses, want)
170
+
}
171
+
for i, s := range statuses {
172
+
if s != want[i] {
173
+
t.Errorf("status[%d] = %s, want %s", i, s, want[i])
174
+
}
175
+
}
176
+
})
177
+
178
+
t.Run("setup failure", func(t *testing.T) {
179
+
eng := &mockEngine{setupErr: errors.New("setup failed")}
180
+
pipeline := &models.Pipeline{
181
+
RepoDid: syntax.DID("did:plc:testrepo"),
182
+
Workflows: map[models.Engine][]models.Workflow{
183
+
eng: {
184
+
{
185
+
Name: "ci-setup-fail",
186
+
Steps: []models.Step{testStep{name: "step-1"}},
187
+
},
188
+
},
189
+
},
190
+
TrustedSource: true,
191
+
}
192
+
193
+
StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId)
194
+
195
+
wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-setup-fail"}
196
+
statuses := getWorkflowStatuses(t, database, wid)
197
+
want := []string{"running", "failed"}
198
+
if len(statuses) != len(want) {
199
+
t.Fatalf("got statuses %v, want %v", statuses, want)
200
+
}
201
+
for i, s := range statuses {
202
+
if s != want[i] {
203
+
t.Errorf("status[%d] = %s, want %s", i, s, want[i])
204
+
}
205
+
}
206
+
})
207
+
208
+
t.Run("run failure", func(t *testing.T) {
209
+
eng := &mockEngine{runErr: errors.New("run failed")}
210
+
pipeline := &models.Pipeline{
211
+
RepoDid: syntax.DID("did:plc:testrepo"),
212
+
Workflows: map[models.Engine][]models.Workflow{
213
+
eng: {
214
+
{
215
+
Name: "ci-run-fail",
216
+
Steps: []models.Step{testStep{name: "step-1"}},
217
+
},
218
+
},
219
+
},
220
+
TrustedSource: true,
221
+
}
222
+
223
+
StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId)
224
+
225
+
wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-run-fail"}
226
+
statuses := getWorkflowStatuses(t, database, wid)
227
+
want := []string{"running", "failed"}
228
+
if len(statuses) != len(want) {
229
+
t.Fatalf("got statuses %v, want %v", statuses, want)
230
+
}
231
+
for i, s := range statuses {
232
+
if s != want[i] {
233
+
t.Errorf("status[%d] = %s, want %s", i, s, want[i])
234
+
}
235
+
}
236
+
})
237
+
}
238
+
239
+
func TestStartWorkflows_RemoteStatusEngine(t *testing.T) {
240
+
database := newTestDB(t)
241
+
vault := &mockSecretsManager{}
242
+
cfg := &config.Config{}
243
+
cfg.Server.LogDir = t.TempDir()
244
+
n := notifier.New()
245
+
l := slog.New(slog.NewTextHandler(io.Discard, nil))
246
+
247
+
pipelineId := models.PipelineId{
248
+
Knot: "test-knot",
249
+
Rkey: "test-rkey",
250
+
}
251
+
252
+
t.Run("acquisition failure writes terminal status", func(t *testing.T) {
253
+
eng := &mockRemoteSlotter{acquireErr: errors.New("placement failed")}
254
+
pipeline := &models.Pipeline{
255
+
RepoDid: syntax.DID("did:plc:testrepo"),
256
+
Workflows: map[models.Engine][]models.Workflow{
257
+
eng: {{
258
+
Name: "ci-remote-acquire-fail",
259
+
Steps: []models.Step{testStep{name: "step-1"}},
260
+
}},
261
+
},
262
+
TrustedSource: true,
263
+
}
264
+
265
+
StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId)
266
+
267
+
wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-remote-acquire-fail"}
268
+
statuses := getWorkflowStatuses(t, database, wid)
269
+
if len(statuses) != 1 || statuses[0] != "failed" {
270
+
t.Fatalf("acquisition statuses = %v, want [failed]", statuses)
271
+
}
272
+
})
273
+
274
+
t.Run("success path writes no status events", func(t *testing.T) {
275
+
eng := &mockRemoteStatusEngine{}
276
+
pipeline := &models.Pipeline{
277
+
RepoDid: syntax.DID("did:plc:testrepo"),
278
+
Workflows: map[models.Engine][]models.Workflow{
279
+
eng: {
280
+
{
281
+
Name: "ci-remote-success",
282
+
Steps: []models.Step{testStep{name: "step-1"}},
283
+
},
284
+
},
285
+
},
286
+
TrustedSource: true,
287
+
}
288
+
289
+
StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId)
290
+
291
+
wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-remote-success"}
292
+
statuses := getWorkflowStatuses(t, database, wid)
293
+
if len(statuses) != 0 {
294
+
t.Fatalf("expected 0 status events, got %v", statuses)
295
+
}
296
+
})
297
+
298
+
t.Run("setup failure writes no status events", func(t *testing.T) {
299
+
eng := &mockRemoteStatusEngine{mockEngine: mockEngine{setupErr: errors.New("setup failed")}}
300
+
pipeline := &models.Pipeline{
301
+
RepoDid: syntax.DID("did:plc:testrepo"),
302
+
Workflows: map[models.Engine][]models.Workflow{
303
+
eng: {
304
+
{
305
+
Name: "ci-remote-setup-fail",
306
+
Steps: []models.Step{testStep{name: "step-1"}},
307
+
},
308
+
},
309
+
},
310
+
TrustedSource: true,
311
+
}
312
+
313
+
StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId)
314
+
315
+
wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-remote-setup-fail"}
316
+
statuses := getWorkflowStatuses(t, database, wid)
317
+
if len(statuses) != 0 {
318
+
t.Fatalf("expected 0 status events, got %v", statuses)
319
+
}
320
+
})
321
+
322
+
t.Run("run failure writes no status events", func(t *testing.T) {
323
+
eng := &mockRemoteStatusEngine{mockEngine: mockEngine{runErr: errors.New("run failed")}}
324
+
pipeline := &models.Pipeline{
325
+
RepoDid: syntax.DID("did:plc:testrepo"),
326
+
Workflows: map[models.Engine][]models.Workflow{
327
+
eng: {
328
+
{
329
+
Name: "ci-remote-run-fail",
330
+
Steps: []models.Step{testStep{name: "step-1"}},
331
+
},
332
+
},
333
+
},
334
+
TrustedSource: true,
335
+
}
336
+
337
+
StartWorkflows(l, vault, cfg, database, &n, context.Background(), pipeline, pipelineId)
338
+
339
+
wid := models.WorkflowId{PipelineId: pipelineId, Name: "ci-remote-run-fail"}
340
+
statuses := getWorkflowStatuses(t, database, wid)
341
+
if len(statuses) != 0 {
342
+
t.Fatalf("expected 0 status events, got %v", statuses)
343
+
}
344
+
})
345
+
}
+8
spindle/engine/manifest_test.go
+8
spindle/engine/manifest_test.go
···
79
79
}
80
80
}
81
81
82
+
func TestDescribeManifestErrorAcceptsRunsOnGenericWorkflowKey(t *testing.T) {
83
+
raw := "engine: microvm\nruns_on: [linux/arm64, kvm]\nimage: nixos\n"
84
+
85
+
if err := DescribeManifestError(raw, testManifest{}); err != nil {
86
+
t.Fatalf("DescribeManifestError(%q) = %v, want nil", raw, err)
87
+
}
88
+
}
89
+
82
90
func TestDescribeManifestErrorNoFalsePositives(t *testing.T) {
83
91
cases := []string{
84
92
// well-formed manifest
+10
spindle/engine/placement.go
+10
spindle/engine/placement.go
+12
-8
spindle/engine/scheduler.go
+12
-8
spindle/engine/scheduler.go
···
50
50
}
51
51
}
52
52
53
-
func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R) (WorkflowSlot, error) {
53
+
// the mill owns the backlog so a NoWait caller must have room immediately or fail
54
+
func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R, mode AcquireMode) (WorkflowSlot, error) {
54
55
if s == nil {
55
56
return NoopSlot{}, nil
56
57
}
···
60
61
s.mu.Unlock()
61
62
return nil, fmt.Errorf("%w: request=%v budget=%v max=%v", ErrNoWorkflowSlots, req, s.budget, s.max)
62
63
}
63
-
if len(s.queue) == 0 && s.used.Add(req).Fits(s.budget) {
64
+
// NoWait ignores the queue because it never blocks
65
+
// Wait only bypasses empty queues to prevent starvation
66
+
if s.used.Add(req).Fits(s.budget) && (mode == NoWait || len(s.queue) == 0) {
64
67
s.used = s.used.Add(req)
65
68
s.mu.Unlock()
66
69
return &resourceLease[R]{scheduler: s, req: req}, nil
67
70
}
71
+
if mode == NoWait {
72
+
used := s.used
73
+
s.mu.Unlock()
74
+
return nil, fmt.Errorf("%w: request=%v used=%v budget=%v", ErrNoWorkflowSlots, req, used, s.budget)
75
+
}
68
76
69
77
waiter := &resourceWaiter[R]{req: req, ready: make(chan struct{}), enqueuedAt: s.now()}
70
78
s.queue = append(s.queue, waiter)
···
108
116
109
117
// start every waiter whose request fits. once a waiter is older than
110
118
// agingThreshold, count its request as already used so younger waiters
111
-
// stop being scheduled ahead of it.
119
+
// stop being scheduled ahead of it
112
120
func (s *ResourceScheduler[R]) schedule() {
113
121
var reserved R
114
122
now := s.now()
···
129
137
}
130
138
131
139
func (s *ResourceScheduler[R]) remove(waiter *resourceWaiter[R]) {
132
-
for i, candidate := range s.queue {
133
-
if candidate != waiter {
134
-
continue
135
-
}
140
+
if i := slices.Index(s.queue, waiter); i >= 0 {
136
141
s.queue = slices.Delete(s.queue, i, i+1)
137
-
return
138
142
}
139
143
}
+75
-9
spindle/engine/scheduler_test.go
+75
-9
spindle/engine/scheduler_test.go
···
36
36
37
37
scheduler := NewResourceScheduler(ru{}, ru{}, 0)
38
38
39
-
slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20})
39
+
slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}, Wait)
40
40
if err != nil {
41
41
t.Fatalf("Acquire() error = %v", err)
42
42
}
···
48
48
49
49
scheduler := NewResourceScheduler(ru{a: 1024, b: 10_000}, ru{a: 512, b: 5_000}, 0)
50
50
51
-
_, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100})
51
+
_, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}, Wait)
52
52
if !errors.Is(err, ErrNoWorkflowSlots) {
53
53
t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err)
54
54
}
55
55
56
-
_, err = scheduler.Acquire(context.Background(), ru{a: 128, b: 12_000})
56
+
_, err = scheduler.Acquire(context.Background(), ru{a: 128, b: 12_000}, Wait)
57
57
if !errors.Is(err, ErrNoWorkflowSlots) {
58
58
t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err)
59
59
}
···
64
64
65
65
scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 0)
66
66
67
-
first, err := scheduler.Acquire(context.Background(), ru{a: 1024})
67
+
first, err := scheduler.Acquire(context.Background(), ru{a: 1024}, Wait)
68
68
if err != nil {
69
69
t.Fatalf("first Acquire() error = %v", err)
70
70
}
···
85
85
86
86
scheduler := NewResourceScheduler(ru{a: 1}, ru{}, 0)
87
87
88
-
slot, err := scheduler.Acquire(context.Background(), ru{a: 1})
88
+
slot, err := scheduler.Acquire(context.Background(), ru{a: 1}, Wait)
89
89
if err != nil {
90
90
t.Fatalf("Acquire() error = %v", err)
91
91
}
···
93
93
slot.Release()
94
94
slot.Release()
95
95
96
-
second, err := scheduler.Acquire(context.Background(), ru{a: 1})
96
+
second, err := scheduler.Acquire(context.Background(), ru{a: 1}, Wait)
97
97
if err != nil {
98
98
t.Fatalf("Acquire() after double release error = %v", err)
99
99
}
···
105
105
106
106
scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, time.Hour) // disable aging so we test pure backfill
107
107
108
-
hold, err := scheduler.Acquire(context.Background(), ru{a: 512})
108
+
hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait)
109
109
if err != nil {
110
110
t.Fatalf("hold Acquire() error = %v", err)
111
111
}
···
128
128
fakeNow := time.Now()
129
129
scheduler.now = func() time.Time { return fakeNow }
130
130
131
-
hold, err := scheduler.Acquire(context.Background(), ru{a: 512})
131
+
hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait)
132
132
if err != nil {
133
133
t.Fatalf("hold Acquire() error = %v", err)
134
134
}
···
151
151
big.Release()
152
152
}
153
153
154
+
func TestResourceSchedulerTryRejectsWhenNoRoomNow(t *testing.T) {
155
+
t.Parallel()
156
+
157
+
scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 0)
158
+
159
+
first, err := scheduler.Acquire(context.Background(), ru{a: 1024}, NoWait)
160
+
if err != nil {
161
+
t.Fatalf("first Acquire(NoWait) error = %v", err)
162
+
}
163
+
164
+
if _, err := scheduler.Acquire(context.Background(), ru{a: 1}, NoWait); !errors.Is(err, ErrNoWorkflowSlots) {
165
+
t.Fatalf("Acquire(NoWait) error = %v, want ErrNoWorkflowSlots", err)
166
+
}
167
+
168
+
first.Release()
169
+
170
+
second, err := scheduler.Acquire(context.Background(), ru{a: 1}, NoWait)
171
+
if err != nil {
172
+
t.Fatalf("Acquire(NoWait) after release error = %v", err)
173
+
}
174
+
second.Release()
175
+
}
176
+
177
+
func TestResourceSchedulerTryRejectsRequestsThatCanNeverFit(t *testing.T) {
178
+
t.Parallel()
179
+
180
+
scheduler := NewResourceScheduler(ru{a: 1024, b: 10_000}, ru{a: 512, b: 5_000}, 0)
181
+
182
+
if _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}, NoWait); !errors.Is(err, ErrNoWorkflowSlots) {
183
+
t.Fatalf("Acquire(NoWait) error = %v, want ErrNoWorkflowSlots", err)
184
+
}
185
+
}
186
+
187
+
func TestResourceSchedulerTryIgnoresQueuedWaiters(t *testing.T) {
188
+
t.Parallel()
189
+
190
+
scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, time.Hour)
191
+
192
+
hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait)
193
+
if err != nil {
194
+
t.Fatalf("hold Acquire() error = %v", err)
195
+
}
196
+
defer hold.Release()
197
+
198
+
bigCh := acquireAsync(context.Background(), scheduler, ru{a: 768})
199
+
assertAcquireBlocked(t, bigCh)
200
+
201
+
slot, err := scheduler.Acquire(context.Background(), ru{a: 256}, NoWait)
202
+
if err != nil {
203
+
t.Fatalf("Acquire(NoWait) error = %v, want success past queued waiter", err)
204
+
}
205
+
slot.Release()
206
+
}
207
+
208
+
func TestResourceSchedulerZeroLimitsTryDoesNotReject(t *testing.T) {
209
+
t.Parallel()
210
+
211
+
scheduler := NewResourceScheduler(ru{}, ru{}, 0)
212
+
213
+
slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}, NoWait)
214
+
if err != nil {
215
+
t.Fatalf("Acquire(NoWait) error = %v", err)
216
+
}
217
+
slot.Release()
218
+
}
219
+
154
220
func acquireAsync(ctx context.Context, scheduler *ResourceScheduler[ru], req ru) <-chan acquireResult {
155
221
ch := make(chan acquireResult, 1)
156
222
go func() {
157
-
slot, err := scheduler.Acquire(ctx, req)
223
+
slot, err := scheduler.Acquire(ctx, req, Wait)
158
224
ch <- acquireResult{slot: slot, err: err}
159
225
}()
160
226
return ch
+21
-2
spindle/engine/slot.go
+21
-2
spindle/engine/slot.go
···
13
13
Release()
14
14
}
15
15
16
+
// governs blocking behaviour when acquiring a slot
17
+
type AcquireMode int
18
+
19
+
const (
20
+
// blocks until a slot is free
21
+
Wait AcquireMode = iota
22
+
// fails immediately if full
23
+
// executors use this because the mill owns the backlog
24
+
NoWait
25
+
)
26
+
16
27
type WorkflowSlotter interface {
17
-
AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error)
28
+
AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode AcquireMode) (WorkflowSlot, error)
18
29
}
19
30
20
31
type releaseFunc func()
···
41
52
return &SemaphoreSlotter{slots: make(chan struct{}, maxConcurrent)}
42
53
}
43
54
44
-
func (a *SemaphoreSlotter) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) {
55
+
func (a *SemaphoreSlotter) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode AcquireMode) (WorkflowSlot, error) {
45
56
if a == nil || a.slots == nil {
46
57
return NoopSlot{}, nil
58
+
}
59
+
if mode == NoWait {
60
+
select {
61
+
case a.slots <- struct{}{}:
62
+
return releaseFunc(func() { <-a.slots }), nil
63
+
default:
64
+
return nil, ErrNoWorkflowSlots
65
+
}
47
66
}
48
67
select {
49
68
case a.slots <- struct{}{}:
+42
-5
spindle/engine/slot_test.go
+42
-5
spindle/engine/slot_test.go
···
15
15
slotter := NewSemaphoreSlotter(0)
16
16
17
17
for range 10 {
18
-
slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil)
18
+
slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait)
19
19
if err != nil {
20
20
t.Fatalf("AcquireWorkflowSlot() error = %v", err)
21
21
}
···
28
28
29
29
slotter := NewSemaphoreSlotter(1)
30
30
31
-
first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil)
31
+
first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait)
32
32
if err != nil {
33
33
t.Fatalf("first AcquireWorkflowSlot() error = %v", err)
34
34
}
···
42
42
acquired := make(chan WorkflowSlot, 1)
43
43
errs := make(chan error, 1)
44
44
go func() {
45
-
slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil)
45
+
slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait)
46
46
if err != nil {
47
47
errs <- err
48
48
return
···
64
64
65
65
slotter := NewSemaphoreSlotter(1)
66
66
67
-
first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil)
67
+
first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait)
68
68
if err != nil {
69
69
t.Fatalf("first AcquireWorkflowSlot() error = %v", err)
70
70
}
···
73
73
ctx, cancel := context.WithCancel(context.Background())
74
74
cancel()
75
75
76
-
_, err = slotter.AcquireWorkflowSlot(ctx, zeroWorkflowID(), nil)
76
+
_, err = slotter.AcquireWorkflowSlot(ctx, zeroWorkflowID(), nil, Wait)
77
77
if !errors.Is(err, context.Canceled) {
78
78
t.Fatalf("AcquireWorkflowSlot() error = %v, want context.Canceled", err)
79
79
}
80
+
}
81
+
82
+
func TestSemaphoreSlotterTryDisabledDoesNotReject(t *testing.T) {
83
+
t.Parallel()
84
+
85
+
slotter := NewSemaphoreSlotter(0)
86
+
87
+
for range 10 {
88
+
slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait)
89
+
if err != nil {
90
+
t.Fatalf("AcquireWorkflowSlot(NoWait) error = %v", err)
91
+
}
92
+
slot.Release()
93
+
}
94
+
}
95
+
96
+
func TestSemaphoreSlotterTryRejectsWhenFull(t *testing.T) {
97
+
t.Parallel()
98
+
99
+
slotter := NewSemaphoreSlotter(1)
100
+
101
+
first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait)
102
+
if err != nil {
103
+
t.Fatalf("first AcquireWorkflowSlot(NoWait) error = %v", err)
104
+
}
105
+
106
+
if _, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait); !errors.Is(err, ErrNoWorkflowSlots) {
107
+
t.Fatalf("AcquireWorkflowSlot(NoWait) error = %v, want ErrNoWorkflowSlots", err)
108
+
}
109
+
110
+
first.Release()
111
+
112
+
second, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait)
113
+
if err != nil {
114
+
t.Fatalf("AcquireWorkflowSlot(NoWait) after release error = %v", err)
115
+
}
116
+
second.Release()
80
117
}
81
118
82
119
func assertNotAcquired(t *testing.T, acquired <-chan WorkflowSlot, errs <-chan error) {
+6
spindle/engines/dummy/engine.go
+6
spindle/engines/dummy/engine.go
···
8
8
9
9
"gopkg.in/yaml.v3"
10
10
"tangled.org/core/api/tangled"
11
+
"tangled.org/core/spindle/engine"
11
12
"tangled.org/core/spindle/models"
12
13
"tangled.org/core/spindle/secrets"
13
14
)
···
77
78
78
79
func (e *DummyEngine) WorkflowTimeout() time.Duration {
79
80
return 5 * time.Minute
81
+
}
82
+
83
+
// no capacity limit, so always a no-op slot regardless of mode
84
+
func (e *DummyEngine) AcquireWorkflowSlot(_ context.Context, _ models.WorkflowId, _ *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) {
85
+
return engine.NoopSlot{}, nil
80
86
}
81
87
82
88
func (e *DummyEngine) DestroyWorkflow(_ context.Context, wid models.WorkflowId) error {
+2
-5
spindle/engines/microvm/budget.go
+2
-5
spindle/engines/microvm/budget.go
···
68
68
return budget, maxReq, cfg.AgingThreshold
69
69
}
70
70
71
-
func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (engine.WorkflowSlot, error) {
71
+
func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode engine.AcquireMode) (engine.WorkflowSlot, error) {
72
72
state, ok := wf.Data.(*workflowState)
73
73
if !ok || state == nil {
74
74
return nil, fmt.Errorf("microVM workflow state is not initialized")
···
77
77
return engine.NoopSlot{}, nil
78
78
}
79
79
req := resourcesForImage(state.ImageSpec)
80
-
if req.MemoryMiB < 0 || req.VCPUs < 0 || req.DiskMiB < 0 {
81
-
return nil, fmt.Errorf("microVM resource request must not be negative: %s", req)
82
-
}
83
-
return e.scheduler.Acquire(ctx, req)
80
+
return e.scheduler.Acquire(ctx, req, mode)
84
81
}
85
82
86
83
func resourcesForImage(spec ImageSpec) Resources {
+4
-4
spindle/engines/microvm/engine.go
+4
-4
spindle/engines/microvm/engine.go
···
52
52
agent *agentHub
53
53
scheduler *engine.ResourceScheduler[Resources]
54
54
cgroupParent *CgroupParent
55
+
budget Resources
56
+
maxWorkflow Resources
55
57
56
58
cleanupMu sync.Mutex
57
59
cleanup map[string][]cleanupFunc
···
91
93
db: d,
92
94
scheduler: engine.NewResourceScheduler(budget, max, agingThreshold),
93
95
cgroupParent: cgroupParent,
96
+
budget: budget,
97
+
maxWorkflow: max,
94
98
cleanup: make(map[string][]cleanupFunc),
95
99
}, nil
96
100
}
···
559
563
}
560
564
}
561
565
return cleanupErr
562
-
}
563
-
564
-
func (e *Engine) FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, w *models.Workflow, wfLogger models.WorkflowLogger) error {
565
-
return nil
566
566
}
567
567
568
568
func (e *Engine) WorkflowTimeout() time.Duration {
+17
-20
spindle/engines/microvm/image.go
+17
-20
spindle/engines/microvm/image.go
···
9
9
"os"
10
10
"path/filepath"
11
11
"strings"
12
+
13
+
"tangled.org/core/spindle/engines/microvm/placement"
12
14
)
13
15
14
16
const imageSpecFileName = "spec.json"
···
209
211
210
212
// check if image name is not a path
211
213
func isPlainImageName(name string) bool {
212
-
if name == "" || name == "." || name == ".." {
213
-
return false
214
-
}
215
-
if filepath.IsAbs(name) || strings.ContainsRune(name, '/') || strings.ContainsRune(name, filepath.Separator) {
216
-
return false
217
-
}
218
-
return true
214
+
return placement.IsPlainImageName(name)
219
215
}
220
216
221
217
// returns candidates, which is either a directory or spec file itself
···
240
236
}
241
237
return "", false, err
242
238
}
243
-
if !info.IsDir() {
244
-
return candidate, true, nil
239
+
if info.IsDir() {
240
+
candidate = filepath.Join(candidate, imageSpecFileName)
241
+
info, err = os.Stat(candidate)
242
+
if err != nil {
243
+
if errors.Is(err, os.ErrNotExist) {
244
+
return "", false, fmt.Errorf("microVM image directory %q does not contain %s", filepath.Dir(candidate), imageSpecFileName)
245
+
}
246
+
return "", false, err
247
+
}
248
+
if info.IsDir() {
249
+
return "", false, fmt.Errorf("microVM image spec %q is a directory", candidate)
250
+
}
245
251
}
246
252
247
-
spec := filepath.Join(candidate, imageSpecFileName)
248
-
info, err = os.Stat(spec)
253
+
path, err := filepath.EvalSymlinks(candidate)
249
254
if err != nil {
250
-
if errors.Is(err, os.ErrNotExist) {
251
-
return "", false, fmt.Errorf("microVM image directory %q does not contain %s", candidate, imageSpecFileName)
252
-
}
253
255
return "", false, err
254
256
}
255
-
// this only happens if there is a directory named `spec.json` which would be very silly.
256
-
// but better output an error for it anyway :p
257
-
if info.IsDir() {
258
-
return "", false, fmt.Errorf("microVM image spec %q is a directory", spec)
259
-
}
260
-
return spec, true, nil
257
+
return path, true, nil
261
258
}
+65
spindle/engines/microvm/image_test.go
+65
spindle/engines/microvm/image_test.go
···
65
65
})
66
66
}
67
67
}
68
+
func TestImageSpecPathPinsSymlinkTarget(t *testing.T) {
69
+
dir := t.TempDir()
70
+
imageA := filepath.Join(dir, "image-a")
71
+
imageB := filepath.Join(dir, "image-b")
72
+
writeSpecFile(t, filepath.Join(imageA, imageSpecFileName))
73
+
writeSpecFile(t, filepath.Join(imageB, imageSpecFileName))
74
+
75
+
alias := filepath.Join(dir, "default")
76
+
if err := os.Symlink(imageA, alias); err != nil {
77
+
t.Fatal(err)
78
+
}
79
+
got, ok, err := imageSpecPath(alias)
80
+
if err != nil {
81
+
t.Fatal(err)
82
+
}
83
+
if !ok {
84
+
t.Fatal("imageSpecPath() did not resolve symlinked image")
85
+
}
86
+
87
+
if err := os.Remove(alias); err != nil {
88
+
t.Fatal(err)
89
+
}
90
+
if err := os.Symlink(imageB, alias); err != nil {
91
+
t.Fatal(err)
92
+
}
93
+
want := filepath.Join(imageA, imageSpecFileName)
94
+
if got != want {
95
+
t.Fatalf("imageSpecPath() = %q after alias resolution, want pinned target %q", got, want)
96
+
}
97
+
}
98
+
99
+
func TestImageCapabilityNamesIncludeExactJSONFilenameAndSuffixlessAlias(t *testing.T) {
100
+
exact, suffixless := imageCapabilityNames("alpine.json")
101
+
if exact != "alpine.json" || suffixless != "alpine" {
102
+
t.Fatalf("imageCapabilityNames() = %q, %q; want %q, %q", exact, suffixless, "alpine.json", "alpine")
103
+
}
104
+
}
68
105
69
106
func TestResolveImageDirectoryMissingSpec(t *testing.T) {
70
107
dir := t.TempDir()
···
135
172
t.Fatalf("spec without shell should fail validation, got: %v", err)
136
173
}
137
174
}
175
+
func TestMkfsExt4ForVolumesSkipsLookupWithoutVolumes(t *testing.T) {
176
+
t.Setenv("PATH", "")
177
+
path, err := mkfsExt4ForVolumes(nil, "")
178
+
if err != nil {
179
+
t.Fatalf("mkfsExt4ForVolumes() with no volumes returned error: %v", err)
180
+
}
181
+
if path != "" {
182
+
t.Fatalf("mkfsExt4ForVolumes() = %q with no volumes, want empty path", path)
183
+
}
184
+
185
+
_, err = mkfsExt4ForVolumes([]Volume{{Image: "workspace"}}, "")
186
+
if err == nil || !strings.Contains(err.Error(), "mkfs.ext4") {
187
+
t.Fatalf("mkfsExt4ForVolumes() with a volume and no formatter returned %v", err)
188
+
}
189
+
}
190
+
191
+
func TestQEMURunnerValidateRejectsUnsupportedNetworkTypeBeforeHostChecks(t *testing.T) {
192
+
spec := validImageSpec()
193
+
spec.NetworkInterfaces = []NetworkInterface{{
194
+
Type: "tap",
195
+
ID: "net0",
196
+
MAC: "02:00:00:00:00:01",
197
+
}}
198
+
err := (qemuRunner{}).Validate(spec, false)
199
+
if err == nil || !strings.Contains(err.Error(), `unsupported microvm network interface type "tap"`) {
200
+
t.Fatalf("Validate() error = %v, want unsupported network type", err)
201
+
}
202
+
}
+15
spindle/engines/microvm/placement/placement.go
+15
spindle/engines/microvm/placement/placement.go
···
1
+
package placement
2
+
3
+
func IsNativeArchitecture(imageArch, goArch string) bool {
4
+
normalize := func(arch string) string {
5
+
switch arch {
6
+
case "x86_64", "amd64":
7
+
return "amd64"
8
+
case "aarch64", "arm64":
9
+
return "arm64"
10
+
default:
11
+
return arch
12
+
}
13
+
}
14
+
return imageArch != "" && normalize(imageArch) == normalize(goArch)
15
+
}
+26
spindle/engines/microvm/placement/placement_test.go
+26
spindle/engines/microvm/placement/placement_test.go
···
1
+
package placement
2
+
3
+
import (
4
+
"testing"
5
+
)
6
+
7
+
func TestIsNativeArchitecture(t *testing.T) {
8
+
tests := []struct {
9
+
image string
10
+
host string
11
+
want bool
12
+
}{
13
+
{image: "x86_64", host: "amd64", want: true},
14
+
{image: "amd64", host: "amd64", want: true},
15
+
{image: "aarch64", host: "arm64", want: true},
16
+
{image: "arm64", host: "arm64", want: true},
17
+
{image: "x86_64", host: "arm64", want: false},
18
+
{image: "aarch64", host: "amd64", want: false},
19
+
{image: "", host: "amd64", want: false},
20
+
}
21
+
for _, tt := range tests {
22
+
if got := IsNativeArchitecture(tt.image, tt.host); got != tt.want {
23
+
t.Errorf("IsNativeArchitecture(%q, %q) = %v, want %v", tt.image, tt.host, got, tt.want)
24
+
}
25
+
}
26
+
}
+61
spindle/engines/microvm/placement_linux.go
+61
spindle/engines/microvm/placement_linux.go
···
1
+
//go:build linux
2
+
3
+
package microvm
4
+
5
+
import (
6
+
"fmt"
7
+
"os/exec"
8
+
"runtime"
9
+
10
+
"tangled.org/core/spindle/engines/microvm/placement"
11
+
"tangled.org/core/spindle/models"
12
+
)
13
+
14
+
func (e *Engine) ValidateWorkflowPlacement(wf *models.Workflow) error {
15
+
state, ok := wf.Data.(*workflowState)
16
+
if !ok || state == nil {
17
+
return fmt.Errorf("microVM workflow state is not initialized")
18
+
}
19
+
return e.validateImagePlacement(state.ImageSpec)
20
+
}
21
+
22
+
func (e *Engine) validateImagePlacement(spec ImageSpec) error {
23
+
if err := spec.Validate(); err != nil {
24
+
return err
25
+
}
26
+
if !placement.IsNativeArchitecture(spec.Arch, runtime.GOARCH) {
27
+
return fmt.Errorf("microVM image architecture %q is not native to executor architecture %q", spec.Arch, runtime.GOARCH)
28
+
}
29
+
if err := spec.validateImageFiles(); err != nil {
30
+
return err
31
+
}
32
+
runner, err := runnerFor(spec.RunnerType)
33
+
if err != nil {
34
+
return err
35
+
}
36
+
if err := runner.Validate(spec, e.cfg.MicroVMPipelines.EnableKVM); err != nil {
37
+
return err
38
+
}
39
+
if len(spec.Volumes) > 0 {
40
+
if _, err := exec.LookPath("mkfs.ext4"); err != nil {
41
+
return fmt.Errorf("required host command %q not found in PATH: %w", "mkfs.ext4", err)
42
+
}
43
+
for _, volume := range spec.Volumes {
44
+
if volume.ReadOnly {
45
+
return fmt.Errorf("read-only microvm volume %q is not supported yet", volume.Image)
46
+
}
47
+
if volume.FSType != "ext4" {
48
+
return fmt.Errorf("microvm volume %q uses unsupported fsType %q", volume.Image, volume.FSType)
49
+
}
50
+
if volume.ImageType != "" && volume.ImageType != "raw" {
51
+
return fmt.Errorf("microvm volume %q uses unsupported imageType %q", volume.Image, volume.ImageType)
52
+
}
53
+
}
54
+
}
55
+
56
+
request := resourcesForImage(spec)
57
+
if !request.Fits(e.budget) || !request.Fits(e.maxWorkflow) {
58
+
return fmt.Errorf("microVM image resources exceed executor limits: request=%v budget=%v max=%v", request, e.budget, e.maxWorkflow)
59
+
}
60
+
return nil
61
+
}
+4
spindle/engines/microvm/qemu.go
+4
spindle/engines/microvm/qemu.go
···
73
73
type qemuRunner struct{}
74
74
75
75
func (qemuRunner) Validate(spec ImageSpec, enableKVM bool) error {
76
+
b := newArgBuilder(len(spec.NetworkInterfaces) * 4)
77
+
if err := addQEMUNetworkArgs(&b, spec.NetworkInterfaces); err != nil {
78
+
return err
79
+
}
76
80
if _, err := exec.LookPath(spec.RunnerCmd()); err != nil {
77
81
return fmt.Errorf("required host command %q not found in PATH: %w", spec.RunnerCmd(), err)
78
82
}
+14
-6
spindle/engines/microvm/vm.go
+14
-6
spindle/engines/microvm/vm.go
···
47
47
return nil
48
48
}
49
49
50
+
func mkfsExt4ForVolumes(volumes []Volume, configured string) (string, error) {
51
+
if len(volumes) == 0 || configured != "" {
52
+
return configured, nil
53
+
}
54
+
path, err := exec.LookPath("mkfs.ext4")
55
+
if err != nil {
56
+
return "", fmt.Errorf("mkfs.ext4 command not found in PATH: %w", err)
57
+
}
58
+
return path, nil
59
+
}
60
+
50
61
func prepareVolumes(ctx context.Context, workDir string, volumes []Volume, mkfsExt4 string) (map[string]string, error) {
51
62
paths := make(map[string]string, len(volumes))
52
63
for _, volume := range volumes {
···
361
372
return nil, err
362
373
}
363
374
364
-
mkfsExt4 := cfg.MkfsExt4
365
-
if mkfsExt4 == "" {
366
-
mkfsExt4, err = exec.LookPath("mkfs.ext4")
367
-
if err != nil {
368
-
return nil, fmt.Errorf("mkfs.ext4 command not found in PATH: %w", err)
369
-
}
375
+
mkfsExt4, err := mkfsExt4ForVolumes(cfg.Image.Volumes, cfg.MkfsExt4)
376
+
if err != nil {
377
+
return nil, err
370
378
}
371
379
volumePaths, err := prepareVolumes(ctx, cfg.WorkDir, cfg.Image.Volumes, mkfsExt4)
372
380
if err != nil {
+2
-1
spindle/engines/nixery/engine.go
+2
-1
spindle/engines/nixery/engine.go
···
198
198
ctx context.Context,
199
199
wid models.WorkflowId,
200
200
wf *models.Workflow,
201
+
mode engine.AcquireMode,
201
202
) (engine.WorkflowSlot, error) {
202
203
if e.slotter == nil {
203
204
return engine.NoopSlot{}, nil
204
205
}
205
206
206
-
return e.slotter.AcquireWorkflowSlot(ctx, wid, wf)
207
+
return e.slotter.AcquireWorkflowSlot(ctx, wid, wf, mode)
207
208
}
208
209
209
210
func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) (err error) {
+42
spindle/logview/logview.go
+42
spindle/logview/logview.go
···
1
+
package logview
2
+
3
+
import (
4
+
"context"
5
+
"io"
6
+
"strings"
7
+
8
+
"github.com/hpcloud/tail"
9
+
"tangled.org/core/spindle/db"
10
+
"tangled.org/core/spindle/models"
11
+
)
12
+
13
+
// streams a wf's log lines. a live mill job reads from the canonical
14
+
// table, everything else (finished jobs, standalone spindles) tails the
15
+
// cold log file. stop ends a live follow early, the channel closes when
16
+
// the source drains or ctx ends
17
+
func Follow(ctx context.Context, d *db.DB, logDir string, wid models.WorkflowId, mill bool, finished bool) (<-chan *tail.Line, func(), error) {
18
+
if mill && !finished {
19
+
followCtx, cancel := context.WithCancel(ctx)
20
+
path := models.LogFilePath(logDir, wid)
21
+
payloads := d.FollowCanonicalLog(followCtx, path)
22
+
lines := make(chan *tail.Line, 64)
23
+
go func() {
24
+
defer close(lines)
25
+
for p := range payloads {
26
+
lines <- &tail.Line{Text: strings.TrimSuffix(string(p), "\n")}
27
+
}
28
+
}()
29
+
return lines, cancel, nil
30
+
}
31
+
32
+
t, err := tail.TailFile(models.LogFilePath(logDir, wid), tail.Config{
33
+
Follow: !finished,
34
+
ReOpen: !finished,
35
+
MustExist: false,
36
+
Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart},
37
+
})
38
+
if err != nil {
39
+
return nil, nil, err
40
+
}
41
+
return t.Lines, func() { _ = t.Stop() }, nil
42
+
}
+159
spindle/mill/README.md
+159
spindle/mill/README.md
···
1
+
# spindle mill
2
+
3
+
This document describes the architecture of the mill, spindle's distributed
4
+
job placement layer. A mill is a spindle that doesn't run jobs itself: it
5
+
places them on remote executors, which run the real engines (microvm, nixery,
6
+
dummy) exactly as they would standalone.
7
+
8
+
## The engine mirroring model
9
+
10
+
Neither side's execution path changes between local and remote runs.
11
+
12
+
On the mill host, engines are registered under the real engine names
13
+
("microvm", "nixery", "dummy"), but each is a stand-in that places jobs
14
+
remotely instead of running them. `InitWorkflow` on the stand-in parses
15
+
nothing, it stashes the raw pipeline and workflow and builds a synthetic
16
+
one-step workflow, so everything upstream (trigger handling, pending status,
17
+
`TANGLED_*` env) behaves exactly like a local run. The real `InitWorkflow`
18
+
runs exactly once, on the executor.
19
+
20
+
On the executor, the reserve handler wraps the real engine in a
21
+
`reservedEngine`: the slot acquired when the reservation is accepted is the
22
+
same slot `StartWorkflows` later runs on. Nothing acquires twice, and from the
23
+
engine's point of view the job is indistinguishable from a local one.
24
+
25
+
The same mirroring applies to output: the executor's jobs write status rows
26
+
and log lines exactly like a standalone spindle, and the mill re-authors them
27
+
into its own event stream and log store, so appview and the log endpoints see
28
+
no difference between a local and a remote job.
29
+
30
+
The only real divergence: secrets are withheld from untrusted pipelines at the
31
+
mill, and they cross the wire exactly once, inside `CommitLease`, to the one
32
+
node that won the bid. Losing bidders never see them.
33
+
34
+
## Placement: labels, seats, and rejection
35
+
36
+
Placement decides which executors can run a workflow right now, using one
37
+
kind of fact:
38
+
39
+
- **labels** are operator-defined strings on the executor's token, checked
40
+
against the workflow's `runs_on`. they're for coarse fleet partitioning
41
+
("this pool is for trusted jobs", "this box has a gpu"), nothing more
42
+
43
+
Matching is exact intersection: candidate sessions whose snapshot has the
44
+
engine available, a free seat, and labels covering `runs_on`. Among the
45
+
eligible candidates the mill ranks least-loaded first and bids the top-K
46
+
concurrently with `ReserveSeat`. The best-ranked accept wins, losers get
47
+
`ReleaseLease` so they free their held seats immediately. If nobody
48
+
accepts, the job waits for a change: a new executor connecting, a snapshot
49
+
flipping availability, or a lease finishing somewhere.
50
+
51
+
Image and arch compatibility is deliberately *not* a mill concern. The mill
52
+
never parses an image name or an arch string; the executor re-validates
53
+
everything at reserve time against its own disk (engine exists, workflow
54
+
parses, the image spec validates and is natively runnable, the runner is
55
+
usable) and rejects what it can't run with `incompatible`. A reject rotates
56
+
the bid to the next candidate, and if every candidate rejects, the user
57
+
gets the collected reasons as the placement error. Multi-arch falls out of
58
+
this: an arm64 box simply cannot accept an x86_64 image, so it never holds
59
+
one. If you *want* to pin an arch or an image explicitly, label the nodes
60
+
and use `runs_on`.
61
+
62
+
### Example: an alpine microvm job
63
+
64
+
Say a workflow asks for the microvm engine with `image: alpine`, no
65
+
`runs_on`, and the fleet looks like this:
66
+
67
+
| node | arch | labels | engines | load |
68
+
|---------|---------|---------|-------------------|--------|
69
+
| ci-1 | x86_64 | [linux] | microvm | busy |
70
+
| ci-2 | aarch64 | [linux] | microvm | idle |
71
+
| ci-3 | x86_64 | [linux] | microvm, nixery | idle |
72
+
| ci-4 | x86_64 | [gpu] | microvm | idle |
73
+
74
+
The walkthrough:
75
+
76
+
```mermaid
77
+
flowchart TD
78
+
W["workflow<br/><small>engine: microvm, image: alpine</small>"] --> F{"candidate filter"}
79
+
F -->|"all pass runs_on (empty)"| L{"has microvm<br/>available?"}
80
+
L -->|all four| R{"rank by load"}
81
+
R -->|"ci-1 busy"| X2["ci-1 ranked last"]
82
+
R --> B["bid top-K: ci-2, ci-4, ci-1"]
83
+
```
84
+
85
+
Say ci-2's alpine image is actually x86_64-only: the mill offers the bid
86
+
anyway, ci-2's reserve-time validation rejects it as incompatible, and the
87
+
bid rotates to ci-4. The mill learns "ci-2 can't run alpine" from the
88
+
rejection, not from any advertisement, and the reason reaches the user if
89
+
every candidate fails the same way.
90
+
91
+
The bid then runs concurrently:
92
+
93
+
```mermaid
94
+
sequenceDiagram
95
+
participant M as mill
96
+
participant C2 as ci-2
97
+
participant C4 as ci-4
98
+
participant C1 as ci-1
99
+
100
+
par bids
101
+
M->>C2: ReserveSeat (raw pipeline+workflow)
102
+
M->>C4: ReserveSeat
103
+
M->>C1: ReserveSeat
104
+
end
105
+
C2-->>M: accept (idle)
106
+
C4-->>M: accept
107
+
C1-->>M: reject (transient, seats full)
108
+
Note over M: ci-2 ranked above ci-4,<br/>ci-4 gets ReleaseLease
109
+
M->>C2: CommitLease (secrets)
110
+
C2-->>M: Committed
111
+
M->>C4: ReleaseLease
112
+
```
113
+
114
+
Both idle nodes accepted, so rank breaks the tie: ci-2 keeps the seat, ci-4
115
+
frees its immediately, and only ci-2 ever sees the secrets. From here ci-2
116
+
runs the job exactly like a standalone spindle would, booting the alpine
117
+
image under QEMU, while the mill blocks on the terminal event.
118
+
119
+
## The protocol's durability model
120
+
121
+
Executor to mill is a single websocket per node, and everything the executor
122
+
reports (status, logs, terminal results) travels as sequenced `Event`s. Two
123
+
identifiers keep it all consistent:
124
+
125
+
- the **epoch** names one lifetime of the executor process. a restarted
126
+
executor connects with a fresh epoch, and anything arriving for an old one
127
+
is invalid. leases are bound to node+epoch, so a zombie from a previous
128
+
process can't act on them
129
+
- the **seqno** is a dense per-epoch counter on events. the executor persists
130
+
events in a local outbox before sending and trims it only when the mill
131
+
acks. on reconnect it resumes from the mill's acked position and replays.
132
+
the mill applies idempotently: replays drop, gaps kill the session
133
+
134
+
The mill is equally restartable: leases, acked positions and the canonical
135
+
log all live in its db. Restored leases start as orphans and must be
136
+
reclaimed by the executor's first snapshot, or a sweep fails them after one
137
+
grace window. The invariant throughout: at any moment, for any lease, exactly
138
+
one of {executor outbox, mill db} holds the newest state, and the seqno/epoch
139
+
pair says who.
140
+
141
+
## Leases
142
+
143
+
A lease is the mill-side handle for one placed job:
144
+
145
+
```mermaid
146
+
stateDiagram-v2
147
+
[*] --> reserved: bid won
148
+
reserved --> committing: CommitLease sent
149
+
committing --> running: Committed
150
+
running --> done: terminal arrived
151
+
reserved --> done: released / expired
152
+
committing --> done: released / expired
153
+
```
154
+
155
+
Commit retries ride reconnects: a reservation outlives one disconnect, so a
156
+
lost session means wait and retry, not a failed job. What ends a job is the
157
+
job timeout, a terminal event, or the executor being declared dead after
158
+
reconnect grace expires, at which point every lease the node held fails
159
+
(preserving a pending cancellation as the reason).
+556
spindle/mill/auth_test.go
+556
spindle/mill/auth_test.go
···
1
+
package mill
2
+
3
+
import (
4
+
"context"
5
+
"io"
6
+
"log/slog"
7
+
"net/http"
8
+
"net/http/httptest"
9
+
"path/filepath"
10
+
"strings"
11
+
"testing"
12
+
"time"
13
+
14
+
"github.com/gorilla/websocket"
15
+
"tangled.org/core/notifier"
16
+
"tangled.org/core/spindle/db"
17
+
"tangled.org/core/spindle/models"
18
+
19
+
millproto "tangled.org/core/spindle/mill/proto"
20
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
21
+
)
22
+
23
+
func discardLogger() *slog.Logger {
24
+
return slog.New(slog.NewTextHandler(io.Discard, nil))
25
+
}
26
+
27
+
func nopEncoder() scriptedEncoder {
28
+
return scriptedEncoder(func(*millproto.Message) error { return nil })
29
+
}
30
+
31
+
func TestHashToken(t *testing.T) {
32
+
const raw = "super-secret-executor-token"
33
+
34
+
if HashToken(raw) != HashToken(raw) {
35
+
t.Fatal("HashToken is not deterministic; the same token would stop authenticating")
36
+
}
37
+
if HashToken("token-a") == HashToken("token-b") {
38
+
t.Fatal("HashToken collided two distinct tokens")
39
+
}
40
+
if HashToken(raw) == raw {
41
+
t.Fatal("HashToken returned the raw token; a hash leak would expose a usable credential")
42
+
}
43
+
}
44
+
45
+
func TestGenerateTokenDistinct(t *testing.T) {
46
+
const n = 100
47
+
seen := make(map[string]struct{}, n)
48
+
for i := range n {
49
+
tok, err := GenerateToken()
50
+
if err != nil {
51
+
t.Fatalf("GenerateToken: %v", err)
52
+
}
53
+
if tok == "" {
54
+
t.Fatalf("GenerateToken returned an empty token on call %d", i)
55
+
}
56
+
if _, dup := seen[tok]; dup {
57
+
t.Fatalf("GenerateToken repeated a token after %d calls: %q", i, tok)
58
+
}
59
+
seen[tok] = struct{}{}
60
+
}
61
+
}
62
+
63
+
func TestAttachSessionRejectsSecondLiveSession(t *testing.T) {
64
+
l := discardLogger()
65
+
m := New(l, Config{ReconnectGrace: time.Minute})
66
+
67
+
sessionOf := func(node string) *millSession {
68
+
m.mu.Lock()
69
+
defer m.mu.Unlock()
70
+
return m.sessions[node]
71
+
}
72
+
73
+
sess1 := newSession("node-1", "inc-1", nil, nopEncoder(), l)
74
+
if _, ok := m.attachSession(sess1); !ok {
75
+
t.Fatal("first attach of a node was rejected; want accept")
76
+
}
77
+
if sessionOf("node-1") != sess1 {
78
+
t.Fatal("first session was not registered as the live session")
79
+
}
80
+
81
+
sess2 := newSession("node-1", "inc-2", nil, nopEncoder(), l)
82
+
if _, ok := m.attachSession(sess2); ok {
83
+
t.Fatal("second live attach for an already-live node was accepted; a valid token hijacked the executor")
84
+
}
85
+
if sessionOf("node-1") != sess1 {
86
+
t.Fatal("rejected newcomer evicted the incumbent session")
87
+
}
88
+
89
+
m.detachSession(sess1)
90
+
sess3 := newSession("node-1", "inc-3", nil, nopEncoder(), l)
91
+
if _, ok := m.attachSession(sess3); !ok {
92
+
t.Fatal("attach during the incumbent's reconnect grace was rejected; want adopt")
93
+
}
94
+
if sessionOf("node-1") != sess3 {
95
+
t.Fatal("adopted session was not installed as the live session")
96
+
}
97
+
}
98
+
99
+
func TestOnAttemptResultIgnoresForeignLease(t *testing.T) {
100
+
ctx := context.Background()
101
+
l := discardLogger()
102
+
bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db"))
103
+
if err != nil {
104
+
t.Fatalf("db.Make: %v", err)
105
+
}
106
+
t.Cleanup(func() { bdb.Close() })
107
+
n := notifier.New()
108
+
109
+
m := New(l, Config{ReconnectGrace: time.Minute})
110
+
m.Attach(bdb, &n)
111
+
112
+
foreign := newLease("lease-foreign", "node-a", "inc-a", "dummy")
113
+
m.mu.Lock()
114
+
m.leases[foreign.id] = foreign
115
+
m.mu.Unlock()
116
+
117
+
sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l)
118
+
m.attachSession(sessB)
119
+
_ = m.onEventBatch(sessB, &millv1.EventBatch{
120
+
Epoch: sessB.epoch,
121
+
Events: []*millv1.Event{
122
+
{
123
+
Seqno: 1,
124
+
LeaseId: foreign.id,
125
+
Payload: &millv1.Event_AttemptResult{
126
+
AttemptResult: &millv1.AttemptResult{
127
+
Status: millv1.TerminalStatus_SUCCESS,
128
+
},
129
+
},
130
+
},
131
+
},
132
+
})
133
+
134
+
if _, ok := pollTerminal(foreign); ok {
135
+
t.Fatal("attempt-result on a foreign lease delivered a terminal; an executor forged another node's job result")
136
+
}
137
+
if foreign.getState() == leaseDone {
138
+
t.Fatal("attempt-result on a foreign lease sealed the lease")
139
+
}
140
+
}
141
+
142
+
func TestOnAttemptResultIgnoresAbsentLease(t *testing.T) {
143
+
ctx := context.Background()
144
+
l := discardLogger()
145
+
bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db"))
146
+
if err != nil {
147
+
t.Fatalf("db.Make: %v", err)
148
+
}
149
+
t.Cleanup(func() { bdb.Close() })
150
+
n := notifier.New()
151
+
152
+
m := New(l, Config{ReconnectGrace: time.Minute})
153
+
m.Attach(bdb, &n)
154
+
155
+
// bystander lease the reporting node owns, proving an absent-lease stream
156
+
// does not spill onto another lease
157
+
bystander := newLease("lease-bystander", "node-b", "inc-b", "dummy")
158
+
m.mu.Lock()
159
+
m.leases[bystander.id] = bystander
160
+
m.mu.Unlock()
161
+
162
+
sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l)
163
+
m.attachSession(sessB)
164
+
165
+
_ = m.onEventBatch(sessB, &millv1.EventBatch{
166
+
Epoch: sessB.epoch,
167
+
Events: []*millv1.Event{
168
+
{
169
+
Seqno: 1,
170
+
LeaseId: "lease-nonexistent",
171
+
Payload: &millv1.Event_AttemptResult{
172
+
AttemptResult: &millv1.AttemptResult{
173
+
Status: millv1.TerminalStatus_SUCCESS,
174
+
},
175
+
},
176
+
},
177
+
},
178
+
})
179
+
180
+
if _, ok := pollTerminal(bystander); ok {
181
+
t.Fatal("attempt-result for an absent lease delivered a terminal to a bystander lease")
182
+
}
183
+
if bystander.getState() == leaseDone {
184
+
t.Fatal("attempt-result for an absent lease sealed a bystander lease")
185
+
}
186
+
}
187
+
188
+
// owned-lease path proves ignore tests above are not passing merely because
189
+
// delivery is broken. correctly owned terminal is delivered
190
+
func TestOnAttemptResultDeliversOwnedLease(t *testing.T) {
191
+
ctx := context.Background()
192
+
l := discardLogger()
193
+
bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db"))
194
+
if err != nil {
195
+
t.Fatalf("db.Make: %v", err)
196
+
}
197
+
t.Cleanup(func() { bdb.Close() })
198
+
n := notifier.New()
199
+
200
+
m := New(l, Config{ReconnectGrace: time.Minute})
201
+
m.Attach(bdb, &n)
202
+
203
+
owned := newLease("lease-owned", "node-b", "inc-b", "dummy")
204
+
m.mu.Lock()
205
+
m.leases[owned.id] = owned
206
+
m.mu.Unlock()
207
+
208
+
sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l)
209
+
m.attachSession(sessB)
210
+
_ = m.onEventBatch(sessB, &millv1.EventBatch{
211
+
Epoch: sessB.epoch,
212
+
Events: []*millv1.Event{
213
+
{
214
+
Seqno: 1,
215
+
LeaseId: owned.id,
216
+
Payload: &millv1.Event_AttemptResult{
217
+
AttemptResult: &millv1.AttemptResult{
218
+
Status: millv1.TerminalStatus_SUCCESS,
219
+
},
220
+
},
221
+
},
222
+
},
223
+
})
224
+
225
+
res, ok := pollTerminal(owned)
226
+
if !ok {
227
+
t.Fatal("attempt-result on an owned lease was not delivered")
228
+
}
229
+
if got := res.GetStatus(); got != millv1.TerminalStatus_SUCCESS {
230
+
t.Fatalf("delivered terminal status = %v, want %v", got, millv1.TerminalStatus_SUCCESS)
231
+
}
232
+
if owned.getState() != leaseDone {
233
+
t.Fatal("owned lease was not sealed after its terminal was delivered")
234
+
}
235
+
}
236
+
237
+
func TestOnLogEventOwnership(t *testing.T) {
238
+
ctx := context.Background()
239
+
l := discardLogger()
240
+
dir := t.TempDir()
241
+
bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db"))
242
+
if err != nil {
243
+
t.Fatalf("db.Make: %v", err)
244
+
}
245
+
t.Cleanup(func() { bdb.Close() })
246
+
n := notifier.New()
247
+
248
+
m := New(l, Config{LogDir: dir, ReconnectGrace: time.Minute})
249
+
m.Attach(bdb, &n)
250
+
251
+
foreign := newLease("lease-foreign", "node-x", "inc-x", "dummy")
252
+
foreign.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "foreign"}, Name: "build"}
253
+
owned := newLease("lease-owned", "node-z", "inc-z", "dummy")
254
+
owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "owned"}, Name: "build"}
255
+
m.mu.Lock()
256
+
m.leases[foreign.id] = foreign
257
+
m.leases[owned.id] = owned
258
+
m.mu.Unlock()
259
+
260
+
sessY := newSession("node-y", "inc-y", nil, nopEncoder(), l)
261
+
m.attachSession(sessY)
262
+
sessZ := newSession("node-z", "inc-z", nil, nopEncoder(), l)
263
+
m.attachSession(sessZ)
264
+
265
+
_ = m.onEventBatch(sessY, &millv1.EventBatch{
266
+
Epoch: sessY.epoch,
267
+
Events: []*millv1.Event{
268
+
{
269
+
Seqno: 1,
270
+
LeaseId: foreign.id,
271
+
Payload: &millv1.Event_LogLine{
272
+
LogLine: &millv1.LogLine{RawJson: []byte(`{"line":"forged"}`)},
273
+
},
274
+
},
275
+
},
276
+
})
277
+
// a forged foreign-lease line must not reach the table at all
278
+
foreignLogs, err := bdb.ListCanonicalLogsFor(models.LogFilePath(dir, foreign.wid))
279
+
if err != nil {
280
+
t.Fatal(err)
281
+
}
282
+
if len(foreignLogs) != 0 {
283
+
t.Fatalf("log stream for a foreign lease wrote canonical rows; an executor forged another node's logs")
284
+
}
285
+
286
+
line := []byte(`{"line":"legit"}`)
287
+
_ = m.onEventBatch(sessZ, &millv1.EventBatch{
288
+
Epoch: sessZ.epoch,
289
+
Events: []*millv1.Event{
290
+
{
291
+
Seqno: 1,
292
+
LeaseId: owned.id,
293
+
Payload: &millv1.Event_LogLine{
294
+
LogLine: &millv1.LogLine{RawJson: line},
295
+
},
296
+
},
297
+
},
298
+
})
299
+
ownedLogs, err := bdb.ListCanonicalLogsFor(models.LogFilePath(dir, owned.wid))
300
+
if err != nil {
301
+
t.Fatalf("owned log stream did not reach the canonical table: %v", err)
302
+
}
303
+
if len(ownedLogs) != 1 {
304
+
t.Fatalf("canonical rows = %d, want 1", len(ownedLogs))
305
+
}
306
+
if want := string(line) + "\n"; string(ownedLogs[0].Payload) != want {
307
+
t.Fatalf("owned log line = %q, want %q", string(ownedLogs[0].Payload), want)
308
+
}
309
+
}
310
+
311
+
func TestOnStatusEventOwnership(t *testing.T) {
312
+
ctx := context.Background()
313
+
l := discardLogger()
314
+
315
+
bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db"))
316
+
if err != nil {
317
+
t.Fatalf("db.Make: %v", err)
318
+
}
319
+
t.Cleanup(func() { bdb.Close() })
320
+
n := notifier.New()
321
+
322
+
m := New(l, Config{ReconnectGrace: time.Minute})
323
+
m.Attach(bdb, &n)
324
+
325
+
foreign := newLease("lease-foreign", "node-x", "inc-x", "dummy")
326
+
foreign.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "foreign"}, Name: "build"}
327
+
owned := newLease("lease-owned", "node-z", "inc-z", "dummy")
328
+
owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "owned"}, Name: "build"}
329
+
m.mu.Lock()
330
+
m.leases[foreign.id] = foreign
331
+
m.leases[owned.id] = owned
332
+
m.mu.Unlock()
333
+
334
+
sessY := newSession("node-y", "inc-y", nil, nopEncoder(), l)
335
+
m.attachSession(sessY)
336
+
sessZ := newSession("node-z", "inc-z", nil, nopEncoder(), l)
337
+
m.attachSession(sessZ)
338
+
339
+
_ = m.onEventBatch(sessY, &millv1.EventBatch{
340
+
Epoch: sessY.epoch,
341
+
Events: []*millv1.Event{
342
+
{
343
+
Seqno: 1,
344
+
LeaseId: foreign.id,
345
+
Payload: &millv1.Event_StatusEvent{
346
+
StatusEvent: &millv1.StatusEvent{
347
+
Status: millv1.NonterminalStatus_RUNNING,
348
+
},
349
+
},
350
+
},
351
+
},
352
+
})
353
+
if _, err := bdb.GetStatus(foreign.wid); err == nil {
354
+
t.Fatal("status stream for a foreign lease authored a status row; an executor forged another pipeline's status")
355
+
}
356
+
357
+
_ = m.onEventBatch(sessZ, &millv1.EventBatch{
358
+
Epoch: sessZ.epoch,
359
+
Events: []*millv1.Event{
360
+
{
361
+
Seqno: 1,
362
+
LeaseId: owned.id,
363
+
Payload: &millv1.Event_StatusEvent{
364
+
StatusEvent: &millv1.StatusEvent{
365
+
Status: millv1.NonterminalStatus_RUNNING,
366
+
},
367
+
},
368
+
},
369
+
},
370
+
})
371
+
st, err := bdb.GetStatus(owned.wid)
372
+
if err != nil {
373
+
t.Fatalf("owned status stream did not author a status row: %v", err)
374
+
}
375
+
if st.Status != "running" {
376
+
t.Fatalf("owned status = %q, want %q", st.Status, "running")
377
+
}
378
+
}
379
+
380
+
func setupTestServer(t *testing.T, authorizedLabels []string) (*Mill, *db.DB, *httptest.Server, string) {
381
+
ctx := context.Background()
382
+
bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db"))
383
+
if err != nil {
384
+
t.Fatalf("db.Make: %v", err)
385
+
}
386
+
t.Cleanup(func() { bdb.Close() })
387
+
n := notifier.New()
388
+
389
+
m := New(discardLogger(), Config{
390
+
ReconnectGrace: time.Minute,
391
+
})
392
+
m.Attach(bdb, &n)
393
+
394
+
const secret = "test-secret"
395
+
if err := bdb.AddExecutorToken("dev-node", HashToken(secret), nil, authorizedLabels); err != nil {
396
+
t.Fatalf("AddExecutorToken: %v", err)
397
+
}
398
+
399
+
server := httptest.NewServer(http.HandlerFunc(m.HandleExecutorConn))
400
+
t.Cleanup(server.Close)
401
+
402
+
return m, bdb, server, secret
403
+
}
404
+
405
+
func TestAuthLabelEscalation(t *testing.T) {
406
+
_, _, server, secret := setupTestServer(t, []string{"linux", "amd64"})
407
+
408
+
wsUrl := "ws" + strings.TrimPrefix(server.URL, "http")
409
+
410
+
{
411
+
header := http.Header{}
412
+
header.Set("Authorization", "Bearer bad-token")
413
+
_, resp, err := websocket.DefaultDialer.Dial(wsUrl, header)
414
+
if err == nil {
415
+
t.Fatal("expected connection with invalid token to fail")
416
+
}
417
+
if resp != nil && resp.StatusCode != http.StatusUnauthorized {
418
+
t.Fatalf("expected 401 Unauthorized, got %d", resp.StatusCode)
419
+
}
420
+
}
421
+
422
+
{
423
+
header := http.Header{}
424
+
header.Set("Authorization", "Bearer "+secret)
425
+
conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header)
426
+
if err != nil {
427
+
t.Fatalf("dial failed: %v", err)
428
+
}
429
+
defer conn.Close()
430
+
431
+
stream := millproto.NewWSStream(conn)
432
+
enc := millproto.NewEncoder(stream)
433
+
dec := millproto.NewDecoder(stream)
434
+
435
+
hello := &millproto.Message{Hello: &millv1.Hello{
436
+
ProtocolVersion: millproto.ProtocolVersion,
437
+
Arch: "amd64",
438
+
Labels: []string{"linux", "gpu"},
439
+
Epoch: "inc-1",
440
+
}}
441
+
if err := enc.Encode(hello); err != nil {
442
+
t.Fatalf("encode hello: %v", err)
443
+
}
444
+
445
+
_, err = dec.Decode()
446
+
if err == nil {
447
+
t.Fatal("expected server to close connection for unauthorized label, but got a message")
448
+
}
449
+
}
450
+
451
+
{
452
+
header := http.Header{}
453
+
header.Set("Authorization", "Bearer "+secret)
454
+
conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header)
455
+
if err != nil {
456
+
t.Fatalf("dial failed: %v", err)
457
+
}
458
+
defer conn.Close()
459
+
460
+
stream := millproto.NewWSStream(conn)
461
+
enc := millproto.NewEncoder(stream)
462
+
dec := millproto.NewDecoder(stream)
463
+
464
+
hello := &millproto.Message{Hello: &millv1.Hello{
465
+
ProtocolVersion: millproto.ProtocolVersion,
466
+
Arch: "amd64",
467
+
Labels: []string{"linux"},
468
+
Epoch: "inc-1",
469
+
}}
470
+
if err := enc.Encode(hello); err != nil {
471
+
t.Fatalf("encode hello: %v", err)
472
+
}
473
+
474
+
msg, err := dec.Decode()
475
+
if err != nil {
476
+
t.Fatalf("expected resume message, got error: %v", err)
477
+
}
478
+
res := msg.GetResume()
479
+
if res == nil {
480
+
t.Fatal("expected Resume message, got nil")
481
+
}
482
+
if res.GetEpoch() != "inc-1" {
483
+
t.Fatalf("expected epoch inc-1, got %q", res.GetEpoch())
484
+
}
485
+
}
486
+
}
487
+
488
+
func TestHandshakeTimeoutAndConcurrency(t *testing.T) {
489
+
_, _, server, secret := setupTestServer(t, []string{"linux"})
490
+
wsUrl := "ws" + strings.TrimPrefix(server.URL, "http")
491
+
492
+
// executor that never sends Hello is dropped after the 5s pre-hello deadline
493
+
{
494
+
header := http.Header{}
495
+
header.Set("Authorization", "Bearer "+secret)
496
+
conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header)
497
+
if err != nil {
498
+
t.Fatalf("dial failed: %v", err)
499
+
}
500
+
defer conn.Close()
501
+
502
+
time.Sleep(6 * time.Second)
503
+
504
+
stream := millproto.NewWSStream(conn)
505
+
enc := millproto.NewEncoder(stream)
506
+
hello := &millproto.Message{Hello: &millv1.Hello{
507
+
ProtocolVersion: millproto.ProtocolVersion,
508
+
Arch: "amd64",
509
+
Labels: []string{"linux"},
510
+
Epoch: "inc-1",
511
+
}}
512
+
err = enc.Encode(hello)
513
+
dec := millproto.NewDecoder(stream)
514
+
_, readErr := dec.Decode()
515
+
if readErr == nil {
516
+
t.Fatal("expected server to have closed connection due to handshake timeout")
517
+
}
518
+
}
519
+
520
+
// second live session for one identity is rejected with 409 before the ws upgrade
521
+
{
522
+
header := http.Header{}
523
+
header.Set("Authorization", "Bearer "+secret)
524
+
525
+
conn1, _, err := websocket.DefaultDialer.Dial(wsUrl, header)
526
+
if err != nil {
527
+
t.Fatalf("dial 1 failed: %v", err)
528
+
}
529
+
defer conn1.Close()
530
+
531
+
stream1 := millproto.NewWSStream(conn1)
532
+
enc1 := millproto.NewEncoder(stream1)
533
+
dec1 := millproto.NewDecoder(stream1)
534
+
hello1 := &millproto.Message{Hello: &millv1.Hello{
535
+
ProtocolVersion: millproto.ProtocolVersion,
536
+
Arch: "amd64",
537
+
Labels: []string{"linux"},
538
+
Epoch: "inc-1",
539
+
}}
540
+
if err := enc1.Encode(hello1); err != nil {
541
+
t.Fatalf("encode hello 1: %v", err)
542
+
}
543
+
_, err = dec1.Decode()
544
+
if err != nil {
545
+
t.Fatalf("first connection handshake failed: %v", err)
546
+
}
547
+
548
+
_, resp, err := websocket.DefaultDialer.Dial(wsUrl, header)
549
+
if err == nil {
550
+
t.Fatal("expected second connection for same live identity to be rejected")
551
+
}
552
+
if resp != nil && resp.StatusCode != http.StatusConflict {
553
+
t.Fatalf("expected 409 Conflict for duplicate session, got %d", resp.StatusCode)
554
+
}
555
+
}
556
+
}
+23
spindle/mill/capability_test.go
+23
spindle/mill/capability_test.go
···
1
+
package mill
2
+
3
+
import (
4
+
"testing"
5
+
6
+
"tangled.org/core/spindle/models"
7
+
)
8
+
9
+
func TestRankCandidatesIntersectsLabels(t *testing.T) {
10
+
m := New(discardLogger(), Config{})
11
+
addCandidateSession(t, m, "wrong-region", []string{"region/us"}, 0.10, nil)
12
+
addCandidateSession(t, m, "no-gpu", []string{"region/eu"}, 0.05, nil)
13
+
addCandidateSession(t, m, "eligible", []string{"region/eu", "gpu"}, 0.70, nil)
14
+
15
+
got := m.rankCandidates("dummy", []string{"region/eu", "gpu"})
16
+
assertRankedNodes(t, got, []string{"eligible"})
17
+
}
18
+
19
+
func TestRequiredLabelsWithoutMillStateAreEmpty(t *testing.T) {
20
+
if got := requiredLabels(&models.Workflow{}); got != nil {
21
+
t.Fatalf("requiredLabels() = %v, want nil", got)
22
+
}
23
+
}
+87
spindle/mill/engine.go
+87
spindle/mill/engine.go
···
1
+
package mill
2
+
3
+
import (
4
+
"context"
5
+
"log/slog"
6
+
"time"
7
+
8
+
"tangled.org/core/api/tangled"
9
+
"tangled.org/core/spindle/engine"
10
+
"tangled.org/core/spindle/models"
11
+
"tangled.org/core/spindle/secrets"
12
+
)
13
+
14
+
// raw pipeline/workflow carried forward, executor runs the real InitWorkflow
15
+
type millWorkflowState struct {
16
+
TargetEngine string
17
+
RawWorkflow tangled.Pipeline_Workflow
18
+
RawPipeline tangled.Pipeline
19
+
Lease *RemoteLease
20
+
}
21
+
22
+
// stand-in for a real engine, registered under the real names
23
+
// ("microvm", "nixery"), all sharing one Mill
24
+
type Engine struct {
25
+
name string
26
+
mill *Mill
27
+
l *slog.Logger
28
+
}
29
+
30
+
func (e *Engine) AuthorsRemoteStatus() {}
31
+
32
+
func NewEngine(name string, mill *Mill) *Engine {
33
+
return &Engine{name: name, mill: mill, l: mill.l.With("engine", "mill:"+name)}
34
+
}
35
+
36
+
// synthetic one-step workflow so processPipeline injects TANGLED_* env
37
+
// and marks pending normally. the real InitWorkflow runs exactly once, on
38
+
// the executor inside ReserveSeat, and commit reuses that workflow
39
+
func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) {
40
+
return &models.Workflow{
41
+
Name: twf.Name,
42
+
Environment: map[string]string{},
43
+
Steps: []models.Step{remoteStep{}},
44
+
Data: &millWorkflowState{
45
+
TargetEngine: e.name,
46
+
RawWorkflow: twf,
47
+
RawPipeline: tpl,
48
+
},
49
+
}, nil
50
+
}
51
+
52
+
// no-op logger for the synthetic workflow. the executor's lines stream
53
+
// into this wid's log directly, a local logger would just write competing
54
+
// lines
55
+
func (e *Engine) WorkflowLogger(wid models.WorkflowId) models.WorkflowLogger {
56
+
return models.NullLogger{}
57
+
}
58
+
59
+
// the placement seam, blocks on remote placement which the user sees as
60
+
// "pending". only StartWorkflows calls this, always Wait
61
+
func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) {
62
+
return e.mill.place(ctx, e.name, wid, wf)
63
+
}
64
+
65
+
// no-op. real setup happens on the executor
66
+
func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error {
67
+
e.l.Info("remote job placed, awaiting commit", "wid", wid)
68
+
return nil
69
+
}
70
+
71
+
// hands over the secrets and blocks on the terminal result streamed over the
72
+
// session
73
+
func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, unlocked []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error {
74
+
return e.mill.commitAndWait(ctx, w, unlocked)
75
+
}
76
+
77
+
// deliberately generous, the executor enforces the real timeout. the mill
78
+
// only caps a hung or silent executor, true death is caught by reconnect grace
79
+
func (e *Engine) WorkflowTimeout() time.Duration {
80
+
return e.mill.cfg.JobTimeout
81
+
}
82
+
83
+
// cancels a still-running attempt. no-op if already terminal
84
+
func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error {
85
+
e.mill.destroy(wid)
86
+
return nil
87
+
}
+119
spindle/mill/executor/capability_test.go
+119
spindle/mill/executor/capability_test.go
···
1
+
package executor
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"errors"
7
+
"io"
8
+
"log/slog"
9
+
"testing"
10
+
11
+
"tangled.org/core/api/tangled"
12
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
13
+
"tangled.org/core/spindle/models"
14
+
)
15
+
16
+
type placementEngine struct {
17
+
*fakeEngine
18
+
validationErr error
19
+
validated bool
20
+
}
21
+
22
+
func (e *placementEngine) ValidateWorkflowPlacement(*models.Workflow) error {
23
+
e.validated = true
24
+
return e.validationErr
25
+
}
26
+
27
+
func TestHandleReserveRejectsMissingTriggerMetadata(t *testing.T) {
28
+
enc := newCaptureEncoder()
29
+
e := &Executor{
30
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
31
+
enc: enc,
32
+
seats: 1,
33
+
engines: map[string]models.Engine{"microvm": &fakeEngine{}},
34
+
active: make(map[string]*reservation),
35
+
}
36
+
twf, err := json.Marshal(tangled.Pipeline_Workflow{Name: "build"})
37
+
if err != nil {
38
+
t.Fatal(err)
39
+
}
40
+
tpl, err := json.Marshal(tangled.Pipeline{})
41
+
if err != nil {
42
+
t.Fatal(err)
43
+
}
44
+
45
+
// engines deref TriggerMetadata unconditionally. this must be a reject,
46
+
// not a panic that takes the whole executor down
47
+
e.handleReserve(context.Background(), &millv1.ReserveSeat{
48
+
LeaseId: "lease-1",
49
+
TargetEngine: "microvm",
50
+
RawWorkflowJson: string(twf),
51
+
RawPipelineJson: string(tpl),
52
+
Knot: "k",
53
+
Rkey: "r",
54
+
})
55
+
56
+
result := (<-enc.messages).GetReserveResult()
57
+
if result == nil {
58
+
t.Fatal("handleReserve() did not send ReserveResult")
59
+
}
60
+
if result.GetAccepted() {
61
+
t.Fatal("handleReserve() accepted a pipeline without trigger metadata")
62
+
}
63
+
if result.GetRejectClass() != millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE {
64
+
t.Fatalf("reject class = %v, want incompatible", result.GetRejectClass())
65
+
}
66
+
if len(e.active) != 0 {
67
+
t.Fatalf("active reservations = %d, want 0", len(e.active))
68
+
}
69
+
}
70
+
71
+
func TestHandleReserveValidatesPlacementBeforeAcquiringSlot(t *testing.T) {
72
+
enc := newCaptureEncoder()
73
+
validationErr := errors.New("image architecture is not native")
74
+
eng := &placementEngine{fakeEngine: &fakeEngine{}, validationErr: validationErr}
75
+
e := &Executor{
76
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
77
+
enc: enc,
78
+
seats: 1,
79
+
engines: map[string]models.Engine{"microvm": eng},
80
+
active: make(map[string]*reservation),
81
+
}
82
+
twf, err := json.Marshal(tangled.Pipeline_Workflow{Name: "build"})
83
+
if err != nil {
84
+
t.Fatal(err)
85
+
}
86
+
tpl, err := json.Marshal(tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}})
87
+
if err != nil {
88
+
t.Fatal(err)
89
+
}
90
+
91
+
e.handleReserve(context.Background(), &millv1.ReserveSeat{
92
+
LeaseId: "lease-1",
93
+
TargetEngine: "microvm",
94
+
RawWorkflowJson: string(twf),
95
+
RawPipelineJson: string(tpl),
96
+
Knot: "k",
97
+
Rkey: "r",
98
+
})
99
+
100
+
result := (<-enc.messages).GetReserveResult()
101
+
if result == nil {
102
+
t.Fatal("handleReserve() did not send ReserveResult")
103
+
}
104
+
if result.GetAccepted() {
105
+
t.Fatal("handleReserve() accepted placement validation failure")
106
+
}
107
+
if result.GetRejectClass() != millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE {
108
+
t.Fatalf("reject class = %v, want incompatible", result.GetRejectClass())
109
+
}
110
+
if !eng.validated {
111
+
t.Fatal("placement validator was not called")
112
+
}
113
+
if eng.acquireCalled {
114
+
t.Fatal("slot acquisition ran after placement validation failed")
115
+
}
116
+
if len(e.active) != 0 {
117
+
t.Fatalf("active reservations = %d, want 0", len(e.active))
118
+
}
119
+
}
+631
spindle/mill/executor/executor.go
+631
spindle/mill/executor/executor.go
···
1
+
package executor
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"errors"
7
+
"fmt"
8
+
"log/slog"
9
+
"maps"
10
+
"net/http"
11
+
"runtime"
12
+
"strings"
13
+
"sync"
14
+
"time"
15
+
16
+
"github.com/bluesky-social/indigo/atproto/syntax"
17
+
18
+
"tangled.org/core/api/tangled"
19
+
"tangled.org/core/notifier"
20
+
"tangled.org/core/spindle/config"
21
+
"tangled.org/core/spindle/db"
22
+
"tangled.org/core/spindle/engine"
23
+
"tangled.org/core/spindle/models"
24
+
"tangled.org/core/util/netutil"
25
+
26
+
millproto "tangled.org/core/spindle/mill/proto"
27
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
28
+
)
29
+
30
+
const (
31
+
dialBackoffMin = 1 * time.Second
32
+
dialBackoffMax = 30 * time.Second
33
+
snapshotEvery = 15 * time.Second
34
+
defaultSeats = 4
35
+
)
36
+
37
+
type Executor struct {
38
+
millURL string
39
+
token string
40
+
nodeID string
41
+
seats int
42
+
labels []string
43
+
44
+
engines map[string]models.Engine
45
+
db *db.DB
46
+
n *notifier.Notifier
47
+
cfg *config.Config
48
+
l *slog.Logger
49
+
50
+
epoch string
51
+
outboxLogBytes int64
52
+
outboxControlBytes int64
53
+
droppedLogs uint64
54
+
maxLogBytes int64
55
+
maxControlBytes int64
56
+
57
+
eventMu sync.Mutex // protects outbox operations
58
+
sendMu sync.Mutex // serializes all writes to enc and sentSeqno
59
+
flushMu sync.Mutex
60
+
61
+
sentSeqno uint64
62
+
flushTimer *time.Timer
63
+
64
+
connMu sync.Mutex
65
+
enc messageEncoder
66
+
sessionCancel context.CancelFunc
67
+
68
+
mu sync.Mutex
69
+
active map[string]*reservation
70
+
draining bool
71
+
72
+
snapshotMu sync.Mutex
73
+
nextSeqno uint64
74
+
75
+
lifecycleCtx context.Context
76
+
jobsWG sync.WaitGroup
77
+
}
78
+
79
+
type reservation struct {
80
+
leaseID string
81
+
wid models.WorkflowId
82
+
realEngine models.Engine
83
+
slot engine.WorkflowSlot
84
+
wf *models.Workflow
85
+
repoDid syntax.DID
86
+
87
+
committed bool
88
+
cancelled bool
89
+
cancel context.CancelFunc
90
+
ttlTimer *time.Timer
91
+
stopTail func()
92
+
}
93
+
94
+
type messageEncoder interface {
95
+
Encode(*millproto.Message) error
96
+
}
97
+
98
+
func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *notifier.Notifier, l *slog.Logger) (*Executor, error) {
99
+
seats := cfg.Mill.Seats
100
+
if seats <= 0 {
101
+
seats = defaultSeats
102
+
}
103
+
labels := normalizeLabels(cfg.Mill.Labels)
104
+
if d == nil || n == nil {
105
+
return nil, fmt.Errorf("executor requires a database and notifier")
106
+
}
107
+
e := &Executor{
108
+
millURL: cfg.Mill.URL,
109
+
token: cfg.Mill.SharedSecret,
110
+
nodeID: cfg.Server.Hostname,
111
+
seats: seats,
112
+
labels: labels,
113
+
engines: engines,
114
+
db: d,
115
+
n: n,
116
+
cfg: cfg,
117
+
l: l.With("component", "mill.executor"),
118
+
active: make(map[string]*reservation),
119
+
maxLogBytes: 100 * 1024 * 1024, // 100 MiB generous default log cap
120
+
maxControlBytes: 10 * 1024 * 1024, // 10 MiB default control cap
121
+
}
122
+
if err := e.initOutbox(); err != nil {
123
+
return nil, fmt.Errorf("initialize executor outbox: %w", err)
124
+
}
125
+
return e, nil
126
+
}
127
+
128
+
func (e *Executor) Connect(ctx context.Context) {
129
+
e.lifecycleCtx = ctx
130
+
sub := e.n.Subscribe()
131
+
cursor, err := e.db.EventHighWater()
132
+
if err != nil {
133
+
e.n.Unsubscribe(sub)
134
+
e.l.Error("establish event cursor failed", "err", err)
135
+
return
136
+
}
137
+
e.drainEvents(&cursor)
138
+
observerCtx, stopObserver := context.WithCancel(ctx)
139
+
observerDone := make(chan struct{})
140
+
go func() {
141
+
defer close(observerDone)
142
+
e.observeLoop(observerCtx, sub, cursor)
143
+
}()
144
+
defer e.n.Unsubscribe(sub)
145
+
146
+
backoff := dialBackoffMin
147
+
for {
148
+
if ctx.Err() != nil {
149
+
break
150
+
}
151
+
err := e.runSession(ctx)
152
+
if ctx.Err() != nil {
153
+
break
154
+
}
155
+
e.l.Warn("mill session ended; reconnecting", "err", err, "backoff", backoff)
156
+
select {
157
+
case <-ctx.Done():
158
+
break
159
+
case <-time.After(backoff):
160
+
}
161
+
backoff = min(backoff*2, dialBackoffMax)
162
+
}
163
+
164
+
e.jobsWG.Wait()
165
+
stopObserver()
166
+
<-observerDone
167
+
e.drainEvents(&cursor)
168
+
}
169
+
170
+
func (e *Executor) runSession(ctx context.Context) error {
171
+
dev := e.cfg.Server.Dev
172
+
if _, err := netutil.EnforceWSSURL(e.millURL, dev); err != nil {
173
+
return fmt.Errorf("mill url: %w", err)
174
+
}
175
+
header := http.Header{}
176
+
if e.token != "" {
177
+
header.Set("Authorization", "Bearer "+e.token)
178
+
}
179
+
conn, _, err := netutil.SSRFWebsocketDialer(dev).DialContext(ctx, e.millURL, header)
180
+
if err != nil {
181
+
return fmt.Errorf("dial mill: %w", err)
182
+
}
183
+
defer conn.Close()
184
+
185
+
sessionCtx, cancelSession := context.WithCancel(ctx)
186
+
defer cancelSession()
187
+
stopClose := context.AfterFunc(sessionCtx, func() { _ = conn.Close() })
188
+
defer stopClose()
189
+
190
+
stream := millproto.NewWSStream(conn)
191
+
enc := millproto.NewEncoder(stream)
192
+
dec := millproto.NewDecoder(stream)
193
+
194
+
hello := &millproto.Message{Hello: &millv1.Hello{
195
+
ProtocolVersion: millproto.ProtocolVersion,
196
+
Arch: runtime.GOARCH,
197
+
Labels: e.labels,
198
+
Epoch: e.epoch,
199
+
}}
200
+
if err := enc.Encode(hello); err != nil {
201
+
return fmt.Errorf("send hello: %w", err)
202
+
}
203
+
204
+
resumeMsg, err := dec.Decode()
205
+
if err != nil {
206
+
return fmt.Errorf("read resume: %w", err)
207
+
}
208
+
resume := resumeMsg.GetResume()
209
+
if resume == nil {
210
+
return fmt.Errorf("expected resume, got something else")
211
+
}
212
+
213
+
if resume.GetEpoch() != e.epoch {
214
+
return fmt.Errorf("resume epoch mismatch: got %q, want %q", resume.GetEpoch(), e.epoch)
215
+
}
216
+
217
+
e.connMu.Lock()
218
+
e.sessionCancel = cancelSession
219
+
e.enc = enc
220
+
e.connMu.Unlock()
221
+
defer func() {
222
+
e.connMu.Lock()
223
+
e.enc = nil
224
+
e.sessionCancel = nil
225
+
e.connMu.Unlock()
226
+
}()
227
+
228
+
readErr := make(chan error, 1)
229
+
go func() {
230
+
for {
231
+
msg, err := dec.Decode()
232
+
if err != nil {
233
+
readErr <- fmt.Errorf("read: %w", err)
234
+
return
235
+
}
236
+
e.dispatch(sessionCtx, msg)
237
+
}
238
+
}()
239
+
240
+
if err := e.replay(resume.GetAckSeqno()); err != nil {
241
+
cancelSession()
242
+
<-readErr
243
+
return fmt.Errorf("replay failed: %w", err)
244
+
}
245
+
e.pushSnapshot()
246
+
e.l.Info("connected to mill", "node", e.nodeID, "resumeFrom", resume.GetAckSeqno())
247
+
248
+
go e.snapshotLoop(sessionCtx, enc)
249
+
return <-readErr
250
+
}
251
+
252
+
func (e *Executor) send(msg *millproto.Message) {
253
+
e.connMu.Lock()
254
+
enc := e.enc
255
+
cancel := e.sessionCancel
256
+
e.connMu.Unlock()
257
+
if enc != nil {
258
+
e.sendMu.Lock()
259
+
err := enc.Encode(msg)
260
+
e.sendMu.Unlock()
261
+
if err != nil {
262
+
e.l.Error("send failed, ending session", "err", err)
263
+
if cancel != nil {
264
+
cancel()
265
+
}
266
+
}
267
+
}
268
+
}
269
+
270
+
func (e *Executor) dispatch(ctx context.Context, msg *millproto.Message) {
271
+
switch {
272
+
case msg.GetReserveSeat() != nil:
273
+
e.handleReserve(ctx, msg.GetReserveSeat())
274
+
case msg.GetCommitLease() != nil:
275
+
e.handleCommit(ctx, msg.GetCommitLease())
276
+
case msg.GetReleaseLease() != nil:
277
+
e.handleRelease(msg.GetReleaseLease().GetLeaseId())
278
+
case msg.GetCancelAttempt() != nil:
279
+
e.handleCancel(msg.GetCancelAttempt().GetLeaseId())
280
+
case msg.GetAck() != nil:
281
+
e.handleAck(msg.GetAck())
282
+
default:
283
+
e.l.Warn("executor received unexpected message")
284
+
}
285
+
}
286
+
287
+
func (e *Executor) sendReject(leaseID string, reason string, class millv1.RejectClass) {
288
+
e.send(&millproto.Message{ReserveResult: &millv1.ReserveResult{
289
+
LeaseId: leaseID,
290
+
Accepted: false,
291
+
RejectReason: reason,
292
+
RejectClass: class,
293
+
}})
294
+
}
295
+
296
+
func (e *Executor) sendCommitted(leaseID string) {
297
+
e.send(&millproto.Message{Committed: &millv1.Committed{LeaseId: leaseID}})
298
+
}
299
+
300
+
func (e *Executor) sendCancelAck(leaseID string) {
301
+
e.send(&millproto.Message{CancelAck: &millv1.CancelAck{LeaseId: leaseID}})
302
+
}
303
+
304
+
func (e *Executor) releaseReservation(cleanup func()) {
305
+
cleanup()
306
+
e.pushSnapshot()
307
+
}
308
+
func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) {
309
+
reject := func(reason string, class millv1.RejectClass) {
310
+
e.sendReject(rs.GetLeaseId(), reason, class)
311
+
}
312
+
313
+
e.mu.Lock()
314
+
draining := e.draining
315
+
e.mu.Unlock()
316
+
if draining {
317
+
reject("draining", millv1.RejectClass_REJECT_CLASS_TRANSIENT)
318
+
return
319
+
}
320
+
321
+
realEngine, ok := e.engines[rs.GetTargetEngine()]
322
+
if !ok {
323
+
reject("unknown engine "+rs.GetTargetEngine(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE)
324
+
return
325
+
}
326
+
slotter, ok := realEngine.(engine.WorkflowSlotter)
327
+
if !ok {
328
+
reject("engine does not support workflow slots", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE)
329
+
return
330
+
}
331
+
332
+
var twf tangled.Pipeline_Workflow
333
+
if err := json.Unmarshal([]byte(rs.GetRawWorkflowJson()), &twf); err != nil {
334
+
reject("bad workflow json", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE)
335
+
return
336
+
}
337
+
var tpl tangled.Pipeline
338
+
if err := json.Unmarshal([]byte(rs.GetRawPipelineJson()), &tpl); err != nil {
339
+
reject("bad pipeline json", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE)
340
+
return
341
+
}
342
+
// engines deref TriggerMetadata blindly. a pipeline without it gets a
343
+
// reject, not a crashed executor
344
+
if tpl.TriggerMetadata == nil {
345
+
reject("pipeline missing trigger metadata", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE)
346
+
return
347
+
}
348
+
349
+
pipelineId := models.PipelineId{Knot: rs.GetKnot(), Rkey: rs.GetRkey()}
350
+
wid := models.WorkflowId{PipelineId: pipelineId, Name: twf.Name}
351
+
352
+
wf, err := realEngine.InitWorkflow(twf, tpl)
353
+
if err != nil {
354
+
reject("init workflow: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE)
355
+
return
356
+
}
357
+
if validator, ok := realEngine.(engine.WorkflowPlacementValidator); ok {
358
+
if err := validator.ValidateWorkflowPlacement(wf); err != nil {
359
+
reject("validate workflow placement: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE)
360
+
return
361
+
}
362
+
}
363
+
// the job skipped processPipeline, so inject the TANGLED_* env vars here
364
+
if wf.Environment == nil {
365
+
wf.Environment = make(map[string]string)
366
+
}
367
+
maps.Copy(wf.Environment, models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId))
368
+
369
+
// nowait because the executor doesn't queue locally, the mill owns the backlog
370
+
slot, err := slotter.AcquireWorkflowSlot(ctx, wid, wf, engine.NoWait)
371
+
if err != nil {
372
+
class := millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE
373
+
if errors.Is(err, engine.ErrNoWorkflowSlots) {
374
+
class = millv1.RejectClass_REJECT_CLASS_TRANSIENT
375
+
}
376
+
reject(err.Error(), class)
377
+
return
378
+
}
379
+
380
+
var repoDid syntax.DID
381
+
if tpl.TriggerMetadata != nil && tpl.TriggerMetadata.Repo != nil && tpl.TriggerMetadata.Repo.RepoDid != nil {
382
+
repoDid, _ = syntax.ParseDID(*tpl.TriggerMetadata.Repo.RepoDid)
383
+
}
384
+
385
+
res := &reservation{
386
+
leaseID: rs.GetLeaseId(),
387
+
wid: wid,
388
+
realEngine: realEngine,
389
+
slot: slot,
390
+
wf: wf,
391
+
repoDid: repoDid,
392
+
}
393
+
394
+
e.snapshotMu.Lock()
395
+
e.mu.Lock()
396
+
e.active[res.leaseID] = res
397
+
// start timer under lock to avoid publish races
398
+
res.ttlTimer = time.AfterFunc(ttlDuration(rs.GetTtlSeconds()), func() { e.expireReservation(res.leaseID) })
399
+
e.mu.Unlock()
400
+
401
+
e.send(&millproto.Message{ReserveResult: &millv1.ReserveResult{
402
+
LeaseId: rs.GetLeaseId(),
403
+
Accepted: true,
404
+
}})
405
+
e.pushSnapshotLocked()
406
+
e.snapshotMu.Unlock()
407
+
}
408
+
409
+
func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) {
410
+
e.mu.Lock()
411
+
res := e.active[cl.GetLeaseId()]
412
+
if res == nil {
413
+
e.mu.Unlock()
414
+
e.sendReject(cl.GetLeaseId(), "reservation missing or expired", millv1.RejectClass_REJECT_CLASS_TRANSIENT)
415
+
return
416
+
}
417
+
if res.committed {
418
+
e.mu.Unlock()
419
+
e.sendCommitted(cl.GetLeaseId())
420
+
return
421
+
}
422
+
res.committed = true
423
+
if res.ttlTimer != nil {
424
+
res.ttlTimer.Stop()
425
+
}
426
+
427
+
// rooted in the lifecycle ctx, not the session, so the job survives reconnects
428
+
jobCtx, cancel := context.WithCancel(e.lifecycleCtx)
429
+
res.cancel = cancel
430
+
e.mu.Unlock()
431
+
432
+
vault := newMemVault(cl.GetSecrets())
433
+
re := newReservedEngine(res.realEngine, res.slot)
434
+
pipeline := &models.Pipeline{
435
+
RepoDid: res.repoDid,
436
+
Workflows: map[models.Engine][]models.Workflow{re: {*res.wf}},
437
+
TrustedSource: true,
438
+
}
439
+
440
+
e.startTail(res)
441
+
442
+
e.jobsWG.Add(1)
443
+
go func() {
444
+
defer e.jobsWG.Done()
445
+
engine.StartWorkflows(e.l, vault, e.cfg, e.db, e.n, jobCtx, pipeline, res.wid.PipelineId)
446
+
}()
447
+
448
+
e.sendCommitted(cl.GetLeaseId())
449
+
}
450
+
451
+
func (e *Executor) handleRelease(leaseID string) {
452
+
cleanup, ok := e.takeUncommittedReservation(leaseID, true)
453
+
if !ok {
454
+
return
455
+
}
456
+
e.releaseReservation(cleanup)
457
+
}
458
+
459
+
func (e *Executor) handleCancel(leaseID string) {
460
+
e.mu.Lock()
461
+
res := e.active[leaseID]
462
+
if res == nil {
463
+
e.mu.Unlock()
464
+
if err := e.appendTerminal(leaseID, string(models.StatusKindCancelled), nil); err != nil {
465
+
e.l.Error("persist cancelled reservation terminal", "lease", leaseID, "err", err)
466
+
return
467
+
}
468
+
e.sendCancelAck(leaseID)
469
+
return
470
+
}
471
+
res.cancelled = true
472
+
cancel := res.cancel
473
+
committed := res.committed
474
+
var cleanup func()
475
+
if !committed {
476
+
cleanup = e.removeReservationLocked(res, true)
477
+
}
478
+
e.mu.Unlock()
479
+
480
+
if !committed {
481
+
if err := e.appendTerminal(leaseID, string(models.StatusKindCancelled), nil); err != nil {
482
+
e.l.Error("persist cancelled reservation terminal", "lease", leaseID, "err", err)
483
+
e.releaseReservation(cleanup)
484
+
return
485
+
}
486
+
e.sendCancelAck(leaseID)
487
+
e.releaseReservation(cleanup)
488
+
return
489
+
}
490
+
491
+
e.sendCancelAck(leaseID)
492
+
if cancel != nil {
493
+
cancel()
494
+
}
495
+
}
496
+
497
+
func (e *Executor) expireReservation(leaseID string) {
498
+
cleanup, ok := e.takeUncommittedReservation(leaseID, true)
499
+
if !ok {
500
+
return
501
+
}
502
+
e.l.Warn("reservation expired before commit", "lease", leaseID)
503
+
e.releaseReservation(cleanup)
504
+
}
505
+
506
+
func (e *Executor) takeUncommittedReservation(leaseID string, releaseSlot bool) (func(), bool) {
507
+
e.mu.Lock()
508
+
defer e.mu.Unlock()
509
+
res := e.active[leaseID]
510
+
if res == nil || res.committed {
511
+
return func() {}, false
512
+
}
513
+
return e.removeReservationLocked(res, releaseSlot), true
514
+
}
515
+
516
+
func (e *Executor) removeReservationLocked(res *reservation, releaseSlot bool) func() {
517
+
delete(e.active, res.leaseID)
518
+
if res.ttlTimer != nil {
519
+
res.ttlTimer.Stop()
520
+
res.ttlTimer = nil
521
+
}
522
+
stopTail := res.stopTail
523
+
res.stopTail = nil
524
+
slot := res.slot
525
+
if releaseSlot {
526
+
res.slot = nil
527
+
}
528
+
return func() {
529
+
if stopTail != nil {
530
+
stopTail()
531
+
}
532
+
if releaseSlot && slot != nil {
533
+
slot.Release()
534
+
}
535
+
}
536
+
}
537
+
538
+
func normalizeLabels(labels []string) []string {
539
+
seen := make(map[string]struct{}, len(labels))
540
+
out := make([]string, 0, len(labels))
541
+
for _, label := range labels {
542
+
label = strings.TrimSpace(label)
543
+
if label == "" {
544
+
continue
545
+
}
546
+
if _, ok := seen[label]; ok {
547
+
continue
548
+
}
549
+
seen[label] = struct{}{}
550
+
out = append(out, label)
551
+
}
552
+
return out
553
+
}
554
+
555
+
func (e *Executor) snapshotLoop(ctx context.Context, enc *millproto.Encoder) {
556
+
t := time.NewTicker(snapshotEvery)
557
+
defer t.Stop()
558
+
for {
559
+
select {
560
+
case <-ctx.Done():
561
+
return
562
+
case <-t.C:
563
+
// stop once connection is replaced
564
+
e.connMu.Lock()
565
+
cur := e.enc
566
+
e.connMu.Unlock()
567
+
if cur != enc {
568
+
return
569
+
}
570
+
e.pushSnapshot()
571
+
}
572
+
}
573
+
}
574
+
575
+
func (e *Executor) pushSnapshot() {
576
+
e.snapshotMu.Lock()
577
+
defer e.snapshotMu.Unlock()
578
+
e.pushSnapshotLocked()
579
+
}
580
+
581
+
func (e *Executor) pushSnapshotLocked() {
582
+
583
+
e.mu.Lock()
584
+
draining := e.draining
585
+
active := len(e.active)
586
+
leaseIDs := make([]string, 0, len(e.active))
587
+
for id := range e.active {
588
+
leaseIDs = append(leaseIDs, id)
589
+
}
590
+
e.mu.Unlock()
591
+
592
+
load := 0.0
593
+
if e.seats > 0 {
594
+
load = float64(active) / float64(e.seats)
595
+
}
596
+
available := !draining && active < e.seats
597
+
loadMap := map[string]float64{"slots": load}
598
+
599
+
engines := make(map[string]*millv1.EngineAvailability, len(e.engines))
600
+
for name := range e.engines {
601
+
engines[name] = &millv1.EngineAvailability{
602
+
Available: available,
603
+
Load: loadMap,
604
+
}
605
+
}
606
+
607
+
e.nextSeqno++
608
+
seq := e.nextSeqno
609
+
610
+
e.send(&millproto.Message{NodeSnapshot: &millv1.NodeSnapshot{
611
+
Seqno: seq,
612
+
Engines: engines,
613
+
ActiveLeaseIds: leaseIDs,
614
+
}})
615
+
}
616
+
617
+
func (e *Executor) Drain() {
618
+
e.mu.Lock()
619
+
e.draining = true
620
+
e.mu.Unlock()
621
+
e.pushSnapshot()
622
+
}
623
+
624
+
func ttlDuration(secs uint32) time.Duration {
625
+
if secs == 0 {
626
+
return defaultReservationTTL
627
+
}
628
+
return time.Duration(secs) * time.Second
629
+
}
630
+
631
+
const defaultReservationTTL = 60 * time.Second
+178
spindle/mill/executor/observe.go
+178
spindle/mill/executor/observe.go
···
1
+
package executor
2
+
3
+
import (
4
+
"context"
5
+
"io"
6
+
"sync"
7
+
"time"
8
+
9
+
"github.com/hpcloud/tail"
10
+
11
+
"tangled.org/core/api/tangled"
12
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
13
+
"tangled.org/core/spindle/models"
14
+
"tangled.org/core/spindle/secrets"
15
+
)
16
+
17
+
// streams status rows of active leases. jobs write status exactly like a
18
+
// standalone spindle, we just forward them
19
+
func (e *Executor) observeLoop(ctx context.Context, sub <-chan struct{}, cursor int64) {
20
+
ticker := time.NewTicker(5 * time.Second)
21
+
defer ticker.Stop()
22
+
23
+
for {
24
+
select {
25
+
case <-ctx.Done():
26
+
return
27
+
case <-sub:
28
+
case <-ticker.C:
29
+
}
30
+
e.drainEvents(&cursor)
31
+
}
32
+
}
33
+
34
+
func (e *Executor) drainEvents(cursor *int64) {
35
+
for {
36
+
evs, err := e.db.GetEvents(*cursor, 100)
37
+
if err != nil {
38
+
e.l.Error("observe GetEvents failed", "err", err)
39
+
return
40
+
}
41
+
for _, ev := range evs {
42
+
if ev.Nsid == tangled.PipelineStatusNSID {
43
+
st, ok := parseStatus(ev.EventJson)
44
+
if ok {
45
+
if err := e.onStatusRow(st); err != nil {
46
+
e.l.Error("persist streamed status", "err", err)
47
+
return
48
+
}
49
+
}
50
+
}
51
+
*cursor = ev.Created
52
+
}
53
+
if len(evs) < 100 {
54
+
return
55
+
}
56
+
}
57
+
}
58
+
59
+
func (e *Executor) onStatusRow(st *tangled.PipelineStatus) error {
60
+
res := e.reservationFor(st.Pipeline, st.Workflow)
61
+
if res == nil {
62
+
return nil
63
+
}
64
+
65
+
kind := models.StatusKind(st.Status)
66
+
switch {
67
+
case kind.IsFinish():
68
+
return e.finishJob(res, st)
69
+
case kind == models.StatusKindRunning:
70
+
return e.appendStatus(res.leaseID, st)
71
+
default:
72
+
return nil
73
+
}
74
+
}
75
+
76
+
// flush the tail first so all log seqnos land before the terminal's
77
+
func (e *Executor) finishJob(res *reservation, st *tangled.PipelineStatus) error {
78
+
e.mu.Lock()
79
+
if e.active[res.leaseID] != res {
80
+
e.mu.Unlock()
81
+
return nil
82
+
}
83
+
cancelled := res.cancelled
84
+
cleanup := e.removeReservationLocked(res, false)
85
+
e.mu.Unlock()
86
+
87
+
cleanup()
88
+
89
+
terminalStatus := st.Status
90
+
if cancelled {
91
+
terminalStatus = string(models.StatusKindCancelled)
92
+
}
93
+
if err := e.appendTerminal(res.leaseID, terminalStatus, st); err != nil {
94
+
return err
95
+
}
96
+
e.pushSnapshot()
97
+
return nil
98
+
}
99
+
100
+
// reservations are few (bounded by seats), so a scan beats a parallel index
101
+
func (e *Executor) reservationFor(pipelineAturi, workflow string) *reservation {
102
+
e.mu.Lock()
103
+
defer e.mu.Unlock()
104
+
for _, res := range e.active {
105
+
if string(res.wid.PipelineId.AtUri()) == pipelineAturi && res.wid.Name == workflow {
106
+
return res
107
+
}
108
+
}
109
+
return nil
110
+
}
111
+
112
+
func (e *Executor) streamLogLine(leaseID, text string) {
113
+
e.appendLog(leaseID, []byte(text))
114
+
}
115
+
116
+
// log lines come pre-encoded as models.LogLine JSON, forward them verbatim
117
+
func (e *Executor) startTail(res *reservation) {
118
+
path := models.LogFilePath(e.cfg.Server.LogDir, res.wid)
119
+
t, err := tail.TailFile(path, tail.Config{
120
+
Follow: true,
121
+
ReOpen: true,
122
+
MustExist: false,
123
+
Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart},
124
+
Logger: tail.DiscardingLogger,
125
+
})
126
+
if err != nil {
127
+
e.l.Error("tail log file failed", "wid", res.wid, "err", err)
128
+
return
129
+
}
130
+
131
+
done := make(chan struct{})
132
+
go func() {
133
+
defer close(done)
134
+
for line := range t.Lines {
135
+
if line == nil || line.Err != nil {
136
+
continue
137
+
}
138
+
e.streamLogLine(res.leaseID, line.Text)
139
+
}
140
+
}()
141
+
142
+
var once sync.Once
143
+
res.stopTail = func() {
144
+
once.Do(func() {
145
+
// Stop() blocks until tailing ends, then the consumer drains
146
+
// what's buffered and closes done
147
+
// waiting for done means every log line streams before finishJob's
148
+
// terminal. a slow drain would let the terminal overtake a log line
149
+
// and the mill drops those
150
+
_ = t.Stop()
151
+
<-done
152
+
})
153
+
}
154
+
}
155
+
156
+
// in-memory secrets manager holding what the mill handed over at commit
157
+
type memVault struct {
158
+
secrets []secrets.UnlockedSecret
159
+
}
160
+
161
+
func newMemVault(pb []*millv1.Secret) *memVault {
162
+
s := make([]secrets.UnlockedSecret, len(pb))
163
+
for i, x := range pb {
164
+
s[i] = secrets.UnlockedSecret{Key: x.GetKey(), Value: x.GetValue()}
165
+
}
166
+
return &memVault{secrets: s}
167
+
}
168
+
169
+
func (v *memVault) GetSecretsUnlocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.UnlockedSecret, error) {
170
+
return v.secrets, nil
171
+
}
172
+
func (v *memVault) GetSecretsLocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.LockedSecret, error) {
173
+
return nil, nil
174
+
}
175
+
func (v *memVault) AddSecret(ctx context.Context, s secrets.UnlockedSecret) error { return nil }
176
+
func (v *memVault) RemoveSecret(ctx context.Context, s secrets.Secret[any]) error { return nil }
177
+
178
+
var _ secrets.Manager = (*memVault)(nil)
+393
spindle/mill/executor/outbox.go
+393
spindle/mill/executor/outbox.go
···
1
+
package executor
2
+
3
+
import (
4
+
"crypto/rand"
5
+
"encoding/hex"
6
+
"encoding/json"
7
+
"fmt"
8
+
"time"
9
+
"unicode/utf8"
10
+
11
+
"google.golang.org/protobuf/proto"
12
+
"tangled.org/core/api/tangled"
13
+
"tangled.org/core/spindle/db"
14
+
millproto "tangled.org/core/spindle/mill/proto"
15
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
16
+
"tangled.org/core/spindle/models"
17
+
)
18
+
19
+
const (
20
+
maxBatchEntries = 1000
21
+
maxBatchBytes = 1 * 1024 * 1024
22
+
maxEventErrorBytes = 64 * 1024
23
+
eventFlushDelay = 10 * time.Millisecond
24
+
)
25
+
26
+
func generateEpoch() string {
27
+
b := make([]byte, 16)
28
+
if _, err := rand.Read(b); err != nil {
29
+
panic(err)
30
+
}
31
+
return hex.EncodeToString(b)
32
+
}
33
+
34
+
func (e *Executor) initOutbox() error {
35
+
e.eventMu.Lock()
36
+
defer e.eventMu.Unlock()
37
+
38
+
inc, nextSeqno, err := e.db.GetOutboxState()
39
+
if err != nil {
40
+
return err
41
+
}
42
+
if inc == "" {
43
+
inc = generateEpoch()
44
+
if err := e.db.SetOutboxEpoch(inc); err != nil {
45
+
return err
46
+
}
47
+
}
48
+
e.epoch = inc
49
+
50
+
rows, err := e.db.ListOutboxRows()
51
+
if err != nil {
52
+
return err
53
+
}
54
+
e.outboxLogBytes = 0
55
+
e.outboxControlBytes = 0
56
+
var previousSeqno uint64
57
+
for _, r := range rows {
58
+
if r.Epoch != inc || r.Seqno == 0 || r.Seqno >= nextSeqno {
59
+
return fmt.Errorf("invalid outbox row epoch=%q seqno=%d next=%d", r.Epoch, r.Seqno, nextSeqno)
60
+
}
61
+
if previousSeqno != 0 && r.Seqno != previousSeqno+1 {
62
+
return fmt.Errorf("outbox seqno gap after %d: got %d", previousSeqno, r.Seqno)
63
+
}
64
+
entry, err := decodeEvent(r.Payload, r.Seqno)
65
+
if err != nil {
66
+
return err
67
+
}
68
+
if entry.GetLeaseId() == "" || entry.GetPayload() == nil {
69
+
return fmt.Errorf("invalid outbox payload at seqno %d", r.Seqno)
70
+
}
71
+
previousSeqno = r.Seqno
72
+
if r.Control {
73
+
e.outboxControlBytes += r.ByteSize
74
+
} else {
75
+
e.outboxLogBytes += r.ByteSize
76
+
}
77
+
}
78
+
return nil
79
+
}
80
+
81
+
func (e *Executor) appendAndSend(leaseID string, payload any, control bool) error {
82
+
entry := &millv1.Event{LeaseId: leaseID}
83
+
switch payload := payload.(type) {
84
+
case *millv1.Event_StatusEvent:
85
+
entry.Payload = payload
86
+
case *millv1.Event_LogLine:
87
+
entry.Payload = payload
88
+
case *millv1.Event_AttemptResult:
89
+
entry.Payload = payload
90
+
default:
91
+
return fmt.Errorf("unsupported stream payload %T", payload)
92
+
}
93
+
isTerminal := false
94
+
if _, ok := payload.(*millv1.Event_AttemptResult); ok {
95
+
isTerminal = true
96
+
}
97
+
entry.Seqno = ^uint64(0)
98
+
wireSize := proto.Size(&millproto.Message{EventBatch: &millv1.EventBatch{
99
+
Epoch: e.epoch,
100
+
Events: []*millv1.Event{entry},
101
+
}})
102
+
entry.Seqno = 0
103
+
if wireSize > maxBatchBytes {
104
+
if !control {
105
+
e.eventMu.Lock()
106
+
e.droppedLogs++
107
+
dropped := e.droppedLogs
108
+
e.eventMu.Unlock()
109
+
e.l.Warn("stream log entry exceeds wire limit; dropping line", "bytes", wireSize, "limit", maxBatchBytes, "droppedCount", dropped)
110
+
return nil
111
+
}
112
+
return fmt.Errorf("control stream entry exceeds wire limit: %d > %d", wireSize, maxBatchBytes)
113
+
}
114
+
115
+
encoded, err := proto.Marshal(entry)
116
+
if err != nil {
117
+
return fmt.Errorf("marshal stream entry: %w", err)
118
+
}
119
+
120
+
e.eventMu.Lock()
121
+
if !control && e.outboxLogBytes+int64(len(encoded)) > e.maxLogBytes {
122
+
e.droppedLogs++
123
+
e.l.Warn("outbox log cap exceeded; dropping log line", "droppedCount", e.droppedLogs, "cap", e.maxLogBytes)
124
+
e.eventMu.Unlock()
125
+
return nil
126
+
}
127
+
if control && !isTerminal && e.maxControlBytes > 0 && e.outboxControlBytes+int64(len(encoded)) > e.maxControlBytes {
128
+
e.l.Warn("control outbox reserve exhausted; dropping nonterminal status", "cap", e.maxControlBytes)
129
+
e.eventMu.Unlock()
130
+
return nil
131
+
}
132
+
if _, err := e.db.AppendOutboxRow(encoded, control); err != nil {
133
+
e.eventMu.Unlock()
134
+
return fmt.Errorf("append outbox row: %w", err)
135
+
}
136
+
if control {
137
+
e.outboxControlBytes += int64(len(encoded))
138
+
} else {
139
+
e.outboxLogBytes += int64(len(encoded))
140
+
}
141
+
e.eventMu.Unlock()
142
+
143
+
if control {
144
+
e.sendPending()
145
+
} else {
146
+
e.schedulePending()
147
+
}
148
+
return nil
149
+
}
150
+
151
+
func (e *Executor) appendStatus(leaseID string, st *tangled.PipelineStatus) error {
152
+
if st.Status != string(models.StatusKindRunning) {
153
+
return fmt.Errorf("unsupported nonterminal status %q", st.Status)
154
+
}
155
+
exit, errStr := parseStatusExitAndError(st)
156
+
payload := &millv1.Event_StatusEvent{StatusEvent: &millv1.StatusEvent{
157
+
Status: millv1.NonterminalStatus_RUNNING,
158
+
Error: errStr,
159
+
ExitCode: exit,
160
+
}}
161
+
return e.appendAndSend(leaseID, payload, true)
162
+
}
163
+
164
+
func (e *Executor) appendLog(leaseID string, raw []byte) {
165
+
logLine := &millv1.LogLine{
166
+
RawJson: raw,
167
+
}
168
+
payload := &millv1.Event_LogLine{LogLine: logLine}
169
+
if err := e.appendAndSend(leaseID, payload, false); err != nil {
170
+
e.l.Error("failed to persist streamed log", "lease", leaseID, "err", err)
171
+
}
172
+
}
173
+
174
+
func (e *Executor) appendTerminal(leaseID, status string, st *tangled.PipelineStatus) error {
175
+
var terminalStatus millv1.TerminalStatus
176
+
switch status {
177
+
case string(models.StatusKindSuccess):
178
+
terminalStatus = millv1.TerminalStatus_SUCCESS
179
+
case string(models.StatusKindFailed):
180
+
terminalStatus = millv1.TerminalStatus_FAILED
181
+
case string(models.StatusKindTimeout):
182
+
terminalStatus = millv1.TerminalStatus_TIMEOUT
183
+
case string(models.StatusKindCancelled):
184
+
terminalStatus = millv1.TerminalStatus_CANCELLED
185
+
default:
186
+
return fmt.Errorf("unsupported terminal status %q", status)
187
+
}
188
+
189
+
exit, errStr := parseStatusExitAndError(st)
190
+
payload := &millv1.Event_AttemptResult{AttemptResult: &millv1.AttemptResult{
191
+
Status: terminalStatus,
192
+
Error: errStr,
193
+
ExitCode: exit,
194
+
}}
195
+
return e.appendAndSend(leaseID, payload, true)
196
+
}
197
+
198
+
func truncateEventString(value string) string {
199
+
if len(value) <= maxEventErrorBytes {
200
+
return value
201
+
}
202
+
end := maxEventErrorBytes
203
+
for end > 0 && !utf8.RuneStart(value[end]) {
204
+
end--
205
+
}
206
+
return value[:end]
207
+
}
208
+
209
+
func (e *Executor) schedulePending() {
210
+
e.flushMu.Lock()
211
+
defer e.flushMu.Unlock()
212
+
if e.flushTimer != nil {
213
+
return
214
+
}
215
+
e.flushTimer = time.AfterFunc(eventFlushDelay, func() {
216
+
e.flushMu.Lock()
217
+
e.flushTimer = nil
218
+
e.flushMu.Unlock()
219
+
e.sendPending()
220
+
})
221
+
}
222
+
223
+
func (e *Executor) sendPending() {
224
+
e.sendMu.Lock()
225
+
defer e.sendMu.Unlock()
226
+
227
+
e.connMu.Lock()
228
+
enc := e.enc
229
+
cancel := e.sessionCancel
230
+
e.connMu.Unlock()
231
+
if enc == nil {
232
+
return
233
+
}
234
+
if err := e.sendPendingLocked(enc); err != nil {
235
+
e.l.Error("failed to send stream batch, ending session", "err", err)
236
+
if cancel != nil {
237
+
cancel()
238
+
}
239
+
}
240
+
}
241
+
242
+
func (e *Executor) sendPendingLocked(enc messageEncoder) error {
243
+
for {
244
+
e.eventMu.Lock()
245
+
rows, err := e.db.ListOutboxRowsAfter(e.sentSeqno, maxBatchEntries)
246
+
e.eventMu.Unlock()
247
+
if err != nil {
248
+
return err
249
+
}
250
+
if len(rows) == 0 {
251
+
return nil
252
+
}
253
+
if err := e.sendRows(enc, rows); err != nil {
254
+
return err
255
+
}
256
+
if len(rows) < maxBatchEntries {
257
+
return nil
258
+
}
259
+
}
260
+
}
261
+
262
+
func (e *Executor) sendRows(enc messageEncoder, rows []db.OutboxRow) error {
263
+
var (
264
+
batch []*millv1.Event
265
+
batchSize int64
266
+
last uint64
267
+
)
268
+
sendBatch := func() error {
269
+
if len(batch) == 0 {
270
+
return nil
271
+
}
272
+
if err := enc.Encode(&millproto.Message{EventBatch: &millv1.EventBatch{
273
+
Epoch: e.epoch,
274
+
Events: batch,
275
+
}}); err != nil {
276
+
return err
277
+
}
278
+
e.sentSeqno = last
279
+
batch = nil
280
+
batchSize = 0
281
+
return nil
282
+
}
283
+
284
+
for _, row := range rows {
285
+
if row.Seqno <= e.sentSeqno {
286
+
continue
287
+
}
288
+
entry, err := decodeEvent(row.Payload, row.Seqno)
289
+
if err != nil {
290
+
return err
291
+
}
292
+
entry.Seqno = row.Seqno
293
+
if len(batch) >= maxBatchEntries || batchSize+row.ByteSize > maxBatchBytes {
294
+
if err := sendBatch(); err != nil {
295
+
return err
296
+
}
297
+
}
298
+
batch = append(batch, entry)
299
+
batchSize += row.ByteSize
300
+
last = row.Seqno
301
+
}
302
+
return sendBatch()
303
+
}
304
+
305
+
func (e *Executor) replay(ackSeqno uint64) error {
306
+
e.sendMu.Lock()
307
+
defer e.sendMu.Unlock()
308
+
309
+
_, nextSeqno, err := e.db.GetOutboxState()
310
+
if err != nil {
311
+
return err
312
+
}
313
+
if ackSeqno >= nextSeqno {
314
+
return fmt.Errorf("mill acknowledged unsent seqno %d (next seqno %d)", ackSeqno, nextSeqno)
315
+
}
316
+
317
+
if err := e.deleteOutboxPrefix(ackSeqno); err != nil {
318
+
return err
319
+
}
320
+
321
+
e.sentSeqno = ackSeqno
322
+
e.connMu.Lock()
323
+
enc := e.enc
324
+
e.connMu.Unlock()
325
+
if enc == nil {
326
+
return nil
327
+
}
328
+
return e.sendPendingLocked(enc)
329
+
}
330
+
331
+
func (e *Executor) handleAck(ack *millv1.Ack) {
332
+
if ack.GetEpoch() != e.epoch {
333
+
e.l.Warn("received Ack for mismatched epoch", "got", ack.GetEpoch(), "want", e.epoch)
334
+
return
335
+
}
336
+
337
+
e.sendMu.Lock()
338
+
if ack.GetUpToSeqno() > e.sentSeqno {
339
+
e.sendMu.Unlock()
340
+
e.l.Warn("received Ack for unsent stream seqno", "got", ack.GetUpToSeqno(), "sent", e.sentSeqno)
341
+
return
342
+
}
343
+
err := e.deleteOutboxPrefix(ack.GetUpToSeqno())
344
+
e.sendMu.Unlock()
345
+
if err != nil {
346
+
e.l.Error("failed to delete outbox prefix on Ack", "upTo", ack.GetUpToSeqno(), "err", err)
347
+
}
348
+
}
349
+
350
+
func (e *Executor) subtractOutboxBytes(deleted db.OutboxDeletion) {
351
+
e.outboxLogBytes = max(0, e.outboxLogBytes-deleted.LogBytes)
352
+
e.outboxControlBytes = max(0, e.outboxControlBytes-deleted.ControlBytes)
353
+
}
354
+
355
+
func parseStatus(raw json.RawMessage) (*tangled.PipelineStatus, bool) {
356
+
var st tangled.PipelineStatus
357
+
if err := json.Unmarshal(raw, &st); err != nil {
358
+
return nil, false
359
+
}
360
+
return &st, true
361
+
}
362
+
func (e *Executor) deleteOutboxPrefix(upTo uint64) error {
363
+
e.eventMu.Lock()
364
+
defer e.eventMu.Unlock()
365
+
deleted, err := e.db.DeleteOutboxPrefix(upTo)
366
+
if err == nil {
367
+
e.subtractOutboxBytes(deleted)
368
+
}
369
+
return err
370
+
}
371
+
372
+
func parseStatusExitAndError(st *tangled.PipelineStatus) (int64, string) {
373
+
if st == nil {
374
+
return 0, ""
375
+
}
376
+
var exit int64
377
+
if st.ExitCode != nil {
378
+
exit = *st.ExitCode
379
+
}
380
+
var errStr string
381
+
if st.Error != nil {
382
+
errStr = truncateEventString(*st.Error)
383
+
}
384
+
return exit, errStr
385
+
}
386
+
387
+
func decodeEvent(payload []byte, seqno uint64) (*millv1.Event, error) {
388
+
var entry millv1.Event
389
+
if err := proto.Unmarshal(payload, &entry); err != nil {
390
+
return nil, fmt.Errorf("decode outbox seqno %d: %w", seqno, err)
391
+
}
392
+
return &entry, nil
393
+
}
+37
spindle/mill/executor/reserved.go
+37
spindle/mill/executor/reserved.go
···
1
+
package executor
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"sync"
7
+
8
+
"tangled.org/core/spindle/engine"
9
+
"tangled.org/core/spindle/models"
10
+
)
11
+
12
+
// wraps a real engine so StartWorkflows gets the slot ReserveSeat already
13
+
// acquired, not a second one. everything else delegates, the execution
14
+
// path runs exactly like standalone
15
+
type reservedEngine struct {
16
+
models.Engine
17
+
slot engine.WorkflowSlot
18
+
once sync.Once
19
+
}
20
+
21
+
func newReservedEngine(inner models.Engine, slot engine.WorkflowSlot) models.Engine {
22
+
return &reservedEngine{Engine: inner, slot: slot}
23
+
}
24
+
25
+
// hands back the pre-acquired slot exactly once, a second acquire would
26
+
// double-count it
27
+
func (e *reservedEngine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) {
28
+
var slot engine.WorkflowSlot
29
+
e.once.Do(func() {
30
+
slot = e.slot
31
+
e.slot = nil
32
+
})
33
+
if slot == nil {
34
+
return nil, fmt.Errorf("reserved slot already consumed")
35
+
}
36
+
return slot, nil
37
+
}
+867
spindle/mill/executor/reserved_test.go
+867
spindle/mill/executor/reserved_test.go
···
1
+
package executor
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"github.com/bluesky-social/indigo/atproto/syntax"
8
+
"github.com/gorilla/websocket"
9
+
"google.golang.org/protobuf/proto"
10
+
"io"
11
+
"log/slog"
12
+
"net/http"
13
+
"net/http/httptest"
14
+
"path/filepath"
15
+
"strings"
16
+
"testing"
17
+
"time"
18
+
19
+
"tangled.org/core/api/tangled"
20
+
"tangled.org/core/notifier"
21
+
"tangled.org/core/spindle/config"
22
+
"tangled.org/core/spindle/db"
23
+
"tangled.org/core/spindle/engine"
24
+
millproto "tangled.org/core/spindle/mill/proto"
25
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
26
+
"tangled.org/core/spindle/models"
27
+
"tangled.org/core/spindle/secrets"
28
+
)
29
+
30
+
type captureEncoder struct {
31
+
messages chan *millproto.Message
32
+
}
33
+
34
+
func newCaptureEncoder() *captureEncoder {
35
+
return &captureEncoder{messages: make(chan *millproto.Message, 4)}
36
+
}
37
+
38
+
func (e *captureEncoder) Encode(msg *millproto.Message) error {
39
+
e.messages <- msg
40
+
return nil
41
+
}
42
+
43
+
type fakeSlot struct{ released int }
44
+
45
+
func (s *fakeSlot) Release() { s.released++ }
46
+
47
+
type fakeEngine struct {
48
+
setupCalled bool
49
+
runCalled bool
50
+
destroyCalled bool
51
+
acquireCalled bool
52
+
secrets chan []secrets.UnlockedSecret
53
+
done chan struct{}
54
+
}
55
+
56
+
func (e *fakeEngine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) {
57
+
return &models.Workflow{Name: twf.Name}, nil
58
+
}
59
+
func (e *fakeEngine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, l models.WorkflowLogger) error {
60
+
e.setupCalled = true
61
+
return nil
62
+
}
63
+
func (e *fakeEngine) WorkflowTimeout() time.Duration { return 7 * time.Minute }
64
+
func (e *fakeEngine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error {
65
+
e.destroyCalled = true
66
+
return nil
67
+
}
68
+
func (e *fakeEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, s []secrets.UnlockedSecret, l models.WorkflowLogger) error {
69
+
e.runCalled = true
70
+
if e.secrets != nil {
71
+
e.secrets <- s
72
+
}
73
+
if e.done != nil {
74
+
close(e.done)
75
+
}
76
+
return nil
77
+
}
78
+
func (e *fakeEngine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode engine.AcquireMode) (engine.WorkflowSlot, error) {
79
+
e.acquireCalled = true
80
+
return &fakeSlot{}, nil
81
+
}
82
+
83
+
type fakeStep struct{}
84
+
85
+
func (fakeStep) Name() string { return "test" }
86
+
func (fakeStep) Command() string { return "true" }
87
+
func (fakeStep) Kind() models.StepKind { return models.StepKindUser }
88
+
89
+
func TestNewFailsWhenOutboxCannotInitialize(t *testing.T) {
90
+
d := testDB(t)
91
+
if err := d.Close(); err != nil {
92
+
t.Fatal(err)
93
+
}
94
+
n := notifier.New()
95
+
cfg := &config.Config{}
96
+
if _, err := New(cfg, nil, d, &n, slog.New(slog.NewTextHandler(io.Discard, nil))); err == nil {
97
+
t.Fatal("New succeeded with an unavailable outbox database")
98
+
}
99
+
}
100
+
101
+
func TestReservedEngineHandsBackHeldSlotOnce(t *testing.T) {
102
+
inner := &fakeEngine{}
103
+
slot := &fakeSlot{}
104
+
re := newReservedEngine(inner, slot)
105
+
106
+
got, err := re.(engine.WorkflowSlotter).AcquireWorkflowSlot(context.Background(), models.WorkflowId{}, nil, engine.Wait)
107
+
if err != nil {
108
+
t.Fatal(err)
109
+
}
110
+
if got != engine.WorkflowSlot(slot) {
111
+
t.Fatal("AcquireWorkflowSlot() did not return the held slot")
112
+
}
113
+
if inner.acquireCalled {
114
+
t.Fatal("wrapper must not call the inner engine's AcquireWorkflowSlot")
115
+
}
116
+
117
+
if _, err := re.(engine.WorkflowSlotter).AcquireWorkflowSlot(context.Background(), models.WorkflowId{}, nil, engine.Wait); err == nil {
118
+
t.Fatal("second AcquireWorkflowSlot() should error")
119
+
}
120
+
}
121
+
122
+
func TestReservedEngineForwardsLifecycle(t *testing.T) {
123
+
inner := &fakeEngine{}
124
+
re := newReservedEngine(inner, &fakeSlot{})
125
+
126
+
if d := re.WorkflowTimeout(); d != 7*time.Minute {
127
+
t.Fatalf("WorkflowTimeout() = %v, want 7m", d)
128
+
}
129
+
if err := re.SetupWorkflow(context.Background(), models.WorkflowId{}, &models.Workflow{}, models.NullLogger{}); err != nil {
130
+
t.Fatal(err)
131
+
}
132
+
if err := re.RunStep(context.Background(), models.WorkflowId{}, &models.Workflow{Steps: []models.Step{}}, 0, nil, models.NullLogger{}); err != nil {
133
+
t.Fatal(err)
134
+
}
135
+
if err := re.DestroyWorkflow(context.Background(), models.WorkflowId{}); err != nil {
136
+
t.Fatal(err)
137
+
}
138
+
if !inner.setupCalled || !inner.runCalled || !inner.destroyCalled {
139
+
t.Fatalf("lifecycle not forwarded: %+v", inner)
140
+
}
141
+
}
142
+
143
+
func TestHandleCommitIsIdempotent(t *testing.T) {
144
+
enc := newCaptureEncoder()
145
+
e := testExecutor(t)
146
+
e.enc = enc
147
+
e.active["lease-1"] = &reservation{leaseID: "lease-1", committed: true}
148
+
149
+
e.handleCommit(context.Background(), &millv1.CommitLease{LeaseId: "lease-1"})
150
+
msg := <-enc.messages
151
+
if got := msg.GetCommitted().GetLeaseId(); got != "lease-1" {
152
+
t.Fatalf("Committed lease = %q, want lease-1", got)
153
+
}
154
+
}
155
+
156
+
func TestHandleCommitRejectsMissingReservation(t *testing.T) {
157
+
enc := newCaptureEncoder()
158
+
e := testExecutor(t)
159
+
e.enc = enc
160
+
161
+
e.handleCommit(context.Background(), &millv1.CommitLease{LeaseId: "expired"})
162
+
result := (<-enc.messages).GetReserveResult()
163
+
if result == nil {
164
+
t.Fatal("missing reservation commit did not receive a ReserveResult")
165
+
}
166
+
if result.GetLeaseId() != "expired" || result.GetAccepted() {
167
+
t.Fatalf("ReserveResult = %+v, want correlated rejection", result)
168
+
}
169
+
}
170
+
171
+
func TestHandleCancelFinalizesExpiredReservation(t *testing.T) {
172
+
enc := newCaptureEncoder()
173
+
e := testExecutor(t)
174
+
e.enc = enc
175
+
176
+
e.handleCancel("lease-expired")
177
+
var ack *millv1.CancelAck
178
+
for ack == nil {
179
+
select {
180
+
case msg := <-enc.messages:
181
+
ack = msg.GetCancelAck()
182
+
case <-time.After(time.Second):
183
+
t.Fatal("cancel acknowledgement timed out")
184
+
}
185
+
}
186
+
if ack.GetLeaseId() != "lease-expired" {
187
+
t.Fatalf("CancelAck = %+v, want lease-expired", ack)
188
+
}
189
+
rows, err := e.db.ListOutboxRows()
190
+
if err != nil {
191
+
t.Fatal(err)
192
+
}
193
+
if len(rows) != 1 {
194
+
t.Fatalf("cancel terminal outbox rows = %d, want 1", len(rows))
195
+
}
196
+
var entry millv1.Event
197
+
if err := proto.Unmarshal(rows[0].Payload, &entry); err != nil {
198
+
t.Fatal(err)
199
+
}
200
+
if got := entry.GetAttemptResult().GetStatus(); got != millv1.TerminalStatus_CANCELLED {
201
+
t.Fatalf("cancel terminal = %v, want CANCELLED", got)
202
+
}
203
+
}
204
+
205
+
func TestHandleCommitPreservesPreauthorizedSecrets(t *testing.T) {
206
+
d := testDB(t)
207
+
n := notifier.New()
208
+
enc := newCaptureEncoder()
209
+
e := &Executor{
210
+
cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}},
211
+
db: d,
212
+
n: &n,
213
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
214
+
active: make(map[string]*reservation),
215
+
maxLogBytes: 100 * 1024 * 1024,
216
+
maxControlBytes: 10 * 1024 * 1024,
217
+
enc: enc,
218
+
}
219
+
if err := e.initOutbox(); err != nil {
220
+
t.Fatal(err)
221
+
}
222
+
e.lifecycleCtx = context.Background()
223
+
224
+
inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})}
225
+
slot := &fakeSlot{}
226
+
repoDid, err := syntax.ParseDID("did:web:example.com")
227
+
if err != nil {
228
+
t.Fatal(err)
229
+
}
230
+
res := &reservation{
231
+
leaseID: "lease-1",
232
+
wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"},
233
+
realEngine: inner,
234
+
slot: slot,
235
+
wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}},
236
+
repoDid: repoDid,
237
+
}
238
+
e.active[res.leaseID] = res
239
+
240
+
e.handleCommit(context.Background(), &millv1.CommitLease{
241
+
LeaseId: res.leaseID,
242
+
Secrets: []*millv1.Secret{{Key: "TOKEN", Value: "secret-value"}},
243
+
})
244
+
if got := (<-enc.messages).GetCommitted().GetLeaseId(); got != res.leaseID {
245
+
t.Fatalf("Committed lease = %q, want %q", got, res.leaseID)
246
+
}
247
+
select {
248
+
case got := <-inner.secrets:
249
+
if len(got) != 1 || got[0].Key != "TOKEN" || got[0].Value != "secret-value" {
250
+
t.Fatalf("RunStep secrets = %+v", got)
251
+
}
252
+
case <-time.After(2 * time.Second):
253
+
t.Fatal("RunStep did not receive CommitLease secrets")
254
+
}
255
+
select {
256
+
case <-inner.done:
257
+
case <-time.After(2 * time.Second):
258
+
t.Fatal("workflow did not finish")
259
+
}
260
+
if res.stopTail != nil {
261
+
res.stopTail()
262
+
}
263
+
e.jobsWG.Wait()
264
+
}
265
+
266
+
func TestRunSessionCancellationClosesStalledWebsocket(t *testing.T) {
267
+
connected := make(chan struct{})
268
+
release := make(chan struct{})
269
+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
270
+
conn, err := websocket.Upgrade(w, r, nil, 1024, 1024)
271
+
if err != nil {
272
+
return
273
+
}
274
+
defer conn.Close()
275
+
close(connected)
276
+
<-release
277
+
}))
278
+
t.Cleanup(func() {
279
+
close(release)
280
+
srv.Close()
281
+
})
282
+
283
+
e := testSessionExecutor(t, "ws"+strings.TrimPrefix(srv.URL, "http"))
284
+
ctx, cancel := context.WithCancel(context.Background())
285
+
done := make(chan error, 1)
286
+
go func() { done <- e.runSession(ctx) }()
287
+
<-connected
288
+
cancel()
289
+
290
+
select {
291
+
case <-done:
292
+
case <-time.After(2 * time.Second):
293
+
t.Fatal("runSession did not return after context cancellation")
294
+
}
295
+
}
296
+
297
+
func TestRunSessionConsumesAcksWhileReplaying(t *testing.T) {
298
+
e := testSessionExecutor(t, "")
299
+
e.appendLog("lease", []byte("first"))
300
+
e.appendLog("lease", make([]byte, 7<<20))
301
+
302
+
serverErr := make(chan error, 1)
303
+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
304
+
conn, err := websocket.Upgrade(w, r, nil, 1024, 1024)
305
+
if err != nil {
306
+
serverErr <- err
307
+
return
308
+
}
309
+
defer conn.Close()
310
+
if tcp, ok := conn.UnderlyingConn().(interface{ SetReadBuffer(int) error }); ok {
311
+
_ = tcp.SetReadBuffer(1024)
312
+
}
313
+
stream := millproto.NewWSStream(conn)
314
+
enc := millproto.NewEncoder(stream)
315
+
dec := millproto.NewDecoder(stream)
316
+
if _, err := dec.Decode(); err != nil {
317
+
serverErr <- err
318
+
return
319
+
}
320
+
if err := enc.Encode(&millproto.Message{Resume: &millv1.Resume{Epoch: e.epoch}}); err != nil {
321
+
serverErr <- err
322
+
return
323
+
}
324
+
first, err := dec.Decode()
325
+
if err != nil {
326
+
serverErr <- err
327
+
return
328
+
}
329
+
batch := first.GetEventBatch()
330
+
if batch == nil || len(batch.GetEvents()) == 0 {
331
+
serverErr <- fmt.Errorf("expected non-empty batch")
332
+
return
333
+
}
334
+
got := batch.GetEvents()[0].GetSeqno()
335
+
if got != 1 {
336
+
serverErr <- fmt.Errorf("first replay seqno = %d, want 1", got)
337
+
return
338
+
}
339
+
if err := enc.Encode(&millproto.Message{Ack: &millv1.Ack{Epoch: e.epoch, UpToSeqno: 1}}); err != nil {
340
+
serverErr <- err
341
+
return
342
+
}
343
+
344
+
deadline := time.Now().Add(10 * time.Second)
345
+
for {
346
+
rows, err := e.db.ListOutboxRows()
347
+
if err == nil && len(rows) <= 1 {
348
+
break
349
+
}
350
+
if time.Now().After(deadline) {
351
+
serverErr <- context.DeadlineExceeded
352
+
return
353
+
}
354
+
time.Sleep(time.Millisecond)
355
+
}
356
+
357
+
serverErr <- nil
358
+
}))
359
+
t.Cleanup(srv.Close)
360
+
e.millURL = "ws" + strings.TrimPrefix(srv.URL, "http")
361
+
362
+
done := make(chan error, 1)
363
+
go func() { done <- e.runSession(context.Background()) }()
364
+
select {
365
+
case err := <-serverErr:
366
+
if err != nil {
367
+
t.Fatalf("server: %v", err)
368
+
}
369
+
case <-time.After(20 * time.Second):
370
+
t.Fatal("replay stalled while the peer waited for its acknowledgement to be consumed")
371
+
}
372
+
select {
373
+
case <-done:
374
+
case <-time.After(5 * time.Second):
375
+
t.Fatal("runSession did not stop after peer closed")
376
+
}
377
+
}
378
+
379
+
func testSessionExecutor(t *testing.T, url string) *Executor {
380
+
d := testDB(t)
381
+
e := &Executor{
382
+
millURL: url,
383
+
seats: 1,
384
+
engines: make(map[string]models.Engine),
385
+
db: d,
386
+
cfg: &config.Config{Server: config.Server{Dev: true}},
387
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
388
+
active: make(map[string]*reservation),
389
+
maxLogBytes: 100 * 1024 * 1024,
390
+
maxControlBytes: 10 * 1024 * 1024,
391
+
}
392
+
if err := e.initOutbox(); err != nil {
393
+
t.Fatal(err)
394
+
}
395
+
return e
396
+
}
397
+
398
+
func testExecutor(t *testing.T) *Executor {
399
+
d := testDB(t)
400
+
e := &Executor{
401
+
db: d,
402
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
403
+
active: make(map[string]*reservation),
404
+
maxLogBytes: 100 * 1024 * 1024,
405
+
maxControlBytes: 10 * 1024 * 1024,
406
+
}
407
+
if err := e.initOutbox(); err != nil {
408
+
t.Fatal(err)
409
+
}
410
+
return e
411
+
}
412
+
413
+
func testDB(t *testing.T) *db.DB {
414
+
d, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "spindle.db"))
415
+
if err != nil {
416
+
t.Fatal(err)
417
+
}
418
+
return d
419
+
}
420
+
421
+
func TestFinishJobReportsCancelledReservationAsCancelled(t *testing.T) {
422
+
d := testDB(t)
423
+
res := &reservation{
424
+
leaseID: "lease-1",
425
+
wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"},
426
+
cancelled: true,
427
+
}
428
+
e := &Executor{
429
+
db: d,
430
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
431
+
active: map[string]*reservation{res.leaseID: res},
432
+
maxLogBytes: 100 * 1024 * 1024,
433
+
maxControlBytes: 10 * 1024 * 1024,
434
+
}
435
+
if err := e.initOutbox(); err != nil {
436
+
t.Fatal(err)
437
+
}
438
+
439
+
e.finishJob(res, &tangled.PipelineStatus{
440
+
Pipeline: string(res.wid.PipelineId.AtUri()),
441
+
Workflow: res.wid.Name,
442
+
Status: string(models.StatusKindFailed),
443
+
})
444
+
445
+
rows, err := d.ListOutboxRows()
446
+
if err != nil {
447
+
t.Fatal(err)
448
+
}
449
+
if len(rows) != 1 {
450
+
t.Fatalf("outbox rows = %d, want 1", len(rows))
451
+
}
452
+
453
+
var entry millv1.Event
454
+
if err := proto.Unmarshal(rows[0].Payload, &entry); err != nil {
455
+
t.Fatal(err)
456
+
}
457
+
got := entry.GetAttemptResult().GetStatus()
458
+
if got != millv1.TerminalStatus_CANCELLED {
459
+
t.Fatalf("terminal status = %v, want CANCELLED", got)
460
+
}
461
+
}
462
+
463
+
func TestRestartOutboxReplay(t *testing.T) {
464
+
d := testDB(t)
465
+
466
+
e1 := &Executor{
467
+
db: d,
468
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
469
+
active: make(map[string]*reservation),
470
+
maxLogBytes: 100 * 1024 * 1024,
471
+
maxControlBytes: 10 * 1024 * 1024,
472
+
}
473
+
if err := e1.initOutbox(); err != nil {
474
+
t.Fatal(err)
475
+
}
476
+
inc1 := e1.epoch
477
+
478
+
e1.appendLog("lease-1", []byte("log 1"))
479
+
480
+
e2 := &Executor{
481
+
db: d,
482
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
483
+
active: make(map[string]*reservation),
484
+
maxLogBytes: 100 * 1024 * 1024,
485
+
maxControlBytes: 10 * 1024 * 1024,
486
+
}
487
+
if err := e2.initOutbox(); err != nil {
488
+
t.Fatal(err)
489
+
}
490
+
491
+
if e2.epoch != inc1 {
492
+
t.Fatalf("reconstructed executor got epoch %q, want %q", e2.epoch, inc1)
493
+
}
494
+
495
+
enc := newCaptureEncoder()
496
+
e2.enc = enc
497
+
498
+
if err := e2.replay(0); err != nil {
499
+
t.Fatal(err)
500
+
}
501
+
502
+
select {
503
+
case msg := <-enc.messages:
504
+
batch := msg.GetEventBatch()
505
+
if batch == nil {
506
+
t.Fatal("expected EventBatch")
507
+
}
508
+
if batch.GetEpoch() != inc1 {
509
+
t.Fatalf("batch epoch = %q, want %q", batch.GetEpoch(), inc1)
510
+
}
511
+
if len(batch.GetEvents()) != 1 {
512
+
t.Fatalf("expected 1 entry, got %d", len(batch.GetEvents()))
513
+
}
514
+
entry := batch.GetEvents()[0]
515
+
if string(entry.GetLogLine().GetRawJson()) != "log 1" {
516
+
t.Fatalf("got log raw = %q, want log 1", entry.GetLogLine().GetRawJson())
517
+
}
518
+
case <-time.After(2 * time.Second):
519
+
t.Fatal("timeout waiting for replay message")
520
+
}
521
+
}
522
+
523
+
func TestReplayRejectsMalformedOutboxRow(t *testing.T) {
524
+
d := testDB(t)
525
+
e := &Executor{
526
+
db: d,
527
+
enc: newCaptureEncoder(),
528
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
529
+
maxLogBytes: 100 * 1024 * 1024,
530
+
}
531
+
if err := e.initOutbox(); err != nil {
532
+
t.Fatal(err)
533
+
}
534
+
if _, err := d.AppendOutboxRow([]byte("not protobuf"), true); err != nil {
535
+
t.Fatal(err)
536
+
}
537
+
if err := e.replay(0); err == nil {
538
+
t.Fatal("replay accepted a malformed row and would leave a permanent seqno gap")
539
+
}
540
+
}
541
+
542
+
func TestBatchAck(t *testing.T) {
543
+
d := testDB(t)
544
+
e := &Executor{
545
+
db: d,
546
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
547
+
active: make(map[string]*reservation),
548
+
maxLogBytes: 100 * 1024 * 1024,
549
+
maxControlBytes: 10 * 1024 * 1024,
550
+
}
551
+
if err := e.initOutbox(); err != nil {
552
+
t.Fatal(err)
553
+
}
554
+
555
+
e.appendLog("lease-1", []byte("log 1"))
556
+
e.appendLog("lease-1", []byte("log 2"))
557
+
e.sentSeqno = 2
558
+
559
+
rows, err := d.ListOutboxRows()
560
+
if err != nil || len(rows) != 2 {
561
+
t.Fatalf("expected 2 rows, got %d", len(rows))
562
+
}
563
+
564
+
e.handleAck(&millv1.Ack{
565
+
Epoch: e.epoch,
566
+
UpToSeqno: 1,
567
+
})
568
+
569
+
rows, err = d.ListOutboxRows()
570
+
if err != nil || len(rows) != 1 {
571
+
t.Fatalf("expected 1 row after Ack, got %d", len(rows))
572
+
}
573
+
if rows[0].Seqno != 2 {
574
+
t.Fatalf("expected remaining row to have seqno 2, got %d", rows[0].Seqno)
575
+
}
576
+
}
577
+
578
+
func TestLiveEventBatchesLogsWithoutResendingUnackedRows(t *testing.T) {
579
+
d := testDB(t)
580
+
enc := newCaptureEncoder()
581
+
e := &Executor{
582
+
db: d,
583
+
enc: enc,
584
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
585
+
active: make(map[string]*reservation),
586
+
maxLogBytes: 100 * 1024 * 1024,
587
+
}
588
+
if err := e.initOutbox(); err != nil {
589
+
t.Fatal(err)
590
+
}
591
+
592
+
e.appendLog("lease-1", []byte("one"))
593
+
e.appendLog("lease-1", []byte("two"))
594
+
select {
595
+
case msg := <-enc.messages:
596
+
entries := msg.GetEventBatch().GetEvents()
597
+
if len(entries) != 2 || entries[0].GetSeqno() != 1 || entries[1].GetSeqno() != 2 {
598
+
t.Fatalf("first batch entries = %+v, want seqnos 1 and 2", entries)
599
+
}
600
+
case <-time.After(time.Second):
601
+
t.Fatal("timed out waiting for coalesced stream batch")
602
+
}
603
+
604
+
e.appendLog("lease-1", []byte("three"))
605
+
select {
606
+
case msg := <-enc.messages:
607
+
entries := msg.GetEventBatch().GetEvents()
608
+
if len(entries) != 1 || entries[0].GetSeqno() != 3 {
609
+
t.Fatalf("second batch entries = %+v, want only seqno 3", entries)
610
+
}
611
+
case <-time.After(time.Second):
612
+
t.Fatal("timed out waiting for second stream batch")
613
+
}
614
+
}
615
+
616
+
func TestSocketCancellationIndependence(t *testing.T) {
617
+
d := testDB(t)
618
+
n := notifier.New()
619
+
e := &Executor{
620
+
db: d,
621
+
n: &n,
622
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
623
+
active: make(map[string]*reservation),
624
+
maxLogBytes: 100 * 1024 * 1024,
625
+
maxControlBytes: 10 * 1024 * 1024,
626
+
cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}},
627
+
}
628
+
if err := e.initOutbox(); err != nil {
629
+
t.Fatal(err)
630
+
}
631
+
632
+
lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background())
633
+
defer cancelLifecycle()
634
+
e.lifecycleCtx = lifecycleCtx
635
+
636
+
inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})}
637
+
slot := &fakeSlot{}
638
+
repoDid, err := syntax.ParseDID("did:web:example.com")
639
+
if err != nil {
640
+
t.Fatal(err)
641
+
}
642
+
res := &reservation{
643
+
leaseID: "lease-1",
644
+
wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"},
645
+
realEngine: inner,
646
+
slot: slot,
647
+
wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}},
648
+
repoDid: repoDid,
649
+
}
650
+
e.active[res.leaseID] = res
651
+
652
+
sessionCtx, cancelSession := context.WithCancel(lifecycleCtx)
653
+
654
+
e.handleCommit(sessionCtx, &millv1.CommitLease{
655
+
LeaseId: "lease-1",
656
+
})
657
+
658
+
cancelSession()
659
+
660
+
// session disconnect must not cancel the running job
661
+
select {
662
+
case <-inner.done:
663
+
case <-time.After(2 * time.Second):
664
+
t.Fatal("workflow did not complete even though websocket session was cancelled")
665
+
}
666
+
667
+
e.jobsWG.Wait()
668
+
}
669
+
670
+
func TestMonotonicSnapshots(t *testing.T) {
671
+
enc := newCaptureEncoder()
672
+
e := &Executor{
673
+
enc: enc,
674
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
675
+
active: make(map[string]*reservation),
676
+
maxLogBytes: 100 * 1024 * 1024,
677
+
maxControlBytes: 10 * 1024 * 1024,
678
+
}
679
+
680
+
e.pushSnapshot()
681
+
msg1 := <-enc.messages
682
+
seq1 := msg1.GetNodeSnapshot().GetSeqno()
683
+
if seq1 != 1 {
684
+
t.Fatalf("first seq = %d, want 1", seq1)
685
+
}
686
+
687
+
e.pushSnapshot()
688
+
msg2 := <-enc.messages
689
+
seq2 := msg2.GetNodeSnapshot().GetSeqno()
690
+
if seq2 != 2 {
691
+
t.Fatalf("second seq = %d, want 2", seq2)
692
+
}
693
+
}
694
+
695
+
func TestTimerRace(t *testing.T) {
696
+
d := testDB(t)
697
+
e := &Executor{
698
+
db: d,
699
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
700
+
active: make(map[string]*reservation),
701
+
maxLogBytes: 100 * 1024 * 1024,
702
+
maxControlBytes: 10 * 1024 * 1024,
703
+
}
704
+
if err := e.initOutbox(); err != nil {
705
+
t.Fatal(err)
706
+
}
707
+
708
+
twf, _ := json.Marshal(tangled.Pipeline_Workflow{Name: "build"})
709
+
tpl, _ := json.Marshal(tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}})
710
+
711
+
inner := &fakeEngine{}
712
+
e.engines = map[string]models.Engine{"microvm": inner}
713
+
714
+
e.handleReserve(context.Background(), &millv1.ReserveSeat{
715
+
LeaseId: "lease-1",
716
+
TargetEngine: "microvm",
717
+
RawWorkflowJson: string(twf),
718
+
RawPipelineJson: string(tpl),
719
+
TtlSeconds: 1,
720
+
})
721
+
722
+
e.mu.Lock()
723
+
res := e.active["lease-1"]
724
+
e.mu.Unlock()
725
+
726
+
if res == nil {
727
+
t.Fatal("reservation was not added")
728
+
}
729
+
730
+
deadline := time.Now().Add(5 * time.Second)
731
+
for {
732
+
e.mu.Lock()
733
+
activeLen := len(e.active)
734
+
e.mu.Unlock()
735
+
if activeLen == 0 {
736
+
break
737
+
}
738
+
if time.Now().After(deadline) {
739
+
t.Fatal("reservation was leaked and never expired")
740
+
}
741
+
time.Sleep(10 * time.Millisecond)
742
+
}
743
+
}
744
+
745
+
func TestOversizedLogBehavior(t *testing.T) {
746
+
d := testDB(t)
747
+
e := &Executor{
748
+
db: d,
749
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
750
+
active: make(map[string]*reservation),
751
+
maxLogBytes: 10,
752
+
maxControlBytes: 10 * 1024 * 1024,
753
+
}
754
+
if err := e.initOutbox(); err != nil {
755
+
t.Fatal(err)
756
+
}
757
+
758
+
e.appendLog("lease-1", []byte("12345678901"))
759
+
760
+
rows, err := d.ListOutboxRows()
761
+
if err != nil {
762
+
t.Fatal(err)
763
+
}
764
+
if len(rows) != 0 {
765
+
t.Fatalf("expected log to be dropped, but got %d outbox rows", len(rows))
766
+
}
767
+
if e.droppedLogs != 1 {
768
+
t.Fatalf("droppedLogs counter = %d, want 1", e.droppedLogs)
769
+
}
770
+
}
771
+
772
+
func TestBoundedOutbox(t *testing.T) {
773
+
d := testDB(t)
774
+
e := &Executor{
775
+
db: d,
776
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
777
+
active: make(map[string]*reservation),
778
+
maxLogBytes: 100,
779
+
maxControlBytes: 10 * 1024 * 1024,
780
+
}
781
+
if err := e.initOutbox(); err != nil {
782
+
t.Fatal(err)
783
+
}
784
+
785
+
e.appendLog("lease-1", []byte("12345"))
786
+
rows, err := d.ListOutboxRows()
787
+
if err != nil || len(rows) != 1 {
788
+
t.Fatalf("expected 1 row, got %d", len(rows))
789
+
}
790
+
791
+
e.appendLog("lease-1", []byte("12345"))
792
+
rows, err = d.ListOutboxRows()
793
+
if err != nil || len(rows) != 2 {
794
+
t.Fatalf("expected 2 rows, got %d", len(rows))
795
+
}
796
+
797
+
e.appendLog("lease-1", make([]byte, 80))
798
+
rows, err = d.ListOutboxRows()
799
+
if err != nil {
800
+
t.Fatal(err)
801
+
}
802
+
if len(rows) != 2 {
803
+
t.Fatalf("expected 3rd log to be dropped, got %d rows", len(rows))
804
+
}
805
+
806
+
// control status is never dropped even over the cap
807
+
if err := e.appendStatus("lease-1", &tangled.PipelineStatus{
808
+
Status: string(models.StatusKindRunning),
809
+
}); err != nil {
810
+
t.Fatal(err)
811
+
}
812
+
813
+
rows, err = d.ListOutboxRows()
814
+
if err != nil || len(rows) != 3 {
815
+
t.Fatalf("expected status to be appended regardless of log cap, got %d rows", len(rows))
816
+
}
817
+
}
818
+
819
+
func TestStructuredShutdown(t *testing.T) {
820
+
d := testDB(t)
821
+
n := notifier.New()
822
+
e := &Executor{
823
+
db: d,
824
+
n: &n,
825
+
l: slog.New(slog.NewTextHandler(io.Discard, nil)),
826
+
active: make(map[string]*reservation),
827
+
maxLogBytes: 100 * 1024 * 1024,
828
+
maxControlBytes: 10 * 1024 * 1024,
829
+
cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}},
830
+
}
831
+
if err := e.initOutbox(); err != nil {
832
+
t.Fatal(err)
833
+
}
834
+
835
+
ctx, cancel := context.WithCancel(context.Background())
836
+
e.lifecycleCtx = ctx
837
+
838
+
inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})}
839
+
slot := &fakeSlot{}
840
+
repoDid, err := syntax.ParseDID("did:web:example.com")
841
+
if err != nil {
842
+
t.Fatal(err)
843
+
}
844
+
res := &reservation{
845
+
leaseID: "lease-1",
846
+
wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"},
847
+
realEngine: inner,
848
+
slot: slot,
849
+
wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}},
850
+
repoDid: repoDid,
851
+
}
852
+
e.active[res.leaseID] = res
853
+
854
+
e.handleCommit(ctx, &millv1.CommitLease{
855
+
LeaseId: "lease-1",
856
+
})
857
+
858
+
cancel()
859
+
860
+
e.jobsWG.Wait()
861
+
862
+
select {
863
+
case <-inner.done:
864
+
default:
865
+
t.Fatal("shutdown returned but running job did not finish")
866
+
}
867
+
}
+183
spindle/mill/handler.go
+183
spindle/mill/handler.go
···
1
+
package mill
2
+
3
+
import (
4
+
"github.com/gorilla/websocket"
5
+
"io"
6
+
"net/http"
7
+
"slices"
8
+
"strings"
9
+
"sync"
10
+
"time"
11
+
12
+
millproto "tangled.org/core/spindle/mill/proto"
13
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
14
+
)
15
+
16
+
var (
17
+
handshakeSem = make(chan struct{}, 16)
18
+
handshakeMu sync.Mutex
19
+
inFlightHandshakes = make(map[string]struct{})
20
+
)
21
+
22
+
type livenessReader struct {
23
+
r io.Reader
24
+
conn *websocket.Conn
25
+
readTimeout time.Duration
26
+
}
27
+
28
+
func (lr *livenessReader) Read(p []byte) (int, error) {
29
+
if err := lr.conn.SetReadDeadline(time.Now().Add(lr.readTimeout)); err != nil {
30
+
return 0, err
31
+
}
32
+
return lr.r.Read(p)
33
+
}
34
+
35
+
var upgrader = websocket.Upgrader{
36
+
ReadBufferSize: 1024,
37
+
WriteBufferSize: 1024,
38
+
}
39
+
40
+
// auth before upgrade, a bad token never opens a socket
41
+
func (m *Mill) HandleExecutorConn(w http.ResponseWriter, r *http.Request) {
42
+
name, authorizedLabels, ok := m.authenticate(r)
43
+
if !ok {
44
+
http.Error(w, "unauthorized", http.StatusUnauthorized)
45
+
return
46
+
}
47
+
48
+
select {
49
+
case handshakeSem <- struct{}{}:
50
+
case <-r.Context().Done():
51
+
return
52
+
}
53
+
handshakeSlotHeld := true
54
+
defer func() {
55
+
if handshakeSlotHeld {
56
+
<-handshakeSem
57
+
}
58
+
}()
59
+
60
+
// enforces one in-flight handshake and one live session at a time per identity
61
+
handshakeMu.Lock()
62
+
if _, ok := inFlightHandshakes[name]; ok {
63
+
handshakeMu.Unlock()
64
+
http.Error(w, "handshake already in progress", http.StatusConflict)
65
+
return
66
+
}
67
+
m.mu.Lock()
68
+
old, exists := m.sessions[name]
69
+
isLive := exists && old.live(m.cfg.ReconnectGrace)
70
+
m.mu.Unlock()
71
+
if isLive {
72
+
handshakeMu.Unlock()
73
+
http.Error(w, "session already active", http.StatusConflict)
74
+
return
75
+
}
76
+
inFlightHandshakes[name] = struct{}{}
77
+
handshakeMu.Unlock()
78
+
identityHandshakeHeld := true
79
+
80
+
defer func() {
81
+
if identityHandshakeHeld {
82
+
handshakeMu.Lock()
83
+
delete(inFlightHandshakes, name)
84
+
handshakeMu.Unlock()
85
+
}
86
+
}()
87
+
88
+
conn, err := upgrader.Upgrade(w, r, nil)
89
+
if err != nil {
90
+
m.l.Error("fleet ws upgrade failed", "err", err)
91
+
return
92
+
}
93
+
defer conn.Close()
94
+
95
+
if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
96
+
m.l.Error("failed to set pre-hello read deadline", "err", err)
97
+
return
98
+
}
99
+
100
+
stream := millproto.NewWSStream(conn)
101
+
enc := millproto.NewEncoder(stream)
102
+
dec := millproto.NewDecoder(stream)
103
+
104
+
hello, err := dec.Decode()
105
+
if err != nil {
106
+
m.l.Error("fleet read hello failed", "err", err)
107
+
return
108
+
}
109
+
h := hello.GetHello()
110
+
if h == nil {
111
+
m.l.Error("fleet first frame was not hello")
112
+
return
113
+
}
114
+
if h.GetProtocolVersion() != millproto.ProtocolVersion {
115
+
m.l.Error("fleet protocol version mismatch", "got", h.GetProtocolVersion(), "want", millproto.ProtocolVersion)
116
+
return
117
+
}
118
+
if h.GetEpoch() == "" {
119
+
m.l.Error("fleet hello missing epoch")
120
+
return
121
+
}
122
+
123
+
for _, l := range h.GetLabels() {
124
+
if !slices.Contains(authorizedLabels, l) {
125
+
m.l.Error("executor requested unauthorized label", "label", l, "authorized", authorizedLabels)
126
+
return
127
+
}
128
+
}
129
+
130
+
sess := newSession(name, h.GetEpoch(), authorizedLabels, enc, m.l)
131
+
sess.closeTransport = conn.Close
132
+
sess.labels = h.GetLabels()
133
+
134
+
resume, ok := m.attachSession(sess)
135
+
if !ok {
136
+
m.l.Warn("rejecting duplicate live executor session", "node", name)
137
+
return
138
+
}
139
+
m.l.Info("executor connected", "node", sess.nodeID, "arch", h.GetArch(), "labels", h.GetLabels(), "resume", resume)
140
+
141
+
if err := sess.send(&millproto.Message{Resume: &millv1.Resume{Epoch: h.GetEpoch(), AckSeqno: resume}}); err != nil {
142
+
m.l.Error("fleet send resume failed", "err", err)
143
+
m.detachSession(sess)
144
+
return
145
+
}
146
+
handshakeSlotHeld = false
147
+
<-handshakeSem
148
+
handshakeMu.Lock()
149
+
delete(inFlightHandshakes, name)
150
+
handshakeMu.Unlock()
151
+
identityHandshakeHeld = false
152
+
m.sessionReady(sess)
153
+
154
+
readTimeout := m.cfg.ReconnectGrace
155
+
if readTimeout <= 0 {
156
+
readTimeout = 45 * time.Second
157
+
}
158
+
liveDec := millproto.NewDecoder(&livenessReader{r: stream, conn: conn, readTimeout: readTimeout})
159
+
160
+
if err := sess.readLoop(m, liveDec); err != nil {
161
+
m.l.Debug("session read ended", "node", sess.nodeID, "err", err)
162
+
}
163
+
m.detachSession(sess)
164
+
}
165
+
166
+
// identity comes from the token hash. unknown or missing token fails closed
167
+
func (m *Mill) authenticate(r *http.Request) (string, []string, bool) {
168
+
const prefix = "Bearer "
169
+
h := r.Header.Get("Authorization")
170
+
if !strings.HasPrefix(h, prefix) {
171
+
return "", nil, false
172
+
}
173
+
token := strings.TrimPrefix(h, prefix)
174
+
if token == "" || m.db == nil {
175
+
return "", nil, false
176
+
}
177
+
name, labels, ok, err := m.db.ResolveExecutorToken(HashToken(token))
178
+
if err != nil {
179
+
m.l.Error("executor token lookup failed", "err", err)
180
+
return "", nil, false
181
+
}
182
+
return name, labels, ok
183
+
}
+280
spindle/mill/integration_test.go
+280
spindle/mill/integration_test.go
···
1
+
package mill
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"io"
7
+
"log/slog"
8
+
"net/http"
9
+
"net/http/httptest"
10
+
"path/filepath"
11
+
"strings"
12
+
"testing"
13
+
"time"
14
+
15
+
"tangled.org/core/api/tangled"
16
+
"tangled.org/core/notifier"
17
+
"tangled.org/core/spindle/config"
18
+
"tangled.org/core/spindle/db"
19
+
"tangled.org/core/spindle/engines/dummy"
20
+
"tangled.org/core/spindle/mill/executor"
21
+
"tangled.org/core/spindle/models"
22
+
)
23
+
24
+
func TestEndToEndDummyJob(t *testing.T) {
25
+
ctx, cancel := context.WithCancel(context.Background())
26
+
defer cancel()
27
+
l := slog.New(slog.NewTextHandler(io.Discard, nil))
28
+
29
+
millDir := t.TempDir()
30
+
bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db"))
31
+
if err != nil {
32
+
t.Fatalf("mill db: %v", err)
33
+
}
34
+
bn := notifier.New()
35
+
mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second})
36
+
mill.Attach(bdb, &bn)
37
+
if err := bdb.AddExecutorToken("exec-1", HashToken("test-token"), nil, nil); err != nil {
38
+
t.Fatalf("register executor token: %v", err)
39
+
}
40
+
41
+
srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn))
42
+
defer srv.Close()
43
+
wsURL := "ws" + strings.TrimPrefix(srv.URL, "http")
44
+
45
+
execDir := t.TempDir()
46
+
edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db"))
47
+
if err != nil {
48
+
t.Fatalf("exec db: %v", err)
49
+
}
50
+
en := notifier.New()
51
+
cfg := &config.Config{}
52
+
cfg.Server.Dev = true
53
+
cfg.Server.LogDir = execDir
54
+
cfg.Server.Hostname = "exec-1"
55
+
cfg.Mill.URL = wsURL
56
+
cfg.Mill.Seats = 2
57
+
cfg.Mill.SharedSecret = "test-token"
58
+
59
+
engines := map[string]models.Engine{"dummy": dummy.New(l)}
60
+
exec, err := executor.New(cfg, engines, edb, &en, l)
61
+
if err != nil {
62
+
t.Fatalf("executor.New: %v", err)
63
+
}
64
+
go exec.Connect(ctx)
65
+
66
+
be := NewEngine("dummy", mill)
67
+
twf := tangled.Pipeline_Workflow{
68
+
Name: "build",
69
+
Raw: "steps:\n - name: hello\n command: echo hi\n",
70
+
}
71
+
wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}})
72
+
if err != nil {
73
+
t.Fatalf("InitWorkflow: %v", err)
74
+
}
75
+
wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"}
76
+
77
+
placeCtx, placeCancel := context.WithTimeout(ctx, 10*time.Second)
78
+
defer placeCancel()
79
+
80
+
slot, err := mill.place(placeCtx, "dummy", wid, wf)
81
+
if err != nil {
82
+
t.Fatalf("place: %v", err)
83
+
}
84
+
defer slot.Release()
85
+
86
+
if err := mill.commitAndWait(placeCtx, wf, nil); err != nil {
87
+
t.Fatalf("commitAndWait: %v, want success", err)
88
+
}
89
+
90
+
if !waitForStatus(t, bdb, wid, "running") {
91
+
events, _ := bdb.GetEvents(0, 1000)
92
+
t.Logf("mill events after completion: %+v", events)
93
+
t.Fatal("mill never saw streamed running status")
94
+
}
95
+
}
96
+
97
+
func TestExecutorConfiguredLabelsAreStoredOnSession(t *testing.T) {
98
+
ctx, cancel := context.WithCancel(context.Background())
99
+
defer cancel()
100
+
l := slog.New(slog.NewTextHandler(io.Discard, nil))
101
+
102
+
millDir := t.TempDir()
103
+
bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db"))
104
+
if err != nil {
105
+
t.Fatalf("mill db: %v", err)
106
+
}
107
+
bn := notifier.New()
108
+
mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second})
109
+
mill.Attach(bdb, &bn)
110
+
if err := bdb.AddExecutorToken("exec-labels", HashToken("test-token"), nil, []string{"linux", "arm64", "gpu"}); err != nil {
111
+
t.Fatalf("register executor token: %v", err)
112
+
}
113
+
114
+
srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn))
115
+
defer srv.Close()
116
+
wsURL := "ws" + strings.TrimPrefix(srv.URL, "http")
117
+
118
+
execDir := t.TempDir()
119
+
edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db"))
120
+
if err != nil {
121
+
t.Fatalf("exec db: %v", err)
122
+
}
123
+
en := notifier.New()
124
+
cfg := &config.Config{}
125
+
cfg.Server.Dev = true
126
+
cfg.Server.LogDir = execDir
127
+
cfg.Server.Hostname = "exec-labels"
128
+
cfg.Mill.URL = wsURL
129
+
cfg.Mill.Seats = 2
130
+
cfg.Mill.SharedSecret = "test-token"
131
+
cfg.Mill.Labels = []string{"linux", "arm64", "gpu"}
132
+
133
+
engines := map[string]models.Engine{"dummy": dummy.New(l)}
134
+
exec, err := executor.New(cfg, engines, edb, &en, l)
135
+
if err != nil {
136
+
t.Fatalf("executor.New: %v", err)
137
+
}
138
+
go exec.Connect(ctx)
139
+
140
+
if !waitForSessionLabels(t, mill, "exec-labels", []string{"linux", "arm64", "gpu"}) {
141
+
t.Fatal("mill session never stored executor labels from hello")
142
+
}
143
+
}
144
+
145
+
func TestEndToEndDummyJobUsesRequiredLabelsAcrossExecutors(t *testing.T) {
146
+
ctx, cancel := context.WithCancel(context.Background())
147
+
defer cancel()
148
+
l := slog.New(slog.NewTextHandler(io.Discard, nil))
149
+
150
+
millDir := t.TempDir()
151
+
bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db"))
152
+
if err != nil {
153
+
t.Fatalf("mill db: %v", err)
154
+
}
155
+
bn := notifier.New()
156
+
mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second})
157
+
mill.Attach(bdb, &bn)
158
+
if err := bdb.AddExecutorToken("exec-x86", HashToken("token-x86"), nil, []string{"linux/amd64", "kvm"}); err != nil {
159
+
t.Fatalf("register x86 executor token: %v", err)
160
+
}
161
+
if err := bdb.AddExecutorToken("exec-arm", HashToken("token-arm"), nil, []string{"linux/arm64", "kvm"}); err != nil {
162
+
t.Fatalf("register arm executor token: %v", err)
163
+
}
164
+
165
+
srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn))
166
+
defer srv.Close()
167
+
wsURL := "ws" + strings.TrimPrefix(srv.URL, "http")
168
+
169
+
startExecutor := func(name, token string, labels []string) {
170
+
t.Helper()
171
+
execDir := t.TempDir()
172
+
edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db"))
173
+
if err != nil {
174
+
t.Fatalf("%s exec db: %v", name, err)
175
+
}
176
+
en := notifier.New()
177
+
cfg := &config.Config{}
178
+
cfg.Server.Dev = true
179
+
cfg.Server.LogDir = execDir
180
+
cfg.Server.Hostname = name
181
+
cfg.Mill.URL = wsURL
182
+
cfg.Mill.Seats = 1
183
+
cfg.Mill.SharedSecret = token
184
+
cfg.Mill.Labels = labels
185
+
186
+
engines := map[string]models.Engine{"dummy": dummy.New(l)}
187
+
exec, err := executor.New(cfg, engines, edb, &en, l)
188
+
if err != nil {
189
+
t.Fatalf("executor.New: %v", err)
190
+
}
191
+
go exec.Connect(ctx)
192
+
}
193
+
startExecutor("exec-x86", "token-x86", []string{"linux/amd64", "kvm"})
194
+
startExecutor("exec-arm", "token-arm", []string{"linux/arm64", "kvm"})
195
+
196
+
if !waitForSessionLabels(t, mill, "exec-x86", []string{"linux/amd64", "kvm"}) {
197
+
t.Fatal("x86 executor did not connect with labels")
198
+
}
199
+
if !waitForSessionLabels(t, mill, "exec-arm", []string{"linux/arm64", "kvm"}) {
200
+
t.Fatal("arm executor did not connect with labels")
201
+
}
202
+
203
+
be := NewEngine("dummy", mill)
204
+
twf := tangled.Pipeline_Workflow{
205
+
Name: "build-arm",
206
+
RunsOn: []string{"linux/arm64"},
207
+
Raw: "steps:\n - name: hello\n command: echo hi\n",
208
+
}
209
+
wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}})
210
+
if err != nil {
211
+
t.Fatalf("InitWorkflow: %v", err)
212
+
}
213
+
wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey-arm"}, Name: "build-arm"}
214
+
215
+
placeCtx, placeCancel := context.WithTimeout(ctx, 10*time.Second)
216
+
defer placeCancel()
217
+
218
+
slot, err := mill.place(placeCtx, "dummy", wid, wf)
219
+
if err != nil {
220
+
t.Fatalf("place: %v", err)
221
+
}
222
+
defer slot.Release()
223
+
224
+
lease := wf.Data.(*millWorkflowState).Lease
225
+
if lease == nil {
226
+
t.Fatal("place did not attach lease to workflow state")
227
+
}
228
+
if lease.nodeID != "exec-arm" {
229
+
t.Fatalf("placed on %q, want exec-arm", lease.nodeID)
230
+
}
231
+
232
+
if err := mill.commitAndWait(placeCtx, wf, nil); err != nil {
233
+
t.Fatalf("commitAndWait: %v, want success", err)
234
+
}
235
+
}
236
+
237
+
func waitForSessionLabels(t *testing.T, m *Mill, nodeID string, want []string) bool {
238
+
t.Helper()
239
+
deadline := time.Now().Add(3 * time.Second)
240
+
for time.Now().Before(deadline) {
241
+
m.mu.Lock()
242
+
sess := m.sessions[nodeID]
243
+
var got []string
244
+
if sess != nil {
245
+
got = append([]string(nil), sess.labels...)
246
+
}
247
+
m.mu.Unlock()
248
+
if sameStringMultiset(got, want) {
249
+
return true
250
+
}
251
+
time.Sleep(10 * time.Millisecond)
252
+
}
253
+
return false
254
+
}
255
+
256
+
func waitForStatus(t *testing.T, d *db.DB, wid models.WorkflowId, want string) bool {
257
+
t.Helper()
258
+
deadline := time.Now().Add(3 * time.Second)
259
+
aturi := string(wid.PipelineId.AtUri())
260
+
for time.Now().Before(deadline) {
261
+
evs, err := d.GetEvents(0, 1000)
262
+
if err != nil {
263
+
t.Fatalf("GetEvents: %v", err)
264
+
}
265
+
for _, ev := range evs {
266
+
if ev.Nsid != tangled.PipelineStatusNSID {
267
+
continue
268
+
}
269
+
var st tangled.PipelineStatus
270
+
if err := json.Unmarshal(ev.EventJson, &st); err != nil {
271
+
continue
272
+
}
273
+
if st.Pipeline == aturi && st.Workflow == wid.Name && st.Status == want {
274
+
return true
275
+
}
276
+
}
277
+
time.Sleep(50 * time.Millisecond)
278
+
}
279
+
return false
280
+
}
+214
spindle/mill/lease.go
+214
spindle/mill/lease.go
···
1
+
package mill
2
+
3
+
import (
4
+
"sync"
5
+
6
+
"tangled.org/core/spindle/models"
7
+
8
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
9
+
)
10
+
11
+
// the mill's view of a remote attempt
12
+
type leaseState int32
13
+
14
+
const (
15
+
// won a bid, executor is holding a seat, not yet committed
16
+
leaseReserved leaseState = iota
17
+
// CommitLease sent. the executor may already be running, but the mill
18
+
// may not have seen Committed yet
19
+
leaseCommitting
20
+
// CommitLease sent and acked. job is running on the executor
21
+
leaseRunning
22
+
// terminal result arrived or we gave up, no further action
23
+
leaseDone
24
+
)
25
+
26
+
type cancelAction int
27
+
28
+
const (
29
+
cancelNoop cancelAction = iota
30
+
cancelLocal
31
+
cancelRemote
32
+
)
33
+
34
+
// mill-side fencing token for one placed job
35
+
type RemoteLease struct {
36
+
id string
37
+
nodeID string
38
+
epoch string
39
+
engine string
40
+
wid models.WorkflowId // job this lease carries, set once placed
41
+
// restored after a mill restart. no RunStep waits on it, so terminals
42
+
// and death are authored directly. set before publication, never mutated
43
+
orphaned bool
44
+
claimed bool
45
+
cancelAcked bool
46
+
cleanedUp bool
47
+
cleanupRetry bool
48
+
49
+
mu sync.Mutex
50
+
state leaseState
51
+
cancel bool
52
+
reason string
53
+
released bool
54
+
terminal chan *millv1.AttemptResult // buffered(1), RunStep waits here
55
+
dead chan struct{} // closed when the executor is lost past grace
56
+
deadOnce sync.Once
57
+
58
+
finishMu sync.Mutex
59
+
60
+
// mill-side byte accounting for streamed logs. the executor caps its own
61
+
// outbox, but a malicious executor won't, so the mill enforces its own cap
62
+
logMu sync.Mutex
63
+
logBytes int64
64
+
droppedLogs int64
65
+
}
66
+
67
+
func newLease(id, nodeID, epoch, engine string) *RemoteLease {
68
+
return &RemoteLease{
69
+
id: id,
70
+
nodeID: nodeID,
71
+
epoch: epoch,
72
+
engine: engine,
73
+
state: leaseReserved,
74
+
terminal: make(chan *millv1.AttemptResult, 1),
75
+
dead: make(chan struct{}),
76
+
}
77
+
}
78
+
79
+
func (l *RemoteLease) setState(s leaseState) {
80
+
l.mu.Lock()
81
+
l.state = s
82
+
l.mu.Unlock()
83
+
}
84
+
85
+
func (l *RemoteLease) markCommitting() bool {
86
+
l.mu.Lock()
87
+
defer l.mu.Unlock()
88
+
if l.state == leaseDone {
89
+
return false
90
+
}
91
+
if l.state == leaseReserved {
92
+
l.state = leaseCommitting
93
+
}
94
+
return true
95
+
}
96
+
97
+
func (l *RemoteLease) markRunning() {
98
+
l.mu.Lock()
99
+
defer l.mu.Unlock()
100
+
if l.state != leaseDone {
101
+
l.state = leaseRunning
102
+
}
103
+
}
104
+
105
+
func (l *RemoteLease) getState() leaseState {
106
+
l.mu.Lock()
107
+
defer l.mu.Unlock()
108
+
return l.state
109
+
}
110
+
111
+
// only one caller gets to mark it done
112
+
func (l *RemoteLease) markDone() bool {
113
+
l.mu.Lock()
114
+
defer l.mu.Unlock()
115
+
if l.state == leaseDone {
116
+
return false
117
+
}
118
+
l.state = leaseDone
119
+
return true
120
+
}
121
+
122
+
func (l *RemoteLease) requestCancel(reason string) cancelAction {
123
+
l.mu.Lock()
124
+
defer l.mu.Unlock()
125
+
if l.state == leaseDone {
126
+
return cancelNoop
127
+
}
128
+
l.cancel = true
129
+
l.reason = reason
130
+
if l.state == leaseReserved {
131
+
return cancelLocal
132
+
}
133
+
return cancelRemote
134
+
}
135
+
136
+
func (l *RemoteLease) cancelRequested() (bool, string) {
137
+
l.mu.Lock()
138
+
defer l.mu.Unlock()
139
+
return l.cancel, l.reason
140
+
}
141
+
142
+
func (l *RemoteLease) releaseState() (leaseState, bool) {
143
+
l.mu.Lock()
144
+
defer l.mu.Unlock()
145
+
l.released = true
146
+
return l.state, l.cancel
147
+
}
148
+
149
+
func (l *RemoteLease) cleanupReady() bool {
150
+
l.mu.Lock()
151
+
defer l.mu.Unlock()
152
+
return l.released && l.state == leaseDone
153
+
}
154
+
155
+
func (l *RemoteLease) deliverCancelled(reason string) {
156
+
l.deliverTerminal(&millv1.AttemptResult{
157
+
Status: millv1.TerminalStatus_CANCELLED,
158
+
Error: reason,
159
+
})
160
+
}
161
+
162
+
// hands the terminal to a waiting RunStep without blocking. duplicates
163
+
// (eg. reconnect replays) just drop, the channel holds one and the lease
164
+
// is already done
165
+
func (l *RemoteLease) deliverTerminal(res *millv1.AttemptResult) {
166
+
if !l.markDone() {
167
+
return
168
+
}
169
+
select {
170
+
case l.terminal <- res:
171
+
default:
172
+
}
173
+
}
174
+
175
+
// executor running this lease is gone for good
176
+
func (l *RemoteLease) markDead() {
177
+
l.deadOnce.Do(func() { close(l.dead) })
178
+
}
179
+
180
+
// counts log lines against the cap. past it, lines drop and count, so a
181
+
// hostile executor can't fill the mill's disk. firstDrop lets the caller
182
+
// warn once. resets on restart, restored leases get a fresh budget
183
+
func (l *RemoteLease) accountLogLine(n, cap int64) (accepted, firstDrop bool) {
184
+
l.logMu.Lock()
185
+
defer l.logMu.Unlock()
186
+
if l.logBytes+n > cap {
187
+
l.droppedLogs++
188
+
return false, l.droppedLogs == 1
189
+
}
190
+
l.logBytes += n
191
+
return true, false
192
+
}
193
+
194
+
// mill's synthetic single step. real steps run on the executor and the
195
+
// mill never mirrors them
196
+
type remoteStep struct{}
197
+
198
+
func (remoteStep) Name() string { return "remote execution" }
199
+
func (remoteStep) Command() string { return "" }
200
+
func (remoteStep) Kind() models.StepKind { return models.StepKindSystem }
201
+
202
+
// what AcquireWorkflowSlot returns. Release unwinds placement
203
+
type millSlot struct {
204
+
fleet *Mill
205
+
lease *RemoteLease
206
+
once sync.Once
207
+
}
208
+
209
+
func (s *millSlot) Release() {
210
+
if s == nil {
211
+
return
212
+
}
213
+
s.once.Do(func() { s.fleet.releaseSlot(s) })
214
+
}
+1074
spindle/mill/mill.go
+1074
spindle/mill/mill.go
···
1
+
package mill
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"errors"
7
+
"fmt"
8
+
"log/slog"
9
+
"slices"
10
+
"strings"
11
+
"sync"
12
+
"time"
13
+
14
+
"tangled.org/core/notifier"
15
+
"tangled.org/core/spindle/db"
16
+
"tangled.org/core/spindle/engine"
17
+
"tangled.org/core/spindle/models"
18
+
"tangled.org/core/spindle/secrets"
19
+
"tangled.org/core/tid"
20
+
21
+
millproto "tangled.org/core/spindle/mill/proto"
22
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
23
+
)
24
+
25
+
const (
26
+
defaultReconnectGrace = 45 * time.Second
27
+
defaultJobTimeout = 24 * time.Hour
28
+
defaultBidTimeout = 5 * time.Second
29
+
defaultTopK = 3
30
+
defaultMaxPending = 100
31
+
defaultMaxLeaseLogBytes = 50 * 1024 * 1024
32
+
)
33
+
34
+
type Config struct {
35
+
MaxPending int
36
+
ReconnectGrace time.Duration
37
+
JobTimeout time.Duration
38
+
BidTimeout time.Duration
39
+
TopK int
40
+
LogDir string
41
+
CancelTimeout time.Duration
42
+
MaxLeaseLogBytes int64
43
+
}
44
+
45
+
type Mill struct {
46
+
l *slog.Logger
47
+
cfg Config
48
+
49
+
db *db.DB
50
+
n *notifier.Notifier
51
+
52
+
mu sync.Mutex
53
+
sessions map[string]*millSession
54
+
leases map[string]*RemoteLease
55
+
reservations map[string]*RemoteLease
56
+
nodeSeqno map[string]uint64
57
+
pending int
58
+
changeCh chan struct{} // closed and replaced to wake placement waiters
59
+
60
+
leaseSeq uint64
61
+
}
62
+
63
+
func New(l *slog.Logger, cfg Config) *Mill {
64
+
if cfg.ReconnectGrace <= 0 {
65
+
cfg.ReconnectGrace = defaultReconnectGrace
66
+
}
67
+
if cfg.JobTimeout <= 0 {
68
+
cfg.JobTimeout = defaultJobTimeout
69
+
}
70
+
if cfg.BidTimeout <= 0 {
71
+
cfg.BidTimeout = defaultBidTimeout
72
+
}
73
+
if cfg.TopK <= 0 {
74
+
cfg.TopK = defaultTopK
75
+
}
76
+
if cfg.MaxPending <= 0 {
77
+
cfg.MaxPending = defaultMaxPending
78
+
}
79
+
if cfg.CancelTimeout <= 0 {
80
+
cfg.CancelTimeout = 10 * time.Second
81
+
}
82
+
if cfg.MaxLeaseLogBytes <= 0 {
83
+
cfg.MaxLeaseLogBytes = defaultMaxLeaseLogBytes
84
+
}
85
+
return &Mill{
86
+
l: l,
87
+
cfg: cfg,
88
+
sessions: make(map[string]*millSession),
89
+
leases: make(map[string]*RemoteLease),
90
+
reservations: make(map[string]*RemoteLease),
91
+
nodeSeqno: make(map[string]uint64),
92
+
changeCh: make(chan struct{}),
93
+
}
94
+
}
95
+
96
+
func (m *Mill) Attach(d *db.DB, n *notifier.Notifier) {
97
+
m.mu.Lock()
98
+
m.db = d
99
+
m.n = n
100
+
m.mu.Unlock()
101
+
}
102
+
103
+
func (m *Mill) nextLeaseID() string {
104
+
m.mu.Lock()
105
+
m.leaseSeq++
106
+
seq := m.leaseSeq
107
+
m.mu.Unlock()
108
+
return fmt.Sprintf("%s-%d", tid.TID(), seq)
109
+
}
110
+
111
+
func (m *Mill) dropReservation(id string) {
112
+
m.mu.Lock()
113
+
delete(m.reservations, id)
114
+
m.mu.Unlock()
115
+
}
116
+
117
+
func (m *Mill) notifyChange() {
118
+
m.mu.Lock()
119
+
m.notifyChangeLocked()
120
+
m.mu.Unlock()
121
+
}
122
+
123
+
func (m *Mill) notifyChangeLocked() {
124
+
close(m.changeCh)
125
+
m.changeCh = make(chan struct{})
126
+
}
127
+
128
+
func (m *Mill) currentChangeCh() <-chan struct{} {
129
+
m.mu.Lock()
130
+
defer m.mu.Unlock()
131
+
return m.changeCh
132
+
}
133
+
134
+
func (m *Mill) attachSession(sess *millSession) (uint64, bool) {
135
+
m.mu.Lock()
136
+
defer m.mu.Unlock()
137
+
138
+
if old := m.sessions[sess.nodeID]; old != nil {
139
+
if old.live(m.cfg.ReconnectGrace) {
140
+
// a second live session for the same identity is a hijack
141
+
// attempt, reject it
142
+
return 0, false
143
+
}
144
+
if old.graceTimer != nil {
145
+
old.graceTimer.Stop()
146
+
}
147
+
old.close()
148
+
if old.disconnected {
149
+
m.l.Info("executor reconnected", "node", sess.nodeID)
150
+
} else {
151
+
m.l.Warn("replacing silent executor session", "node", sess.nodeID)
152
+
}
153
+
}
154
+
m.sessions[sess.nodeID] = sess
155
+
// wakes commit retries waiting out reconnect grace
156
+
m.notifyChangeLocked()
157
+
return m.nodeSeqno[sess.nodeID+"/"+sess.epoch], true
158
+
}
159
+
160
+
func (m *Mill) touchSession(sess *millSession) bool {
161
+
m.mu.Lock()
162
+
defer m.mu.Unlock()
163
+
if m.sessions[sess.nodeID] != sess || sess.disconnected {
164
+
return false
165
+
}
166
+
sess.lastSeen = time.Now()
167
+
return true
168
+
}
169
+
170
+
func (m *Mill) detachSession(sess *millSession) {
171
+
m.mu.Lock()
172
+
if m.sessions[sess.nodeID] != sess {
173
+
// already replaced by a reconnect
174
+
m.mu.Unlock()
175
+
sess.close()
176
+
return
177
+
}
178
+
sess.disconnected = true
179
+
sess.graceTimer = time.AfterFunc(m.cfg.ReconnectGrace, func() { m.failLeasesAfterGrace(sess) })
180
+
m.mu.Unlock()
181
+
182
+
sess.close()
183
+
m.l.Warn("executor session lost; entering reconnect grace", "node", sess.nodeID, "grace", m.cfg.ReconnectGrace)
184
+
m.notifyChange()
185
+
}
186
+
187
+
func (m *Mill) sessionReady(sess *millSession) {
188
+
for _, lease := range m.cancelledLeasesForNode(sess.nodeID) {
189
+
_, reason := lease.cancelRequested()
190
+
m.sendCancel(sess, lease, reason)
191
+
}
192
+
m.notifyChange()
193
+
}
194
+
195
+
func (m *Mill) cancelledLeasesForNode(nodeID string) []*RemoteLease {
196
+
m.mu.Lock()
197
+
var candidates []*RemoteLease
198
+
for _, lease := range m.leases {
199
+
if lease.nodeID == nodeID {
200
+
candidates = append(candidates, lease)
201
+
}
202
+
}
203
+
m.mu.Unlock()
204
+
205
+
var leases []*RemoteLease
206
+
for _, lease := range candidates {
207
+
if cancelled, _ := lease.cancelRequested(); cancelled && lease.getState() != leaseDone {
208
+
leases = append(leases, lease)
209
+
}
210
+
}
211
+
return leases
212
+
}
213
+
214
+
// reconnect grace ran out without the executor coming back. fails every
215
+
// lease the node held, and reschedules itself if any failure didn't stick
216
+
func (m *Mill) failLeasesAfterGrace(sess *millSession) {
217
+
m.mu.Lock()
218
+
if m.sessions[sess.nodeID] != sess || !sess.disconnected {
219
+
// reconnected in the meantime
220
+
m.mu.Unlock()
221
+
return
222
+
}
223
+
var dead []*RemoteLease
224
+
for _, lease := range m.leases {
225
+
if lease.nodeID == sess.nodeID {
226
+
dead = append(dead, lease)
227
+
}
228
+
}
229
+
m.mu.Unlock()
230
+
231
+
m.l.Warn("executor declared dead; failing its in-flight jobs", "node", sess.nodeID, "jobs", len(dead))
232
+
deadReason := "executor lost"
233
+
success := true
234
+
for _, lease := range dead {
235
+
switch {
236
+
case lease.orphaned:
237
+
// restored leases have no RunStep waiting, fail them straight
238
+
// into the event stream
239
+
if err := m.finishOrphan(lease, string(models.StatusKindFailed), &deadReason, nil); err != nil {
240
+
m.l.Error("finish orphan after executor loss failed, will retry", "lease", lease.id, "err", err)
241
+
success = false
242
+
}
243
+
default:
244
+
// live leases have a blocked RunStep, keep a pending cancellation
245
+
// as the terminal reason
246
+
status := string(models.StatusKindFailed)
247
+
reason := deadReason
248
+
if cancelled, cancelReason := lease.cancelRequested(); cancelled {
249
+
status = string(models.StatusKindCancelled)
250
+
reason = cancelReason
251
+
}
252
+
if err := m.finishLiveLease(lease, status, reason); err != nil {
253
+
m.l.Error("finish lease after executor loss failed, will retry", "lease", lease.id, "err", err)
254
+
success = false
255
+
}
256
+
}
257
+
}
258
+
259
+
m.mu.Lock()
260
+
if !success {
261
+
// something didn't finish cleanly, try again shortly. finished
262
+
// leases skip themselves on the next pass
263
+
sess.graceTimer = time.AfterFunc(5*time.Second, func() { m.failLeasesAfterGrace(sess) })
264
+
m.mu.Unlock()
265
+
return
266
+
}
267
+
268
+
// everything failed cleanly. forget the node and wake placement
269
+
if m.sessions[sess.nodeID] == sess {
270
+
delete(m.sessions, sess.nodeID)
271
+
}
272
+
m.mu.Unlock()
273
+
m.notifyChange()
274
+
m.sweepUnclaimedOrphans()
275
+
}
276
+
277
+
func (m *Mill) place(ctx context.Context, engineName string, wid models.WorkflowId, wf *models.Workflow) (engine.WorkflowSlot, error) {
278
+
m.mu.Lock()
279
+
if m.cfg.MaxPending > 0 && m.pending >= m.cfg.MaxPending {
280
+
max := m.cfg.MaxPending
281
+
cur := m.pending
282
+
m.mu.Unlock()
283
+
return nil, fmt.Errorf("%w: mill has %d pending jobs (max %d)", engine.ErrNoWorkflowSlots, cur, max)
284
+
}
285
+
m.pending++
286
+
m.mu.Unlock()
287
+
defer func() {
288
+
m.mu.Lock()
289
+
m.pending--
290
+
m.mu.Unlock()
291
+
}()
292
+
293
+
for {
294
+
if err := ctx.Err(); err != nil {
295
+
return nil, err
296
+
}
297
+
298
+
// grab the channel before bidding. a change mid-bid closes it, so
299
+
// the wait below re-bids right away
300
+
ch := m.currentChangeCh()
301
+
302
+
lease, err := m.bid(ctx, engineName, wid, wf)
303
+
if err != nil {
304
+
return nil, err
305
+
}
306
+
if lease != nil {
307
+
lease.wid = wid
308
+
if err := m.persistLease(lease, leaseRowReserved); err != nil {
309
+
m.releaseRemote(lease)
310
+
m.mu.Lock()
311
+
delete(m.reservations, lease.id)
312
+
m.mu.Unlock()
313
+
return nil, fmt.Errorf("persist reserved mill lease: %w", err)
314
+
}
315
+
m.mu.Lock()
316
+
delete(m.reservations, lease.id)
317
+
m.leases[lease.id] = lease
318
+
if st, ok := wf.Data.(*millWorkflowState); ok && st != nil {
319
+
st.Lease = lease
320
+
}
321
+
m.mu.Unlock()
322
+
return &millSlot{fleet: m, lease: lease}, nil
323
+
}
324
+
325
+
// no executor available. wait for a change or ctx
326
+
select {
327
+
case <-ctx.Done():
328
+
return nil, ctx.Err()
329
+
case <-ch:
330
+
}
331
+
}
332
+
}
333
+
334
+
func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId, wf *models.Workflow) (*RemoteLease, error) {
335
+
rawPipeline, rawWorkflow, err := marshalJob(wf)
336
+
if err != nil {
337
+
return nil, err
338
+
}
339
+
340
+
requiredLabels := requiredLabels(wf)
341
+
candidates := m.rankCandidates(engineName, requiredLabels)
342
+
if len(candidates) == 0 {
343
+
return nil, nil
344
+
}
345
+
346
+
type bidResult struct {
347
+
sess *millSession
348
+
lease *RemoteLease
349
+
rank int
350
+
incompatible bool
351
+
reason string
352
+
}
353
+
limit := m.cfg.TopK
354
+
if limit <= 0 {
355
+
limit = len(candidates)
356
+
}
357
+
if len(candidates) > limit {
358
+
candidates = candidates[:limit]
359
+
}
360
+
results := make(chan bidResult, limit)
361
+
ask := func(rank int, sess *millSession) {
362
+
bidCtx, cancel := context.WithTimeout(ctx, m.cfg.BidTimeout)
363
+
defer cancel()
364
+
leaseID := m.nextLeaseID()
365
+
lease := newLease(leaseID, sess.nodeID, sess.epoch, engineName)
366
+
m.mu.Lock()
367
+
m.reservations[leaseID] = lease
368
+
m.mu.Unlock()
369
+
msg := &millproto.Message{ReserveSeat: &millv1.ReserveSeat{
370
+
LeaseId: leaseID,
371
+
TargetEngine: engineName,
372
+
RawPipelineJson: rawPipeline,
373
+
RawWorkflowJson: rawWorkflow,
374
+
Knot: wid.Knot,
375
+
Rkey: wid.Rkey,
376
+
TtlSeconds: uint32(m.cfg.ReconnectGrace / time.Second),
377
+
}}
378
+
resp, err := sess.request(bidCtx, leaseID, msg)
379
+
if err != nil {
380
+
m.dropReservation(leaseID)
381
+
results <- bidResult{}
382
+
return
383
+
}
384
+
rr := resp.GetReserveResult()
385
+
if rr == nil {
386
+
m.dropReservation(leaseID)
387
+
results <- bidResult{}
388
+
return
389
+
}
390
+
if !rr.GetAccepted() {
391
+
m.dropReservation(leaseID)
392
+
if rr.GetRejectClass() == millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE {
393
+
results <- bidResult{sess: sess, rank: rank, incompatible: true, reason: rr.GetRejectReason()}
394
+
return
395
+
}
396
+
results <- bidResult{}
397
+
return
398
+
}
399
+
results <- bidResult{sess: sess, lease: lease, rank: rank}
400
+
}
401
+
next := 0
402
+
inFlight := 0
403
+
for next < len(candidates) && inFlight < limit {
404
+
inFlight++
405
+
go ask(next, candidates[next])
406
+
next++
407
+
}
408
+
409
+
var winner *bidResult
410
+
var losers []*RemoteLease
411
+
var incompatible []string
412
+
// any soft reject (transient or timeout) means the fleet was just
413
+
// busy, so an all-incompatible outcome isn't a hard placement error
414
+
softReject := false
415
+
for inFlight > 0 {
416
+
r := <-results
417
+
inFlight--
418
+
// incompatible rejects get reported to the user, any other failure
419
+
// just means the fleet is busy
420
+
if r.incompatible {
421
+
if r.reason != "" {
422
+
incompatible = append(incompatible, r.reason)
423
+
}
424
+
} else if r.lease == nil {
425
+
softReject = true
426
+
}
427
+
// a failed bid means ask the next candidate, unless someone won
428
+
if r.lease == nil {
429
+
for winner == nil && next < len(candidates) && inFlight < limit {
430
+
inFlight++
431
+
go ask(next, candidates[next])
432
+
next++
433
+
}
434
+
continue
435
+
}
436
+
// if this bid is worse than the winner it goes to the losers pile
437
+
if winner != nil && r.rank >= winner.rank {
438
+
losers = append(losers, r.lease)
439
+
continue
440
+
}
441
+
// otherwise it's the new best and the old winner joins the losers
442
+
if winner != nil {
443
+
losers = append(losers, winner.lease)
444
+
}
445
+
winner = &r
446
+
}
447
+
448
+
// let the losers go so they free their seats right away
449
+
for _, l := range losers {
450
+
m.dropReservation(l.id)
451
+
m.releaseRemote(l)
452
+
}
453
+
454
+
if winner == nil {
455
+
if len(incompatible) > 0 && !softReject {
456
+
return nil, fmt.Errorf("no compatible executor for %s: %s", engineName, strings.Join(incompatible, "; "))
457
+
}
458
+
return nil, nil
459
+
}
460
+
return winner.lease, nil
461
+
}
462
+
463
+
// ranks nodes that are least busy first. if a resource is used a lot
464
+
// then that node will lose to one that is more even across the board.
465
+
func (m *Mill) rankCandidates(engineName string, requiredLabels []string) []*millSession {
466
+
m.mu.Lock()
467
+
defer m.mu.Unlock()
468
+
469
+
type ranked struct {
470
+
sess *millSession
471
+
worst float64
472
+
sum float64
473
+
}
474
+
var rs []ranked
475
+
for _, s := range m.sessions {
476
+
// only live, reporting sessions can take work
477
+
if s.disconnected {
478
+
continue
479
+
}
480
+
if s.snapshot == nil {
481
+
continue
482
+
}
483
+
// the engine has to exist and have room right now
484
+
ea, ok := s.snapshot.GetEngines()[engineName]
485
+
if !ok || !ea.GetAvailable() {
486
+
continue
487
+
}
488
+
// and satisfy the wf's label requirements
489
+
if !hasLabels(s.labels, requiredLabels) {
490
+
continue
491
+
}
492
+
worst, sum := loadScore(ea.GetLoad())
493
+
rs = append(rs, ranked{sess: s, worst: worst, sum: sum})
494
+
}
495
+
slices.SortStableFunc(rs, func(a, b ranked) int {
496
+
if a.worst < b.worst {
497
+
return -1
498
+
}
499
+
if a.worst > b.worst {
500
+
return 1
501
+
}
502
+
if a.sum < b.sum {
503
+
return -1
504
+
}
505
+
if a.sum > b.sum {
506
+
return 1
507
+
}
508
+
return 0
509
+
})
510
+
511
+
out := make([]*millSession, len(rs))
512
+
for i := range rs {
513
+
out[i] = rs[i].sess
514
+
}
515
+
return out
516
+
}
517
+
518
+
func loadScore(load map[string]float64) (worst, sum float64) {
519
+
for _, v := range load {
520
+
if v > worst {
521
+
worst = v
522
+
}
523
+
sum += v
524
+
}
525
+
return worst, sum
526
+
}
527
+
528
+
func requiredLabels(wf *models.Workflow) []string {
529
+
st, ok := wf.Data.(*millWorkflowState)
530
+
if !ok || st == nil {
531
+
return nil
532
+
}
533
+
return st.RawWorkflow.RunsOn
534
+
}
535
+
536
+
func hasLabels(labels []string, required []string) bool {
537
+
for _, want := range required {
538
+
if !slices.Contains(labels, want) {
539
+
return false
540
+
}
541
+
}
542
+
return true
543
+
}
544
+
545
+
func (m *Mill) commitAndWait(ctx context.Context, wf *models.Workflow, unlocked []secrets.UnlockedSecret) error {
546
+
st, ok := wf.Data.(*millWorkflowState)
547
+
if !ok || st == nil || st.Lease == nil {
548
+
return fmt.Errorf("mill workflow state missing lease")
549
+
}
550
+
lease := st.Lease
551
+
552
+
pbSecrets := make([]*millv1.Secret, len(unlocked))
553
+
for i, s := range unlocked {
554
+
pbSecrets[i] = &millv1.Secret{Key: s.Key, Value: s.Value}
555
+
}
556
+
557
+
commit := &millproto.Message{CommitLease: &millv1.CommitLease{
558
+
LeaseId: lease.id,
559
+
Secrets: pbSecrets,
560
+
}}
561
+
562
+
// commit retries ride reconnects, a reservation outlives one
563
+
// disconnect. lost session or slow executor just means wait and retry,
564
+
// only job timeout or a dead lease stops the loop
565
+
for {
566
+
if res, ok := pollTerminal(lease); ok {
567
+
return terminalError(res.Status)
568
+
}
569
+
if !lease.markCommitting() {
570
+
return engine.ErrWorkflowFailed
571
+
}
572
+
573
+
sess := m.sessionForNode(lease.nodeID)
574
+
if sess == nil {
575
+
if done, err := m.waitCommitRetry(ctx, lease); done || err != nil {
576
+
return err
577
+
}
578
+
continue
579
+
}
580
+
581
+
reqCtx, cancel := context.WithTimeout(ctx, m.cfg.BidTimeout)
582
+
resp, err := sess.request(reqCtx, lease.id, commit)
583
+
cancel()
584
+
if err != nil {
585
+
switch {
586
+
case errors.Is(err, errSessionClosed):
587
+
// session died mid-request. wait out the grace, then retry on
588
+
// the new one
589
+
if done, err := m.waitCommitRetry(ctx, lease); done || err != nil {
590
+
return err
591
+
}
592
+
continue
593
+
case errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil:
594
+
// executor didn't answer in time, but its seat is still held so
595
+
// retrying is safe
596
+
continue
597
+
case errors.Is(err, context.DeadlineExceeded):
598
+
// the job ctx itself ran out, a real timeout
599
+
return engine.ErrTimedOut
600
+
case errors.Is(err, context.Canceled):
601
+
return err
602
+
default:
603
+
m.l.Warn("commit lease send failed; waiting for reconnect", "lease", lease.id, "node", lease.nodeID, "err", err)
604
+
if done, err := m.waitCommitRetry(ctx, lease); done || err != nil {
605
+
return err
606
+
}
607
+
continue
608
+
}
609
+
}
610
+
if resp.GetCommitted() == nil {
611
+
return engine.ErrWorkflowFailed
612
+
}
613
+
lease.markRunning()
614
+
if err := m.persistLease(lease, leaseRowRunning); err != nil {
615
+
m.l.Error("persist running mill lease", "lease", lease.id, "err", err)
616
+
}
617
+
if cancelled, reason := lease.cancelRequested(); cancelled {
618
+
m.sendCancel(sess, lease, reason)
619
+
}
620
+
break
621
+
}
622
+
623
+
select {
624
+
case res := <-lease.terminal:
625
+
return terminalError(res.Status)
626
+
case <-lease.dead:
627
+
m.l.Warn("executor lost while running job", "lease", lease.id, "node", lease.nodeID)
628
+
return engine.ErrWorkflowFailed
629
+
case <-ctx.Done():
630
+
if ctx.Err() == context.DeadlineExceeded {
631
+
return engine.ErrTimedOut
632
+
}
633
+
return ctx.Err()
634
+
}
635
+
}
636
+
637
+
func (m *Mill) waitCommitRetry(ctx context.Context, lease *RemoteLease) (bool, error) {
638
+
// grab the channel before checking for a live session again
639
+
// a reconnect will still close the channel if it happens in between
640
+
ch := m.currentChangeCh()
641
+
if m.sessionForNode(lease.nodeID) != nil {
642
+
return false, nil
643
+
}
644
+
select {
645
+
case res := <-lease.terminal:
646
+
return true, terminalError(res.Status)
647
+
case <-lease.dead:
648
+
return true, engine.ErrWorkflowFailed
649
+
case <-ctx.Done():
650
+
if ctx.Err() == context.DeadlineExceeded {
651
+
return true, engine.ErrTimedOut
652
+
}
653
+
return true, ctx.Err()
654
+
case <-ch:
655
+
return false, nil
656
+
}
657
+
}
658
+
659
+
func terminalError(status millv1.TerminalStatus) error {
660
+
switch status {
661
+
case millv1.TerminalStatus_SUCCESS:
662
+
return nil
663
+
case millv1.TerminalStatus_TIMEOUT:
664
+
return engine.ErrTimedOut
665
+
case millv1.TerminalStatus_CANCELLED:
666
+
return engine.ErrCancelled
667
+
default:
668
+
return engine.ErrWorkflowFailed
669
+
}
670
+
}
671
+
672
+
func pollTerminal(lease *RemoteLease) (*millv1.AttemptResult, bool) {
673
+
select {
674
+
case res := <-lease.terminal:
675
+
return res, true
676
+
default:
677
+
return nil, false
678
+
}
679
+
}
680
+
681
+
func (m *Mill) destroy(wid models.WorkflowId) {
682
+
m.mu.Lock()
683
+
var lease *RemoteLease
684
+
for _, l := range m.leases {
685
+
if l.wid == wid {
686
+
lease = l
687
+
break
688
+
}
689
+
}
690
+
m.mu.Unlock()
691
+
if lease == nil {
692
+
return
693
+
}
694
+
reason := "workflow destroyed"
695
+
switch lease.requestCancel(reason) {
696
+
case cancelLocal:
697
+
if sess := m.sessionForNode(lease.nodeID); sess != nil {
698
+
_ = sess.send(&millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: lease.id}})
699
+
}
700
+
lease.deliverCancelled(reason)
701
+
case cancelRemote:
702
+
if sess := m.sessionForNode(lease.nodeID); sess != nil {
703
+
m.sendCancel(sess, lease, reason)
704
+
}
705
+
}
706
+
}
707
+
708
+
func (m *Mill) releaseSlot(s *millSlot) {
709
+
lease := s.lease
710
+
state, cancelled := lease.releaseState()
711
+
if state == leaseReserved {
712
+
m.releaseRemote(lease)
713
+
} else if cancelled && state != leaseDone {
714
+
return
715
+
}
716
+
if err := m.cleanupLease(lease); err != nil {
717
+
m.l.Error("releaseSlot cleanupLease failed", "lease", lease.id, "err", err)
718
+
}
719
+
}
720
+
func (m *Mill) cleanupLease(lease *RemoteLease) error {
721
+
lease.finishMu.Lock()
722
+
defer lease.finishMu.Unlock()
723
+
return m.cleanupLeaseLocked(lease)
724
+
}
725
+
726
+
func (m *Mill) cleanupLeaseLocked(lease *RemoteLease) error {
727
+
if lease.cleanedUp {
728
+
return nil
729
+
}
730
+
if m.db != nil {
731
+
logPath := models.LogFilePath(m.cfg.LogDir, lease.wid)
732
+
if err := m.flushCanonicalLog(logPath); err != nil {
733
+
m.scheduleCleanupLocked(lease)
734
+
return err
735
+
}
736
+
if err := m.db.DeleteMillLease(lease.id); err != nil {
737
+
m.scheduleCleanupLocked(lease)
738
+
return err
739
+
}
740
+
}
741
+
m.mu.Lock()
742
+
delete(m.leases, lease.id)
743
+
m.mu.Unlock()
744
+
lease.cleanedUp = true
745
+
m.notifyChange()
746
+
return nil
747
+
}
748
+
749
+
func (m *Mill) scheduleCleanupLocked(lease *RemoteLease) {
750
+
if lease.cleanedUp || lease.cleanupRetry {
751
+
return
752
+
}
753
+
lease.cleanupRetry = true
754
+
time.AfterFunc(5*time.Second, func() {
755
+
lease.finishMu.Lock()
756
+
lease.cleanupRetry = false
757
+
err := m.cleanupLeaseLocked(lease)
758
+
lease.finishMu.Unlock()
759
+
if err != nil {
760
+
m.l.Error("retry lease cleanup failed", "lease", lease.id, "err", err)
761
+
}
762
+
})
763
+
}
764
+
765
+
func (m *Mill) releaseRemote(lease *RemoteLease) {
766
+
lease.setState(leaseDone)
767
+
if sess := m.sessionForNode(lease.nodeID); sess != nil {
768
+
_ = sess.send(&millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: lease.id}})
769
+
}
770
+
}
771
+
772
+
func (m *Mill) sendCancel(sess *millSession, lease *RemoteLease, reason string) {
773
+
if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{
774
+
LeaseId: lease.id,
775
+
Reason: reason,
776
+
}}); err != nil {
777
+
return
778
+
}
779
+
time.AfterFunc(m.cfg.CancelTimeout, func() { m.checkCancelDeadline(lease) })
780
+
}
781
+
782
+
func (m *Mill) sessionForNode(nodeID string) *millSession {
783
+
m.mu.Lock()
784
+
defer m.mu.Unlock()
785
+
sess := m.sessions[nodeID]
786
+
if sess == nil || sess.disconnected {
787
+
return nil
788
+
}
789
+
return sess
790
+
}
791
+
792
+
func (m *Mill) onSnapshot(sess *millSession, snap *millv1.NodeSnapshot) error {
793
+
m.mu.Lock()
794
+
if sess.snapshot != nil && snap.Seqno <= sess.snapshot.Seqno {
795
+
m.mu.Unlock()
796
+
return fmt.Errorf("protocol error: snapshot seqno regression. Got %d, last seen %d", snap.Seqno, sess.snapshot.Seqno)
797
+
}
798
+
sess.snapshot = snap
799
+
m.mu.Unlock()
800
+
801
+
if err := m.reconcileLeases(sess, snap.GetActiveLeaseIds()); err != nil {
802
+
return err
803
+
}
804
+
m.mu.Lock()
805
+
for _, id := range snap.GetActiveLeaseIds() {
806
+
if lease := m.leases[id]; lease != nil && lease.nodeID == sess.nodeID && lease.epoch == sess.epoch {
807
+
lease.claimed = true
808
+
}
809
+
}
810
+
m.mu.Unlock()
811
+
m.notifyChange()
812
+
return nil
813
+
}
814
+
815
+
func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error {
816
+
if batch == nil {
817
+
return nil
818
+
}
819
+
820
+
if batch.Epoch != sess.epoch {
821
+
return fmt.Errorf("protocol error: batch epoch %q does not match session %q", batch.Epoch, sess.epoch)
822
+
}
823
+
824
+
m.mu.Lock()
825
+
currentKey := sess.nodeID + "/" + sess.epoch
826
+
current := m.nodeSeqno[currentKey]
827
+
m.mu.Unlock()
828
+
829
+
expected := current + 1
830
+
var newEntries []*millv1.Event
831
+
for _, entry := range batch.Events {
832
+
// reconnect replays old seqnos, drop those
833
+
if entry.Seqno <= current {
834
+
continue
835
+
}
836
+
// gap means executor lost rows. so dont apply a partial batch
837
+
if entry.Seqno != expected {
838
+
return fmt.Errorf("protocol error: gap in stream seqnos. Expected %d, got %d", expected, entry.Seqno)
839
+
}
840
+
newEntries = append(newEntries, entry)
841
+
expected++
842
+
}
843
+
844
+
// all replays, still ack so the executor can trim its outbox
845
+
if len(newEntries) == 0 {
846
+
return m.sendAck(sess, current)
847
+
}
848
+
849
+
type pendingTerminal struct {
850
+
lease *RemoteLease
851
+
ar *millv1.AttemptResult
852
+
}
853
+
var pendingTerminals []pendingTerminal
854
+
finishedInBatch := make(map[string]struct{})
855
+
856
+
var highestSeqno uint64 = current
857
+
858
+
applyFunc := func(tx *db.EventBatchTx) error {
859
+
for _, entry := range newEntries {
860
+
m.mu.Lock()
861
+
lease := m.leases[entry.LeaseId]
862
+
m.mu.Unlock()
863
+
864
+
// events for leases this node doesn't own are skipped but still
865
+
// count as processed
866
+
if lease == nil || lease.nodeID != sess.nodeID {
867
+
highestSeqno = entry.Seqno
868
+
continue
869
+
}
870
+
871
+
// events arriving for a different epoch are invalid (different session)
872
+
if lease.epoch != "" && lease.epoch != sess.epoch {
873
+
return fmt.Errorf("protocol error: lease %q epoch %q does not match session %q", lease.id, lease.epoch, sess.epoch)
874
+
}
875
+
876
+
lease.mu.Lock()
877
+
state := lease.state
878
+
lease.mu.Unlock()
879
+
880
+
// done leases can replay terminals on reconnect, skip them
881
+
if state == leaseDone {
882
+
highestSeqno = entry.Seqno
883
+
continue
884
+
}
885
+
886
+
if _, finished := finishedInBatch[lease.id]; finished {
887
+
return fmt.Errorf("protocol error: stream entry follows terminal for lease %q", lease.id)
888
+
}
889
+
890
+
switch {
891
+
case entry.GetStatusEvent() != nil:
892
+
ev := entry.GetStatusEvent()
893
+
statusStr := string(models.StatusKindRunning)
894
+
var errMsg *string
895
+
if e := ev.GetError(); e != "" {
896
+
errMsg = &e
897
+
}
898
+
var exitCode *int64
899
+
if c := ev.GetExitCode(); c != 0 {
900
+
exitCode = &c
901
+
}
902
+
pipelineAtUri := string(lease.wid.PipelineId.AtUri())
903
+
if tx != nil {
904
+
if err := tx.InsertStatusEvent(pipelineAtUri, lease.wid.Name, statusStr, errMsg, exitCode); err != nil {
905
+
return err
906
+
}
907
+
}
908
+
909
+
case entry.GetLogLine() != nil:
910
+
raw := entry.GetLogLine().GetRawJson()
911
+
line := make([]byte, len(raw)+1)
912
+
copy(line, raw)
913
+
line[len(raw)] = '\n'
914
+
accepted, firstDrop := lease.accountLogLine(int64(len(line)), m.cfg.MaxLeaseLogBytes)
915
+
if !accepted {
916
+
if firstDrop {
917
+
m.l.Warn("mill-side log cap reached, dropping further lines", "lease", lease.id, "cap", m.cfg.MaxLeaseLogBytes)
918
+
}
919
+
highestSeqno = entry.Seqno
920
+
continue
921
+
}
922
+
logPath := models.LogFilePath(m.cfg.LogDir, lease.wid)
923
+
if tx != nil {
924
+
if err := tx.AddCanonicalLog(sess.nodeID, sess.epoch, entry.Seqno, logPath, line); err != nil {
925
+
return err
926
+
}
927
+
}
928
+
929
+
case entry.GetAttemptResult() != nil:
930
+
ar := entry.GetAttemptResult()
931
+
statusStr := "success"
932
+
switch ar.Status {
933
+
case millv1.TerminalStatus_SUCCESS:
934
+
statusStr = "success"
935
+
case millv1.TerminalStatus_FAILED:
936
+
statusStr = "failed"
937
+
case millv1.TerminalStatus_TIMEOUT:
938
+
statusStr = "timeout"
939
+
case millv1.TerminalStatus_CANCELLED:
940
+
statusStr = "cancelled"
941
+
default:
942
+
return fmt.Errorf("protocol error: unsupported terminal status %v", ar.Status)
943
+
}
944
+
var errMsg *string
945
+
if e := ar.GetError(); e != "" {
946
+
errMsg = &e
947
+
}
948
+
var exitCode *int64
949
+
if c := ar.GetExitCode(); c != 0 {
950
+
exitCode = &c
951
+
}
952
+
finishedInBatch[lease.id] = struct{}{}
953
+
pipelineAtUri := string(lease.wid.PipelineId.AtUri())
954
+
if tx != nil {
955
+
if err := tx.InsertStatusEvent(pipelineAtUri, lease.wid.Name, statusStr, errMsg, exitCode); err != nil {
956
+
return err
957
+
}
958
+
if err := tx.DeleteLease(lease.id); err != nil {
959
+
return err
960
+
}
961
+
}
962
+
pendingTerminals = append(pendingTerminals, pendingTerminal{
963
+
lease: lease,
964
+
ar: ar,
965
+
})
966
+
}
967
+
968
+
highestSeqno = entry.Seqno
969
+
}
970
+
971
+
if tx != nil {
972
+
return tx.AdvanceCursor(sess.nodeID, sess.epoch, highestSeqno)
973
+
}
974
+
return nil
975
+
}
976
+
977
+
var err error
978
+
if m.db != nil {
979
+
err = m.db.ApplyEventBatch(m.n, applyFunc)
980
+
} else {
981
+
err = applyFunc(nil)
982
+
}
983
+
984
+
if err != nil {
985
+
return err
986
+
}
987
+
988
+
m.mu.Lock()
989
+
if highestSeqno > m.nodeSeqno[currentKey] {
990
+
m.nodeSeqno[currentKey] = highestSeqno
991
+
}
992
+
m.mu.Unlock()
993
+
994
+
for _, pt := range pendingTerminals {
995
+
// orphans have no waiting RunStep, just mark and clean up
996
+
if pt.lease.orphaned {
997
+
pt.lease.markDone()
998
+
_ = m.cleanupLease(pt.lease)
999
+
continue
1000
+
}
1001
+
pt.lease.deliverTerminal(pt.ar)
1002
+
if pt.lease.cleanupReady() {
1003
+
_ = m.cleanupLease(pt.lease)
1004
+
}
1005
+
}
1006
+
1007
+
return m.sendAck(sess, highestSeqno)
1008
+
}
1009
+
1010
+
func (m *Mill) sendAck(sess *millSession, seqno uint64) error {
1011
+
msg := &millproto.Message{Ack: &millv1.Ack{
1012
+
Epoch: sess.epoch,
1013
+
UpToSeqno: seqno,
1014
+
}}
1015
+
if err := sess.send(msg); err != nil {
1016
+
return fmt.Errorf("send ack message: %w", err)
1017
+
}
1018
+
return nil
1019
+
}
1020
+
1021
+
func (m *Mill) onCancelAck(sess *millSession, ca *millv1.CancelAck) {
1022
+
m.mu.Lock()
1023
+
lease := m.leases[ca.GetLeaseId()]
1024
+
m.mu.Unlock()
1025
+
if lease == nil || lease.nodeID != sess.nodeID {
1026
+
return
1027
+
}
1028
+
lease.mu.Lock()
1029
+
lease.cancelAcked = true
1030
+
lease.mu.Unlock()
1031
+
}
1032
+
1033
+
func (m *Mill) checkCancelDeadline(lease *RemoteLease) {
1034
+
lease.mu.Lock()
1035
+
isDone := (lease.state == leaseDone)
1036
+
isAcked := lease.cancelAcked
1037
+
lease.mu.Unlock()
1038
+
1039
+
if isDone && isAcked {
1040
+
return
1041
+
}
1042
+
1043
+
m.l.Warn("node failed to comply with cancel request within deadline, quarantining", "node", lease.nodeID, "lease", lease.id, "done", isDone, "acked", isAcked)
1044
+
1045
+
reason := fmt.Sprintf("cancel noncompliance for lease %s (done: %t, acked: %t)", lease.id, isDone, isAcked)
1046
+
if m.db != nil {
1047
+
if err := m.db.QuarantineExecutor(lease.nodeID, reason); err != nil {
1048
+
m.l.Error("failed to quarantine executor", "node", lease.nodeID, "err", err)
1049
+
}
1050
+
}
1051
+
1052
+
m.mu.Lock()
1053
+
sess := m.sessions[lease.nodeID]
1054
+
m.mu.Unlock()
1055
+
if sess != nil {
1056
+
sess.close()
1057
+
}
1058
+
}
1059
+
1060
+
func marshalJob(wf *models.Workflow) (pipeline string, workflow string, err error) {
1061
+
st, ok := wf.Data.(*millWorkflowState)
1062
+
if !ok || st == nil {
1063
+
return "", "", fmt.Errorf("mill workflow state missing")
1064
+
}
1065
+
p, err := json.Marshal(st.RawPipeline)
1066
+
if err != nil {
1067
+
return "", "", fmt.Errorf("marshal pipeline: %w", err)
1068
+
}
1069
+
w, err := json.Marshal(st.RawWorkflow)
1070
+
if err != nil {
1071
+
return "", "", fmt.Errorf("marshal workflow: %w", err)
1072
+
}
1073
+
return string(p), string(w), nil
1074
+
}
+874
spindle/mill/mill_test.go
+874
spindle/mill/mill_test.go
···
1
+
package mill
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"io"
7
+
"log/slog"
8
+
"sync"
9
+
"testing"
10
+
"time"
11
+
12
+
"tangled.org/core/api/tangled"
13
+
"tangled.org/core/spindle/engine"
14
+
"tangled.org/core/spindle/models"
15
+
16
+
millproto "tangled.org/core/spindle/mill/proto"
17
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
18
+
)
19
+
20
+
type scriptedEncoder func(*millproto.Message) error
21
+
22
+
func (e scriptedEncoder) Encode(msg *millproto.Message) error { return e(msg) }
23
+
24
+
func testWorkflow(name string) *models.Workflow {
25
+
return &models.Workflow{
26
+
Name: name,
27
+
Environment: map[string]string{},
28
+
Steps: []models.Step{remoteStep{}},
29
+
Data: &millWorkflowState{
30
+
TargetEngine: "dummy",
31
+
RawWorkflow: tangled.Pipeline_Workflow{Name: name},
32
+
RawPipeline: tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}},
33
+
},
34
+
}
35
+
}
36
+
37
+
func testWorkflowWithRunsOn(name string, runsOn []string) *models.Workflow {
38
+
wf := testWorkflow(name)
39
+
wf.Data.(*millWorkflowState).RawWorkflow.RunsOn = runsOn
40
+
return wf
41
+
}
42
+
43
+
func addCandidateSession(t *testing.T, m *Mill, nodeID string, labels []string, load float64, enc messageEncoder) *millSession {
44
+
t.Helper()
45
+
if enc == nil {
46
+
enc = scriptedEncoder(func(*millproto.Message) error { return nil })
47
+
}
48
+
sess := newSession(nodeID, "inc-"+nodeID, labels, enc, slog.New(slog.NewTextHandler(io.Discard, nil)))
49
+
sess.snapshot = &millv1.NodeSnapshot{
50
+
Seqno: 1,
51
+
Engines: map[string]*millv1.EngineAvailability{
52
+
"dummy": {Available: load < 1.0, Load: map[string]float64{"slots": load}},
53
+
},
54
+
}
55
+
m.mu.Lock()
56
+
m.sessions[nodeID] = sess
57
+
m.mu.Unlock()
58
+
return sess
59
+
}
60
+
61
+
func assertRankedNodes(t *testing.T, got []*millSession, want []string) {
62
+
t.Helper()
63
+
if len(got) != len(want) {
64
+
t.Fatalf("rankCandidates() returned %d candidates, want %d: got %v want %v", len(got), len(want), sessionIDs(got), want)
65
+
}
66
+
for i := range want {
67
+
if got[i].nodeID != want[i] {
68
+
t.Fatalf("rankCandidates()[%d] = %q, want %q; full order got %v want %v", i, got[i].nodeID, want[i], sessionIDs(got), want)
69
+
}
70
+
}
71
+
}
72
+
73
+
func sessionIDs(sessions []*millSession) []string {
74
+
out := make([]string, len(sessions))
75
+
for i, sess := range sessions {
76
+
out[i] = sess.nodeID
77
+
}
78
+
return out
79
+
}
80
+
81
+
func sameStringMultiset(a, b []string) bool {
82
+
if len(a) != len(b) {
83
+
return false
84
+
}
85
+
counts := make(map[string]int, len(a))
86
+
for _, s := range a {
87
+
counts[s]++
88
+
}
89
+
for _, s := range b {
90
+
if counts[s] == 0 {
91
+
return false
92
+
}
93
+
counts[s]--
94
+
}
95
+
return true
96
+
}
97
+
98
+
type reserveReply struct {
99
+
accepted bool
100
+
rejectClass millv1.RejectClass
101
+
reason string
102
+
}
103
+
104
+
func addReplyingCandidateSession(t *testing.T, m *Mill, nodeID string, labels []string, load float64, asked chan<- string, reply reserveReply) *millSession {
105
+
t.Helper()
106
+
var sess *millSession
107
+
sess = addCandidateSession(t, m, nodeID, labels, load, scriptedEncoder(func(msg *millproto.Message) error {
108
+
rs := msg.GetReserveSeat()
109
+
if rs == nil {
110
+
return nil
111
+
}
112
+
if asked != nil {
113
+
asked <- nodeID
114
+
}
115
+
sess.deliver(rs.GetLeaseId(), &millproto.Message{ReserveResult: &millv1.ReserveResult{
116
+
LeaseId: rs.GetLeaseId(),
117
+
Accepted: reply.accepted,
118
+
RejectReason: reply.reason,
119
+
RejectClass: reply.rejectClass,
120
+
}})
121
+
return nil
122
+
}))
123
+
return sess
124
+
}
125
+
126
+
func drainAsked(ch <-chan string) []string {
127
+
var out []string
128
+
for {
129
+
select {
130
+
case nodeID := <-ch:
131
+
out = append(out, nodeID)
132
+
default:
133
+
return out
134
+
}
135
+
}
136
+
}
137
+
138
+
func TestCommitRetriesAfterSessionCloseBeforeCommitted(t *testing.T) {
139
+
l := slog.New(slog.NewTextHandler(io.Discard, nil))
140
+
m := New(l, Config{BidTimeout: 25 * time.Millisecond, ReconnectGrace: time.Second})
141
+
wf := testWorkflow("build")
142
+
wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
143
+
lease := newLease("lease-1", "node-1", "inc-1", "dummy")
144
+
lease.wid = wid
145
+
wf.Data.(*millWorkflowState).Lease = lease
146
+
147
+
m.mu.Lock()
148
+
m.leases[lease.id] = lease
149
+
m.mu.Unlock()
150
+
151
+
var sess1 *millSession
152
+
firstCommit := make(chan struct{})
153
+
sess1 = newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error {
154
+
if msg.GetCommitLease() != nil {
155
+
close(firstCommit)
156
+
m.detachSession(sess1)
157
+
}
158
+
return nil
159
+
}), l)
160
+
m.attachSession(sess1)
161
+
162
+
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
163
+
defer cancel()
164
+
done := make(chan error, 1)
165
+
go func() { done <- m.commitAndWait(ctx, wf, nil) }()
166
+
167
+
select {
168
+
case <-firstCommit:
169
+
case <-ctx.Done():
170
+
t.Fatal("first commit was not sent")
171
+
}
172
+
173
+
var sess2 *millSession
174
+
sess2 = newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error {
175
+
if msg.GetCommitLease() == nil {
176
+
return nil
177
+
}
178
+
leaseID := msg.GetCommitLease().GetLeaseId()
179
+
sess2.deliver(leaseID, &millproto.Message{Committed: &millv1.Committed{LeaseId: leaseID}})
180
+
_ = m.onEventBatch(sess2, &millv1.EventBatch{
181
+
Epoch: sess2.epoch,
182
+
Events: []*millv1.Event{
183
+
{
184
+
Seqno: 1,
185
+
LeaseId: leaseID,
186
+
Payload: &millv1.Event_AttemptResult{
187
+
AttemptResult: &millv1.AttemptResult{
188
+
Status: millv1.TerminalStatus_SUCCESS,
189
+
},
190
+
},
191
+
},
192
+
},
193
+
})
194
+
return nil
195
+
}), l)
196
+
m.attachSession(sess2)
197
+
m.sessionReady(sess2)
198
+
199
+
select {
200
+
case err := <-done:
201
+
if err != nil {
202
+
t.Fatalf("commitAndWait() error = %v, want success after reconnect", err)
203
+
}
204
+
case <-ctx.Done():
205
+
t.Fatal("commitAndWait() did not finish after reconnect")
206
+
}
207
+
}
208
+
209
+
func TestDestroyRunningLeaseDoesNotDropCancelledTerminal(t *testing.T) {
210
+
l := slog.New(slog.NewTextHandler(io.Discard, nil))
211
+
m := New(l, Config{})
212
+
wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
213
+
lease := newLease("lease-1", "node-1", "inc-1", "dummy")
214
+
lease.wid = wid
215
+
lease.setState(leaseRunning)
216
+
217
+
m.mu.Lock()
218
+
m.leases[lease.id] = lease
219
+
m.mu.Unlock()
220
+
221
+
m.destroy(wid)
222
+
if lease.getState() == leaseDone {
223
+
t.Fatal("destroy sealed the lease before the terminal result")
224
+
}
225
+
226
+
lease.deliverTerminal(&millv1.AttemptResult{
227
+
Status: millv1.TerminalStatus_CANCELLED,
228
+
})
229
+
res := <-lease.terminal
230
+
if err := terminalError(res.Status); !errors.Is(err, engine.ErrCancelled) {
231
+
t.Fatalf("terminalError() = %v, want ErrCancelled", err)
232
+
}
233
+
}
234
+
235
+
func TestPlaceBlocksWhenNoCapacity(t *testing.T) {
236
+
l := slog.New(slog.NewTextHandler(io.Discard, nil))
237
+
m := New(l, Config{})
238
+
wf := testWorkflow("build")
239
+
wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
240
+
241
+
// no executors at all: place must block until ctx expires (user sees pending)
242
+
ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond)
243
+
defer cancel()
244
+
245
+
_, err := m.place(ctx, "dummy", wid, wf)
246
+
if err != context.DeadlineExceeded {
247
+
t.Fatalf("place() error = %v, want DeadlineExceeded", err)
248
+
}
249
+
}
250
+
251
+
func TestRankCandidatesFiltersRequiredLabelsWithANDSemantics(t *testing.T) {
252
+
m := New(slog.New(slog.NewTextHandler(io.Discard, nil)), Config{})
253
+
addCandidateSession(t, m, "linux-high", []string{"linux"}, 0.0, nil)
254
+
addCandidateSession(t, m, "linux-arm", []string{"linux", "arm64"}, 0.25, nil)
255
+
addCandidateSession(t, m, "unlabeled", nil, 0.5, nil)
256
+
addCandidateSession(t, m, "linux-arm-gpu", []string{"linux", "arm64", "gpu"}, 0.75, nil)
257
+
addCandidateSession(t, m, "linux-arm-full", []string{"linux", "arm64"}, 1.0, nil)
258
+
259
+
tests := []struct {
260
+
name string
261
+
requiredLabels []string
262
+
want []string
263
+
}{
264
+
{
265
+
name: "no required labels keeps old capacity ranking",
266
+
want: []string{"linux-high", "linux-arm", "unlabeled", "linux-arm-gpu"},
267
+
},
268
+
{
269
+
name: "single required label includes every candidate carrying it",
270
+
requiredLabels: []string{"linux"},
271
+
want: []string{"linux-high", "linux-arm", "linux-arm-gpu"},
272
+
},
273
+
{
274
+
name: "all required labels must be present",
275
+
requiredLabels: []string{"linux", "arm64"},
276
+
want: []string{"linux-arm", "linux-arm-gpu"},
277
+
},
278
+
{
279
+
name: "one missing required label excludes the candidate",
280
+
requiredLabels: []string{"linux", "arm64", "gpu"},
281
+
want: []string{"linux-arm-gpu"},
282
+
},
283
+
{
284
+
name: "unknown required label leaves no candidate",
285
+
requiredLabels: []string{"linux", "arm64", "metal"},
286
+
},
287
+
}
288
+
289
+
for _, tt := range tests {
290
+
t.Run(tt.name, func(t *testing.T) {
291
+
assertRankedNodes(t, m.rankCandidates("dummy", tt.requiredLabels), tt.want)
292
+
})
293
+
}
294
+
}
295
+
296
+
func TestPlaceWithMissingRequiredLabelsStaysPendingWithoutReserve(t *testing.T) {
297
+
l := slog.New(slog.NewTextHandler(io.Discard, nil))
298
+
m := New(l, Config{BidTimeout: 10 * time.Millisecond})
299
+
reserveSent := make(chan struct{}, 1)
300
+
addCandidateSession(t, m, "linux-only", []string{"linux"}, 0.75, scriptedEncoder(func(msg *millproto.Message) error {
301
+
if msg.GetReserveSeat() != nil {
302
+
select {
303
+
case reserveSent <- struct{}{}:
304
+
default:
305
+
}
306
+
}
307
+
return nil
308
+
}))
309
+
wf := testWorkflowWithRunsOn("build", []string{"linux", "arm64"})
310
+
wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
311
+
312
+
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond)
313
+
defer cancel()
314
+
_, err := m.place(ctx, "dummy", wid, wf)
315
+
if err != context.DeadlineExceeded {
316
+
t.Fatalf("place() error = %v, want DeadlineExceeded while job remains pending", err)
317
+
}
318
+
select {
319
+
case <-reserveSent:
320
+
t.Fatal("place() sent ReserveSeat to executor missing a required label")
321
+
default:
322
+
}
323
+
}
324
+
325
+
func TestMaxPendingRejects(t *testing.T) {
326
+
l := slog.New(slog.NewTextHandler(io.Discard, nil))
327
+
m := New(l, Config{MaxPending: 1})
328
+
329
+
m.mu.Lock()
330
+
m.pending = 1
331
+
m.mu.Unlock()
332
+
333
+
wf2 := testWorkflow("b")
334
+
_, err := m.place(context.Background(), "dummy", models.WorkflowId{Name: "b"}, wf2)
335
+
if err == nil {
336
+
t.Fatal("place() past maxPending should error")
337
+
}
338
+
}
339
+
340
+
func TestCancelledRunningLeaseSurvivesReleaseForReconnectReplay(t *testing.T) {
341
+
m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
342
+
wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
343
+
lease := newLease("lease-1", "node-1", "inc-1", "dummy")
344
+
lease.wid = wid
345
+
lease.setState(leaseRunning)
346
+
if err := m.persistLease(lease, leaseRowRunning); err != nil {
347
+
t.Fatalf("persistLease: %v", err)
348
+
}
349
+
m.mu.Lock()
350
+
m.leases[lease.id] = lease
351
+
m.mu.Unlock()
352
+
353
+
m.destroy(wid)
354
+
slot := &millSlot{fleet: m, lease: lease}
355
+
slot.Release()
356
+
357
+
m.mu.Lock()
358
+
_, retained := m.leases[lease.id]
359
+
m.mu.Unlock()
360
+
if !retained {
361
+
t.Fatal("slot release removed a cancellation-requested running lease before its terminal result")
362
+
}
363
+
if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 1 {
364
+
t.Fatalf("durable leases after slot release = %+v, err = %v; want retained lease", rows, err)
365
+
}
366
+
367
+
var sentMu sync.Mutex
368
+
var sent []*millproto.Message
369
+
sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error {
370
+
sentMu.Lock()
371
+
sent = append(sent, msg)
372
+
sentMu.Unlock()
373
+
return nil
374
+
}), discardLogger())
375
+
if _, ok := m.attachSession(sess); !ok {
376
+
t.Fatal("attachSession rejected reconnect")
377
+
}
378
+
m.sessionReady(sess)
379
+
sentMu.Lock()
380
+
var replayed bool
381
+
for _, msg := range sent {
382
+
if cancel := msg.GetCancelAttempt(); cancel != nil && cancel.GetLeaseId() == lease.id {
383
+
replayed = true
384
+
}
385
+
}
386
+
sentMu.Unlock()
387
+
if !replayed {
388
+
t.Fatal("reconnect did not replay CancelAttempt for retained lease")
389
+
}
390
+
391
+
if err := m.onEventBatch(sess, &millv1.EventBatch{
392
+
Epoch: sess.epoch,
393
+
Events: []*millv1.Event{
394
+
{
395
+
Seqno: 1,
396
+
LeaseId: lease.id,
397
+
Payload: &millv1.Event_AttemptResult{
398
+
AttemptResult: &millv1.AttemptResult{
399
+
Status: millv1.TerminalStatus_CANCELLED,
400
+
},
401
+
},
402
+
},
403
+
},
404
+
}); err != nil {
405
+
t.Fatalf("onEventBatch: %v", err)
406
+
}
407
+
m.mu.Lock()
408
+
_, retained = m.leases[lease.id]
409
+
m.mu.Unlock()
410
+
if retained {
411
+
t.Fatal("terminal result did not clean retained cancelled lease")
412
+
}
413
+
if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 0 {
414
+
t.Fatalf("durable leases after terminal = %+v, err = %v; want none", rows, err)
415
+
}
416
+
slot.Release()
417
+
}
418
+
419
+
func TestSessionRequestCancelledBeforeRegistrationOrSend(t *testing.T) {
420
+
sent := 0
421
+
sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(*millproto.Message) error {
422
+
sent++
423
+
return nil
424
+
}), discardLogger())
425
+
ctx, cancel := context.WithCancel(context.Background())
426
+
cancel()
427
+
_, err := sess.request(ctx, "lease-1", &millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: "lease-1"}})
428
+
if !errors.Is(err, context.Canceled) {
429
+
t.Fatalf("request error = %v, want context.Canceled", err)
430
+
}
431
+
if sent != 0 {
432
+
t.Fatalf("request sent %d messages for an already-cancelled context, want 0", sent)
433
+
}
434
+
sess.mu.Lock()
435
+
pending := len(sess.pending)
436
+
sess.mu.Unlock()
437
+
if pending != 0 {
438
+
t.Fatalf("request left %d pending waiters, want 0", pending)
439
+
}
440
+
}
441
+
442
+
func TestAttachSessionReplacesSilentIncumbentButRejectsActiveDuplicate(t *testing.T) {
443
+
m := New(discardLogger(), Config{ReconnectGrace: time.Minute})
444
+
old := newSession("node-1", "inc-old", nil, nopEncoder(), discardLogger())
445
+
transportClosed := make(chan struct{})
446
+
old.closeTransport = func() error {
447
+
close(transportClosed)
448
+
return nil
449
+
}
450
+
if _, ok := m.attachSession(old); !ok {
451
+
t.Fatal("first attach rejected")
452
+
}
453
+
m.mu.Lock()
454
+
old.lastSeen = time.Now().Add(-2 * m.cfg.ReconnectGrace)
455
+
m.mu.Unlock()
456
+
457
+
replacement := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger())
458
+
if _, ok := m.attachSession(replacement); !ok {
459
+
t.Fatal("silent incumbent blocked authenticated replacement")
460
+
}
461
+
select {
462
+
case <-transportClosed:
463
+
default:
464
+
t.Fatal("replacing a silent incumbent did not close its transport")
465
+
}
466
+
m.mu.Lock()
467
+
replacement.lastSeen = time.Now().Add(-2 * m.cfg.ReconnectGrace)
468
+
m.mu.Unlock()
469
+
if err := replacement.dispatch(m, &millproto.Message{NodeSnapshot: &millv1.NodeSnapshot{Seqno: 1}}); err != nil {
470
+
t.Fatalf("periodic snapshot dispatch: %v", err)
471
+
}
472
+
if _, ok := m.attachSession(newSession("node-1", "inc-dup", nil, nopEncoder(), discardLogger())); ok {
473
+
t.Fatal("active replacement did not reject a duplicate session")
474
+
}
475
+
}
476
+
477
+
func TestPlaceReleasesRemoteReservationWhenInitialPersistenceFails(t *testing.T) {
478
+
m, bdb := restoreTestMill(t, Config{BidTimeout: time.Second})
479
+
if err := bdb.Close(); err != nil {
480
+
t.Fatalf("close db: %v", err)
481
+
}
482
+
released := make(chan string, 1)
483
+
var sess *millSession
484
+
sess = addCandidateSession(t, m, "node-1", nil, 0, scriptedEncoder(func(msg *millproto.Message) error {
485
+
switch {
486
+
case msg.GetReserveSeat() != nil:
487
+
leaseID := msg.GetReserveSeat().GetLeaseId()
488
+
sess.deliver(leaseID, &millproto.Message{ReserveResult: &millv1.ReserveResult{
489
+
LeaseId: leaseID,
490
+
Accepted: true,
491
+
}})
492
+
case msg.GetReleaseLease() != nil:
493
+
released <- msg.GetReleaseLease().GetLeaseId()
494
+
}
495
+
return nil
496
+
}))
497
+
498
+
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
499
+
defer cancel()
500
+
slot, err := m.place(
501
+
ctx,
502
+
"dummy",
503
+
models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"},
504
+
testWorkflow("build"),
505
+
)
506
+
if err == nil {
507
+
t.Fatal("place succeeded after reserved lease persistence failed")
508
+
}
509
+
if slot != nil {
510
+
t.Fatalf("place returned slot %T after persistence failure", slot)
511
+
}
512
+
select {
513
+
case leaseID := <-released:
514
+
if leaseID == "" {
515
+
t.Fatal("ReleaseLease had empty lease id")
516
+
}
517
+
case <-time.After(time.Second):
518
+
t.Fatal("persistence failure did not compensate with ReleaseLease")
519
+
}
520
+
m.mu.Lock()
521
+
leases := len(m.leases)
522
+
m.mu.Unlock()
523
+
if leases != 0 {
524
+
t.Fatalf("mill published %d leases after initial persistence failure, want 0", leases)
525
+
}
526
+
}
527
+
528
+
func TestGapsAndDuplicates(t *testing.T) {
529
+
m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
530
+
sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
531
+
m.attachSession(sess)
532
+
533
+
owned := newLease("lease-1", "node-1", "inc-1", "dummy")
534
+
owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
535
+
m.mu.Lock()
536
+
m.leases[owned.id] = owned
537
+
m.mu.Unlock()
538
+
539
+
err := m.onEventBatch(sess, &millv1.EventBatch{
540
+
Epoch: sess.epoch,
541
+
Events: []*millv1.Event{
542
+
{
543
+
Seqno: 0,
544
+
LeaseId: owned.id,
545
+
Payload: &millv1.Event_StatusEvent{
546
+
StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING},
547
+
},
548
+
},
549
+
},
550
+
})
551
+
if err != nil {
552
+
t.Fatalf("expected duplicate to be skipped without error, got: %v", err)
553
+
}
554
+
555
+
err = m.onEventBatch(sess, &millv1.EventBatch{
556
+
Epoch: sess.epoch,
557
+
Events: []*millv1.Event{
558
+
{
559
+
Seqno: 2,
560
+
LeaseId: owned.id,
561
+
Payload: &millv1.Event_StatusEvent{
562
+
StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING},
563
+
},
564
+
},
565
+
},
566
+
})
567
+
if err == nil {
568
+
t.Fatal("expected error due to seqno gap, got nil")
569
+
}
570
+
}
571
+
572
+
func TestAtomicBatchRollback(t *testing.T) {
573
+
m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
574
+
sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
575
+
m.attachSession(sess)
576
+
577
+
owned := newLease("lease-1", "node-1", "inc-1", "dummy")
578
+
owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
579
+
m.mu.Lock()
580
+
m.leases[owned.id] = owned
581
+
m.mu.Unlock()
582
+
583
+
if _, err := bdb.Exec(`
584
+
create trigger reject_status_event
585
+
before insert on events
586
+
begin
587
+
select raise(abort, 'forced status event failure');
588
+
end
589
+
`); err != nil {
590
+
t.Fatalf("failed to create fail trigger: %v", err)
591
+
}
592
+
defer bdb.Exec("drop trigger reject_status_event")
593
+
594
+
err := m.onEventBatch(sess, &millv1.EventBatch{
595
+
Epoch: sess.epoch,
596
+
Events: []*millv1.Event{
597
+
{
598
+
Seqno: 1,
599
+
LeaseId: owned.id,
600
+
Payload: &millv1.Event_StatusEvent{
601
+
StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING},
602
+
},
603
+
},
604
+
},
605
+
})
606
+
if err == nil {
607
+
t.Fatal("expected status event insertion to fail due to trigger")
608
+
}
609
+
610
+
m.mu.Lock()
611
+
seqno := m.nodeSeqno["node-1/inc-1"]
612
+
m.mu.Unlock()
613
+
if seqno != 0 {
614
+
t.Fatalf("expected seqno 0 due to rollback, got %d", seqno)
615
+
}
616
+
}
617
+
618
+
func TestTerminalBeforeACK(t *testing.T) {
619
+
m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
620
+
621
+
ackSent := make(chan struct{})
622
+
sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error {
623
+
if msg.GetAck() != nil {
624
+
wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
625
+
st, err := bdb.GetStatus(wid)
626
+
if err != nil || st.Status != "success" {
627
+
t.Errorf("expected terminal status success at ACK time, got status: %v, err: %v", st, err)
628
+
}
629
+
close(ackSent)
630
+
}
631
+
return nil
632
+
}), discardLogger())
633
+
m.attachSession(sess)
634
+
635
+
owned := newLease("lease-1", "node-1", "inc-1", "dummy")
636
+
owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
637
+
m.mu.Lock()
638
+
m.leases[owned.id] = owned
639
+
m.mu.Unlock()
640
+
641
+
err := m.onEventBatch(sess, &millv1.EventBatch{
642
+
Epoch: sess.epoch,
643
+
Events: []*millv1.Event{
644
+
{
645
+
Seqno: 1,
646
+
LeaseId: owned.id,
647
+
Payload: &millv1.Event_AttemptResult{
648
+
AttemptResult: &millv1.AttemptResult{Status: millv1.TerminalStatus_SUCCESS},
649
+
},
650
+
},
651
+
},
652
+
})
653
+
if err != nil {
654
+
t.Fatalf("onEventBatch: %v", err)
655
+
}
656
+
657
+
select {
658
+
case <-ackSent:
659
+
case <-time.After(2 * time.Second):
660
+
t.Fatal("ACK was not sent")
661
+
}
662
+
}
663
+
664
+
func TestExecutorRestartEmptySnapshot(t *testing.T) {
665
+
m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
666
+
667
+
lease := newLease("lease-1", "node-1", "inc-old", "dummy")
668
+
lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
669
+
m.mu.Lock()
670
+
m.leases[lease.id] = lease
671
+
m.mu.Unlock()
672
+
if err := m.persistLease(lease, leaseRowRunning); err != nil {
673
+
t.Fatalf("persistLease: %v", err)
674
+
}
675
+
676
+
sess := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger())
677
+
m.attachSession(sess)
678
+
679
+
err := m.onSnapshot(sess, &millv1.NodeSnapshot{
680
+
Seqno: 1,
681
+
ActiveLeaseIds: nil,
682
+
})
683
+
if err != nil {
684
+
t.Fatalf("onSnapshot: %v", err)
685
+
}
686
+
687
+
m.mu.Lock()
688
+
_, stillActive := m.leases["lease-1"]
689
+
m.mu.Unlock()
690
+
if stillActive {
691
+
t.Fatal("expected old epoch lease to be reconciled and failed")
692
+
}
693
+
}
694
+
695
+
func TestReplacementLostBeforeSnapshotFailsOldEpochLease(t *testing.T) {
696
+
m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
697
+
lease := newLease("lease-1", "node-1", "inc-old", "dummy")
698
+
lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
699
+
lease.setState(leaseRunning)
700
+
if err := m.persistLease(lease, leaseRowRunning); err != nil {
701
+
t.Fatalf("persistLease: %v", err)
702
+
}
703
+
m.mu.Lock()
704
+
m.leases[lease.id] = lease
705
+
m.mu.Unlock()
706
+
707
+
replacement := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger())
708
+
if _, ok := m.attachSession(replacement); !ok {
709
+
t.Fatal("attachSession rejected replacement")
710
+
}
711
+
replacement.disconnected = true
712
+
m.failLeasesAfterGrace(replacement)
713
+
714
+
m.mu.Lock()
715
+
_, stillActive := m.leases[lease.id]
716
+
m.mu.Unlock()
717
+
if stillActive {
718
+
t.Fatal("replacement loss stranded old-epoch lease")
719
+
}
720
+
}
721
+
722
+
func TestSeqRegression(t *testing.T) {
723
+
m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
724
+
sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
725
+
m.attachSession(sess)
726
+
727
+
err := m.onSnapshot(sess, &millv1.NodeSnapshot{
728
+
Seqno: 5,
729
+
})
730
+
if err != nil {
731
+
t.Fatalf("first snapshot: %v", err)
732
+
}
733
+
734
+
err = m.onSnapshot(sess, &millv1.NodeSnapshot{
735
+
Seqno: 4,
736
+
})
737
+
if err == nil {
738
+
t.Fatal("expected seqno regression to be rejected")
739
+
}
740
+
}
741
+
742
+
func TestClaimedSweep(t *testing.T) {
743
+
m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
744
+
745
+
lease := newLease("lease-1", "node-1", "inc-1", "dummy")
746
+
lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
747
+
lease.orphaned = true
748
+
lease.claimed = false
749
+
m.mu.Lock()
750
+
m.leases[lease.id] = lease
751
+
m.mu.Unlock()
752
+
753
+
sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
754
+
m.attachSession(sess)
755
+
err := m.onSnapshot(sess, &millv1.NodeSnapshot{
756
+
Seqno: 1,
757
+
ActiveLeaseIds: []string{"lease-1"},
758
+
})
759
+
if err != nil {
760
+
t.Fatalf("onSnapshot: %v", err)
761
+
}
762
+
763
+
m.detachSession(sess)
764
+
765
+
m.sweepUnclaimedOrphans()
766
+
767
+
m.mu.Lock()
768
+
_, stillRunning := m.leases["lease-1"]
769
+
m.mu.Unlock()
770
+
if !stillRunning {
771
+
t.Fatal("claimed lease was incorrectly swept by startup sweep")
772
+
}
773
+
}
774
+
775
+
func TestCancelDeadline(t *testing.T) {
776
+
m, _ := restoreTestMill(t, Config{CancelTimeout: 50 * time.Millisecond})
777
+
778
+
sessClosed := make(chan struct{})
779
+
sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error {
780
+
return nil
781
+
}), discardLogger())
782
+
sess.closeTransport = func() error {
783
+
close(sessClosed)
784
+
return nil
785
+
}
786
+
m.attachSession(sess)
787
+
788
+
lease := newLease("lease-1", "node-1", "inc-1", "dummy")
789
+
lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
790
+
lease.setState(leaseRunning)
791
+
m.mu.Lock()
792
+
m.leases[lease.id] = lease
793
+
m.mu.Unlock()
794
+
795
+
m.destroy(lease.wid)
796
+
797
+
select {
798
+
case <-sessClosed:
799
+
case <-time.After(1 * time.Second):
800
+
t.Fatal("session was not closed after cancel deadline expiration")
801
+
}
802
+
}
803
+
804
+
func TestCleanupRetry(t *testing.T) {
805
+
m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
806
+
807
+
lease := newLease("lease-1", "node-1", "inc-1", "dummy")
808
+
lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
809
+
if err := m.persistLease(lease, leaseRowRunning); err != nil {
810
+
t.Fatalf("persist lease: %v", err)
811
+
}
812
+
m.mu.Lock()
813
+
m.leases[lease.id] = lease
814
+
m.mu.Unlock()
815
+
816
+
if _, err := bdb.Exec(`
817
+
create trigger reject_cleanup_delete
818
+
before delete on mill_leases
819
+
begin
820
+
select raise(abort, 'forced delete failure');
821
+
end
822
+
`); err != nil {
823
+
t.Fatalf("failed to create fail trigger: %v", err)
824
+
}
825
+
826
+
err := m.cleanupLease(lease)
827
+
if err == nil {
828
+
t.Fatal("expected cleanupLease to fail")
829
+
}
830
+
831
+
m.mu.Lock()
832
+
_, stillRunning := m.leases["lease-1"]
833
+
m.mu.Unlock()
834
+
if !stillRunning {
835
+
t.Fatal("lease was removed from memory despite cleanup failure")
836
+
}
837
+
838
+
if _, err := bdb.Exec("drop trigger reject_cleanup_delete"); err != nil {
839
+
t.Fatalf("drop trigger: %v", err)
840
+
}
841
+
842
+
err = m.cleanupLease(lease)
843
+
if err != nil {
844
+
t.Fatalf("expected retry cleanup to succeed, got: %v", err)
845
+
}
846
+
847
+
m.mu.Lock()
848
+
_, stillRunning = m.leases["lease-1"]
849
+
m.mu.Unlock()
850
+
if stillRunning {
851
+
t.Fatal("lease still in memory after successful cleanup retry")
852
+
}
853
+
}
854
+
855
+
func TestBoundedBidding(t *testing.T) {
856
+
m := New(discardLogger(), Config{TopK: 2, BidTimeout: 10 * time.Millisecond})
857
+
858
+
addCandidateSession(t, m, "node-1", nil, 0, nil)
859
+
addCandidateSession(t, m, "node-2", nil, 0, nil)
860
+
addCandidateSession(t, m, "node-3", nil, 0, nil)
861
+
addCandidateSession(t, m, "node-4", nil, 0, nil)
862
+
addCandidateSession(t, m, "node-5", nil, 0, nil)
863
+
864
+
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
865
+
defer cancel()
866
+
867
+
lease, err := m.bid(ctx, "dummy", models.WorkflowId{}, testWorkflow("build"))
868
+
if err != nil {
869
+
t.Fatalf("bid: %v", err)
870
+
}
871
+
if lease != nil {
872
+
t.Fatalf("did not expect a lease, got %+v", lease)
873
+
}
874
+
}
+1612
spindle/mill/proto/gen/mill.pb.go
+1612
spindle/mill/proto/gen/mill.pb.go
···
1
+
// Code generated by protoc-gen-go. DO NOT EDIT.
2
+
// versions:
3
+
// protoc-gen-go v1.36.11
4
+
// protoc (unknown)
5
+
// source: spindle/mill/v1/mill.proto
6
+
7
+
package millv1
8
+
9
+
import (
10
+
_ "buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go/buf/validate"
11
+
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
12
+
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
13
+
reflect "reflect"
14
+
sync "sync"
15
+
unsafe "unsafe"
16
+
)
17
+
18
+
const (
19
+
// Verify that this generated code is sufficiently up-to-date.
20
+
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
21
+
// Verify that runtime/protoimpl is sufficiently up-to-date.
22
+
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
23
+
)
24
+
25
+
type RejectClass int32
26
+
27
+
const (
28
+
RejectClass_REJECT_CLASS_UNSPECIFIED RejectClass = 0
29
+
RejectClass_REJECT_CLASS_TRANSIENT RejectClass = 1
30
+
RejectClass_REJECT_CLASS_INCOMPATIBLE RejectClass = 2
31
+
)
32
+
33
+
// Enum value maps for RejectClass.
34
+
var (
35
+
RejectClass_name = map[int32]string{
36
+
0: "REJECT_CLASS_UNSPECIFIED",
37
+
1: "REJECT_CLASS_TRANSIENT",
38
+
2: "REJECT_CLASS_INCOMPATIBLE",
39
+
}
40
+
RejectClass_value = map[string]int32{
41
+
"REJECT_CLASS_UNSPECIFIED": 0,
42
+
"REJECT_CLASS_TRANSIENT": 1,
43
+
"REJECT_CLASS_INCOMPATIBLE": 2,
44
+
}
45
+
)
46
+
47
+
func (x RejectClass) Enum() *RejectClass {
48
+
p := new(RejectClass)
49
+
*p = x
50
+
return p
51
+
}
52
+
53
+
func (x RejectClass) String() string {
54
+
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
55
+
}
56
+
57
+
func (RejectClass) Descriptor() protoreflect.EnumDescriptor {
58
+
return file_spindle_mill_v1_mill_proto_enumTypes[0].Descriptor()
59
+
}
60
+
61
+
func (RejectClass) Type() protoreflect.EnumType {
62
+
return &file_spindle_mill_v1_mill_proto_enumTypes[0]
63
+
}
64
+
65
+
func (x RejectClass) Number() protoreflect.EnumNumber {
66
+
return protoreflect.EnumNumber(x)
67
+
}
68
+
69
+
// Deprecated: Use RejectClass.Descriptor instead.
70
+
func (RejectClass) EnumDescriptor() ([]byte, []int) {
71
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{0}
72
+
}
73
+
74
+
type NonterminalStatus int32
75
+
76
+
const (
77
+
NonterminalStatus_NONTERMINAL_STATUS_UNSPECIFIED NonterminalStatus = 0
78
+
NonterminalStatus_RUNNING NonterminalStatus = 1
79
+
)
80
+
81
+
// Enum value maps for NonterminalStatus.
82
+
var (
83
+
NonterminalStatus_name = map[int32]string{
84
+
0: "NONTERMINAL_STATUS_UNSPECIFIED",
85
+
1: "RUNNING",
86
+
}
87
+
NonterminalStatus_value = map[string]int32{
88
+
"NONTERMINAL_STATUS_UNSPECIFIED": 0,
89
+
"RUNNING": 1,
90
+
}
91
+
)
92
+
93
+
func (x NonterminalStatus) Enum() *NonterminalStatus {
94
+
p := new(NonterminalStatus)
95
+
*p = x
96
+
return p
97
+
}
98
+
99
+
func (x NonterminalStatus) String() string {
100
+
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
101
+
}
102
+
103
+
func (NonterminalStatus) Descriptor() protoreflect.EnumDescriptor {
104
+
return file_spindle_mill_v1_mill_proto_enumTypes[1].Descriptor()
105
+
}
106
+
107
+
func (NonterminalStatus) Type() protoreflect.EnumType {
108
+
return &file_spindle_mill_v1_mill_proto_enumTypes[1]
109
+
}
110
+
111
+
func (x NonterminalStatus) Number() protoreflect.EnumNumber {
112
+
return protoreflect.EnumNumber(x)
113
+
}
114
+
115
+
// Deprecated: Use NonterminalStatus.Descriptor instead.
116
+
func (NonterminalStatus) EnumDescriptor() ([]byte, []int) {
117
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{1}
118
+
}
119
+
120
+
type TerminalStatus int32
121
+
122
+
const (
123
+
TerminalStatus_TERMINAL_STATUS_UNSPECIFIED TerminalStatus = 0
124
+
TerminalStatus_SUCCESS TerminalStatus = 1
125
+
TerminalStatus_FAILED TerminalStatus = 2
126
+
TerminalStatus_TIMEOUT TerminalStatus = 3
127
+
TerminalStatus_CANCELLED TerminalStatus = 4
128
+
)
129
+
130
+
// Enum value maps for TerminalStatus.
131
+
var (
132
+
TerminalStatus_name = map[int32]string{
133
+
0: "TERMINAL_STATUS_UNSPECIFIED",
134
+
1: "SUCCESS",
135
+
2: "FAILED",
136
+
3: "TIMEOUT",
137
+
4: "CANCELLED",
138
+
}
139
+
TerminalStatus_value = map[string]int32{
140
+
"TERMINAL_STATUS_UNSPECIFIED": 0,
141
+
"SUCCESS": 1,
142
+
"FAILED": 2,
143
+
"TIMEOUT": 3,
144
+
"CANCELLED": 4,
145
+
}
146
+
)
147
+
148
+
func (x TerminalStatus) Enum() *TerminalStatus {
149
+
p := new(TerminalStatus)
150
+
*p = x
151
+
return p
152
+
}
153
+
154
+
func (x TerminalStatus) String() string {
155
+
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
156
+
}
157
+
158
+
func (TerminalStatus) Descriptor() protoreflect.EnumDescriptor {
159
+
return file_spindle_mill_v1_mill_proto_enumTypes[2].Descriptor()
160
+
}
161
+
162
+
func (TerminalStatus) Type() protoreflect.EnumType {
163
+
return &file_spindle_mill_v1_mill_proto_enumTypes[2]
164
+
}
165
+
166
+
func (x TerminalStatus) Number() protoreflect.EnumNumber {
167
+
return protoreflect.EnumNumber(x)
168
+
}
169
+
170
+
// Deprecated: Use TerminalStatus.Descriptor instead.
171
+
func (TerminalStatus) EnumDescriptor() ([]byte, []int) {
172
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{2}
173
+
}
174
+
175
+
// executor identity, sent on connect
176
+
type Hello struct {
177
+
state protoimpl.MessageState `protogen:"open.v1"`
178
+
ProtocolVersion uint32 `protobuf:"varint,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"`
179
+
// GOARCH of the node, informational only
180
+
Arch string `protobuf:"bytes,2,opt,name=arch,proto3" json:"arch,omitempty"`
181
+
// operator-defined labels, matched against runs_on
182
+
Labels []string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty"`
183
+
Epoch string `protobuf:"bytes,4,opt,name=epoch,proto3" json:"epoch,omitempty"`
184
+
unknownFields protoimpl.UnknownFields
185
+
sizeCache protoimpl.SizeCache
186
+
}
187
+
188
+
func (x *Hello) Reset() {
189
+
*x = Hello{}
190
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[0]
191
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
192
+
ms.StoreMessageInfo(mi)
193
+
}
194
+
195
+
func (x *Hello) String() string {
196
+
return protoimpl.X.MessageStringOf(x)
197
+
}
198
+
199
+
func (*Hello) ProtoMessage() {}
200
+
201
+
func (x *Hello) ProtoReflect() protoreflect.Message {
202
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[0]
203
+
if x != nil {
204
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
205
+
if ms.LoadMessageInfo() == nil {
206
+
ms.StoreMessageInfo(mi)
207
+
}
208
+
return ms
209
+
}
210
+
return mi.MessageOf(x)
211
+
}
212
+
213
+
// Deprecated: Use Hello.ProtoReflect.Descriptor instead.
214
+
func (*Hello) Descriptor() ([]byte, []int) {
215
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{0}
216
+
}
217
+
218
+
func (x *Hello) GetProtocolVersion() uint32 {
219
+
if x != nil {
220
+
return x.ProtocolVersion
221
+
}
222
+
return 0
223
+
}
224
+
225
+
func (x *Hello) GetArch() string {
226
+
if x != nil {
227
+
return x.Arch
228
+
}
229
+
return ""
230
+
}
231
+
232
+
func (x *Hello) GetLabels() []string {
233
+
if x != nil {
234
+
return x.Labels
235
+
}
236
+
return nil
237
+
}
238
+
239
+
func (x *Hello) GetEpoch() string {
240
+
if x != nil {
241
+
return x.Epoch
242
+
}
243
+
return ""
244
+
}
245
+
246
+
// reconnect state for an existing epoch
247
+
type Resume struct {
248
+
state protoimpl.MessageState `protogen:"open.v1"`
249
+
Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"`
250
+
AckSeqno uint64 `protobuf:"varint,2,opt,name=ack_seqno,json=ackSeqno,proto3" json:"ack_seqno,omitempty"`
251
+
unknownFields protoimpl.UnknownFields
252
+
sizeCache protoimpl.SizeCache
253
+
}
254
+
255
+
func (x *Resume) Reset() {
256
+
*x = Resume{}
257
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[1]
258
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
259
+
ms.StoreMessageInfo(mi)
260
+
}
261
+
262
+
func (x *Resume) String() string {
263
+
return protoimpl.X.MessageStringOf(x)
264
+
}
265
+
266
+
func (*Resume) ProtoMessage() {}
267
+
268
+
func (x *Resume) ProtoReflect() protoreflect.Message {
269
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[1]
270
+
if x != nil {
271
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
272
+
if ms.LoadMessageInfo() == nil {
273
+
ms.StoreMessageInfo(mi)
274
+
}
275
+
return ms
276
+
}
277
+
return mi.MessageOf(x)
278
+
}
279
+
280
+
// Deprecated: Use Resume.ProtoReflect.Descriptor instead.
281
+
func (*Resume) Descriptor() ([]byte, []int) {
282
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{1}
283
+
}
284
+
285
+
func (x *Resume) GetEpoch() string {
286
+
if x != nil {
287
+
return x.Epoch
288
+
}
289
+
return ""
290
+
}
291
+
292
+
func (x *Resume) GetAckSeqno() uint64 {
293
+
if x != nil {
294
+
return x.AckSeqno
295
+
}
296
+
return 0
297
+
}
298
+
299
+
// per-engine state of a node
300
+
type EngineAvailability struct {
301
+
state protoimpl.MessageState `protogen:"open.v1"`
302
+
Available bool `protobuf:"varint,1,opt,name=available,proto3" json:"available,omitempty"`
303
+
// opaque engine-defined load metrics, higher means more loaded
304
+
Load map[string]float64 `protobuf:"bytes,2,rep,name=load,proto3" json:"load,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"fixed64,2,opt,name=value"`
305
+
unknownFields protoimpl.UnknownFields
306
+
sizeCache protoimpl.SizeCache
307
+
}
308
+
309
+
func (x *EngineAvailability) Reset() {
310
+
*x = EngineAvailability{}
311
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[2]
312
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
313
+
ms.StoreMessageInfo(mi)
314
+
}
315
+
316
+
func (x *EngineAvailability) String() string {
317
+
return protoimpl.X.MessageStringOf(x)
318
+
}
319
+
320
+
func (*EngineAvailability) ProtoMessage() {}
321
+
322
+
func (x *EngineAvailability) ProtoReflect() protoreflect.Message {
323
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[2]
324
+
if x != nil {
325
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
326
+
if ms.LoadMessageInfo() == nil {
327
+
ms.StoreMessageInfo(mi)
328
+
}
329
+
return ms
330
+
}
331
+
return mi.MessageOf(x)
332
+
}
333
+
334
+
// Deprecated: Use EngineAvailability.ProtoReflect.Descriptor instead.
335
+
func (*EngineAvailability) Descriptor() ([]byte, []int) {
336
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{2}
337
+
}
338
+
339
+
func (x *EngineAvailability) GetAvailable() bool {
340
+
if x != nil {
341
+
return x.Available
342
+
}
343
+
return false
344
+
}
345
+
346
+
func (x *EngineAvailability) GetLoad() map[string]float64 {
347
+
if x != nil {
348
+
return x.Load
349
+
}
350
+
return nil
351
+
}
352
+
353
+
// full node state
354
+
type NodeSnapshot struct {
355
+
state protoimpl.MessageState `protogen:"open.v1"`
356
+
Seqno uint64 `protobuf:"varint,1,opt,name=seqno,proto3" json:"seqno,omitempty"`
357
+
Engines map[string]*EngineAvailability `protobuf:"bytes,2,rep,name=engines,proto3" json:"engines,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
358
+
// every lease the executor currently holds, reserved or running
359
+
ActiveLeaseIds []string `protobuf:"bytes,3,rep,name=active_lease_ids,json=activeLeaseIds,proto3" json:"active_lease_ids,omitempty"`
360
+
unknownFields protoimpl.UnknownFields
361
+
sizeCache protoimpl.SizeCache
362
+
}
363
+
364
+
func (x *NodeSnapshot) Reset() {
365
+
*x = NodeSnapshot{}
366
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[3]
367
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
368
+
ms.StoreMessageInfo(mi)
369
+
}
370
+
371
+
func (x *NodeSnapshot) String() string {
372
+
return protoimpl.X.MessageStringOf(x)
373
+
}
374
+
375
+
func (*NodeSnapshot) ProtoMessage() {}
376
+
377
+
func (x *NodeSnapshot) ProtoReflect() protoreflect.Message {
378
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[3]
379
+
if x != nil {
380
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
381
+
if ms.LoadMessageInfo() == nil {
382
+
ms.StoreMessageInfo(mi)
383
+
}
384
+
return ms
385
+
}
386
+
return mi.MessageOf(x)
387
+
}
388
+
389
+
// Deprecated: Use NodeSnapshot.ProtoReflect.Descriptor instead.
390
+
func (*NodeSnapshot) Descriptor() ([]byte, []int) {
391
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{3}
392
+
}
393
+
394
+
func (x *NodeSnapshot) GetSeqno() uint64 {
395
+
if x != nil {
396
+
return x.Seqno
397
+
}
398
+
return 0
399
+
}
400
+
401
+
func (x *NodeSnapshot) GetEngines() map[string]*EngineAvailability {
402
+
if x != nil {
403
+
return x.Engines
404
+
}
405
+
return nil
406
+
}
407
+
408
+
func (x *NodeSnapshot) GetActiveLeaseIds() []string {
409
+
if x != nil {
410
+
return x.ActiveLeaseIds
411
+
}
412
+
return nil
413
+
}
414
+
415
+
// a seat reservation, carries no secrets
416
+
type ReserveSeat struct {
417
+
state protoimpl.MessageState `protogen:"open.v1"`
418
+
LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"`
419
+
TargetEngine string `protobuf:"bytes,2,opt,name=target_engine,json=targetEngine,proto3" json:"target_engine,omitempty"`
420
+
RawPipelineJson string `protobuf:"bytes,3,opt,name=raw_pipeline_json,json=rawPipelineJson,proto3" json:"raw_pipeline_json,omitempty"`
421
+
RawWorkflowJson string `protobuf:"bytes,4,opt,name=raw_workflow_json,json=rawWorkflowJson,proto3" json:"raw_workflow_json,omitempty"`
422
+
// pipeline id, the executor reconstructs the exact WorkflowId from it
423
+
Knot string `protobuf:"bytes,5,opt,name=knot,proto3" json:"knot,omitempty"`
424
+
Rkey string `protobuf:"bytes,6,opt,name=rkey,proto3" json:"rkey,omitempty"`
425
+
TtlSeconds uint32 `protobuf:"varint,7,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"`
426
+
unknownFields protoimpl.UnknownFields
427
+
sizeCache protoimpl.SizeCache
428
+
}
429
+
430
+
func (x *ReserveSeat) Reset() {
431
+
*x = ReserveSeat{}
432
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[4]
433
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
434
+
ms.StoreMessageInfo(mi)
435
+
}
436
+
437
+
func (x *ReserveSeat) String() string {
438
+
return protoimpl.X.MessageStringOf(x)
439
+
}
440
+
441
+
func (*ReserveSeat) ProtoMessage() {}
442
+
443
+
func (x *ReserveSeat) ProtoReflect() protoreflect.Message {
444
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[4]
445
+
if x != nil {
446
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
447
+
if ms.LoadMessageInfo() == nil {
448
+
ms.StoreMessageInfo(mi)
449
+
}
450
+
return ms
451
+
}
452
+
return mi.MessageOf(x)
453
+
}
454
+
455
+
// Deprecated: Use ReserveSeat.ProtoReflect.Descriptor instead.
456
+
func (*ReserveSeat) Descriptor() ([]byte, []int) {
457
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{4}
458
+
}
459
+
460
+
func (x *ReserveSeat) GetLeaseId() string {
461
+
if x != nil {
462
+
return x.LeaseId
463
+
}
464
+
return ""
465
+
}
466
+
467
+
func (x *ReserveSeat) GetTargetEngine() string {
468
+
if x != nil {
469
+
return x.TargetEngine
470
+
}
471
+
return ""
472
+
}
473
+
474
+
func (x *ReserveSeat) GetRawPipelineJson() string {
475
+
if x != nil {
476
+
return x.RawPipelineJson
477
+
}
478
+
return ""
479
+
}
480
+
481
+
func (x *ReserveSeat) GetRawWorkflowJson() string {
482
+
if x != nil {
483
+
return x.RawWorkflowJson
484
+
}
485
+
return ""
486
+
}
487
+
488
+
func (x *ReserveSeat) GetKnot() string {
489
+
if x != nil {
490
+
return x.Knot
491
+
}
492
+
return ""
493
+
}
494
+
495
+
func (x *ReserveSeat) GetRkey() string {
496
+
if x != nil {
497
+
return x.Rkey
498
+
}
499
+
return ""
500
+
}
501
+
502
+
func (x *ReserveSeat) GetTtlSeconds() uint32 {
503
+
if x != nil {
504
+
return x.TtlSeconds
505
+
}
506
+
return 0
507
+
}
508
+
509
+
type ReserveResult struct {
510
+
state protoimpl.MessageState `protogen:"open.v1"`
511
+
LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"`
512
+
Accepted bool `protobuf:"varint,2,opt,name=accepted,proto3" json:"accepted,omitempty"`
513
+
RejectReason string `protobuf:"bytes,3,opt,name=reject_reason,json=rejectReason,proto3" json:"reject_reason,omitempty"`
514
+
RejectClass RejectClass `protobuf:"varint,4,opt,name=reject_class,json=rejectClass,proto3,enum=spindle.mill.v1.RejectClass" json:"reject_class,omitempty"`
515
+
unknownFields protoimpl.UnknownFields
516
+
sizeCache protoimpl.SizeCache
517
+
}
518
+
519
+
func (x *ReserveResult) Reset() {
520
+
*x = ReserveResult{}
521
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[5]
522
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
523
+
ms.StoreMessageInfo(mi)
524
+
}
525
+
526
+
func (x *ReserveResult) String() string {
527
+
return protoimpl.X.MessageStringOf(x)
528
+
}
529
+
530
+
func (*ReserveResult) ProtoMessage() {}
531
+
532
+
func (x *ReserveResult) ProtoReflect() protoreflect.Message {
533
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[5]
534
+
if x != nil {
535
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
536
+
if ms.LoadMessageInfo() == nil {
537
+
ms.StoreMessageInfo(mi)
538
+
}
539
+
return ms
540
+
}
541
+
return mi.MessageOf(x)
542
+
}
543
+
544
+
// Deprecated: Use ReserveResult.ProtoReflect.Descriptor instead.
545
+
func (*ReserveResult) Descriptor() ([]byte, []int) {
546
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{5}
547
+
}
548
+
549
+
func (x *ReserveResult) GetLeaseId() string {
550
+
if x != nil {
551
+
return x.LeaseId
552
+
}
553
+
return ""
554
+
}
555
+
556
+
func (x *ReserveResult) GetAccepted() bool {
557
+
if x != nil {
558
+
return x.Accepted
559
+
}
560
+
return false
561
+
}
562
+
563
+
func (x *ReserveResult) GetRejectReason() string {
564
+
if x != nil {
565
+
return x.RejectReason
566
+
}
567
+
return ""
568
+
}
569
+
570
+
func (x *ReserveResult) GetRejectClass() RejectClass {
571
+
if x != nil {
572
+
return x.RejectClass
573
+
}
574
+
return RejectClass_REJECT_CLASS_UNSPECIFIED
575
+
}
576
+
577
+
// a single unlocked secret
578
+
type Secret struct {
579
+
state protoimpl.MessageState `protogen:"open.v1"`
580
+
Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
581
+
Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
582
+
unknownFields protoimpl.UnknownFields
583
+
sizeCache protoimpl.SizeCache
584
+
}
585
+
586
+
func (x *Secret) Reset() {
587
+
*x = Secret{}
588
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[6]
589
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
590
+
ms.StoreMessageInfo(mi)
591
+
}
592
+
593
+
func (x *Secret) String() string {
594
+
return protoimpl.X.MessageStringOf(x)
595
+
}
596
+
597
+
func (*Secret) ProtoMessage() {}
598
+
599
+
func (x *Secret) ProtoReflect() protoreflect.Message {
600
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[6]
601
+
if x != nil {
602
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
603
+
if ms.LoadMessageInfo() == nil {
604
+
ms.StoreMessageInfo(mi)
605
+
}
606
+
return ms
607
+
}
608
+
return mi.MessageOf(x)
609
+
}
610
+
611
+
// Deprecated: Use Secret.ProtoReflect.Descriptor instead.
612
+
func (*Secret) Descriptor() ([]byte, []int) {
613
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{6}
614
+
}
615
+
616
+
func (x *Secret) GetKey() string {
617
+
if x != nil {
618
+
return x.Key
619
+
}
620
+
return ""
621
+
}
622
+
623
+
func (x *Secret) GetValue() string {
624
+
if x != nil {
625
+
return x.Value
626
+
}
627
+
return ""
628
+
}
629
+
630
+
// promotes a reservation to a running job and hands over the secrets
631
+
type CommitLease struct {
632
+
state protoimpl.MessageState `protogen:"open.v1"`
633
+
LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"`
634
+
Secrets []*Secret `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty"`
635
+
unknownFields protoimpl.UnknownFields
636
+
sizeCache protoimpl.SizeCache
637
+
}
638
+
639
+
func (x *CommitLease) Reset() {
640
+
*x = CommitLease{}
641
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[7]
642
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
643
+
ms.StoreMessageInfo(mi)
644
+
}
645
+
646
+
func (x *CommitLease) String() string {
647
+
return protoimpl.X.MessageStringOf(x)
648
+
}
649
+
650
+
func (*CommitLease) ProtoMessage() {}
651
+
652
+
func (x *CommitLease) ProtoReflect() protoreflect.Message {
653
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[7]
654
+
if x != nil {
655
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
656
+
if ms.LoadMessageInfo() == nil {
657
+
ms.StoreMessageInfo(mi)
658
+
}
659
+
return ms
660
+
}
661
+
return mi.MessageOf(x)
662
+
}
663
+
664
+
// Deprecated: Use CommitLease.ProtoReflect.Descriptor instead.
665
+
func (*CommitLease) Descriptor() ([]byte, []int) {
666
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{7}
667
+
}
668
+
669
+
func (x *CommitLease) GetLeaseId() string {
670
+
if x != nil {
671
+
return x.LeaseId
672
+
}
673
+
return ""
674
+
}
675
+
676
+
func (x *CommitLease) GetSecrets() []*Secret {
677
+
if x != nil {
678
+
return x.Secrets
679
+
}
680
+
return nil
681
+
}
682
+
683
+
type Committed struct {
684
+
state protoimpl.MessageState `protogen:"open.v1"`
685
+
LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"`
686
+
unknownFields protoimpl.UnknownFields
687
+
sizeCache protoimpl.SizeCache
688
+
}
689
+
690
+
func (x *Committed) Reset() {
691
+
*x = Committed{}
692
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[8]
693
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
694
+
ms.StoreMessageInfo(mi)
695
+
}
696
+
697
+
func (x *Committed) String() string {
698
+
return protoimpl.X.MessageStringOf(x)
699
+
}
700
+
701
+
func (*Committed) ProtoMessage() {}
702
+
703
+
func (x *Committed) ProtoReflect() protoreflect.Message {
704
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[8]
705
+
if x != nil {
706
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
707
+
if ms.LoadMessageInfo() == nil {
708
+
ms.StoreMessageInfo(mi)
709
+
}
710
+
return ms
711
+
}
712
+
return mi.MessageOf(x)
713
+
}
714
+
715
+
// Deprecated: Use Committed.ProtoReflect.Descriptor instead.
716
+
func (*Committed) Descriptor() ([]byte, []int) {
717
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{8}
718
+
}
719
+
720
+
func (x *Committed) GetLeaseId() string {
721
+
if x != nil {
722
+
return x.LeaseId
723
+
}
724
+
return ""
725
+
}
726
+
727
+
// drops a reservation that was never committed
728
+
type ReleaseLease struct {
729
+
state protoimpl.MessageState `protogen:"open.v1"`
730
+
LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"`
731
+
unknownFields protoimpl.UnknownFields
732
+
sizeCache protoimpl.SizeCache
733
+
}
734
+
735
+
func (x *ReleaseLease) Reset() {
736
+
*x = ReleaseLease{}
737
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[9]
738
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
739
+
ms.StoreMessageInfo(mi)
740
+
}
741
+
742
+
func (x *ReleaseLease) String() string {
743
+
return protoimpl.X.MessageStringOf(x)
744
+
}
745
+
746
+
func (*ReleaseLease) ProtoMessage() {}
747
+
748
+
func (x *ReleaseLease) ProtoReflect() protoreflect.Message {
749
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[9]
750
+
if x != nil {
751
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
752
+
if ms.LoadMessageInfo() == nil {
753
+
ms.StoreMessageInfo(mi)
754
+
}
755
+
return ms
756
+
}
757
+
return mi.MessageOf(x)
758
+
}
759
+
760
+
// Deprecated: Use ReleaseLease.ProtoReflect.Descriptor instead.
761
+
func (*ReleaseLease) Descriptor() ([]byte, []int) {
762
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{9}
763
+
}
764
+
765
+
func (x *ReleaseLease) GetLeaseId() string {
766
+
if x != nil {
767
+
return x.LeaseId
768
+
}
769
+
return ""
770
+
}
771
+
772
+
// cancels a running attempt
773
+
type CancelAttempt struct {
774
+
state protoimpl.MessageState `protogen:"open.v1"`
775
+
LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"`
776
+
Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"`
777
+
unknownFields protoimpl.UnknownFields
778
+
sizeCache protoimpl.SizeCache
779
+
}
780
+
781
+
func (x *CancelAttempt) Reset() {
782
+
*x = CancelAttempt{}
783
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[10]
784
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
785
+
ms.StoreMessageInfo(mi)
786
+
}
787
+
788
+
func (x *CancelAttempt) String() string {
789
+
return protoimpl.X.MessageStringOf(x)
790
+
}
791
+
792
+
func (*CancelAttempt) ProtoMessage() {}
793
+
794
+
func (x *CancelAttempt) ProtoReflect() protoreflect.Message {
795
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[10]
796
+
if x != nil {
797
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
798
+
if ms.LoadMessageInfo() == nil {
799
+
ms.StoreMessageInfo(mi)
800
+
}
801
+
return ms
802
+
}
803
+
return mi.MessageOf(x)
804
+
}
805
+
806
+
// Deprecated: Use CancelAttempt.ProtoReflect.Descriptor instead.
807
+
func (*CancelAttempt) Descriptor() ([]byte, []int) {
808
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{10}
809
+
}
810
+
811
+
func (x *CancelAttempt) GetLeaseId() string {
812
+
if x != nil {
813
+
return x.LeaseId
814
+
}
815
+
return ""
816
+
}
817
+
818
+
func (x *CancelAttempt) GetReason() string {
819
+
if x != nil {
820
+
return x.Reason
821
+
}
822
+
return ""
823
+
}
824
+
825
+
type CancelAck struct {
826
+
state protoimpl.MessageState `protogen:"open.v1"`
827
+
LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"`
828
+
unknownFields protoimpl.UnknownFields
829
+
sizeCache protoimpl.SizeCache
830
+
}
831
+
832
+
func (x *CancelAck) Reset() {
833
+
*x = CancelAck{}
834
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[11]
835
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
836
+
ms.StoreMessageInfo(mi)
837
+
}
838
+
839
+
func (x *CancelAck) String() string {
840
+
return protoimpl.X.MessageStringOf(x)
841
+
}
842
+
843
+
func (*CancelAck) ProtoMessage() {}
844
+
845
+
func (x *CancelAck) ProtoReflect() protoreflect.Message {
846
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[11]
847
+
if x != nil {
848
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
849
+
if ms.LoadMessageInfo() == nil {
850
+
ms.StoreMessageInfo(mi)
851
+
}
852
+
return ms
853
+
}
854
+
return mi.MessageOf(x)
855
+
}
856
+
857
+
// Deprecated: Use CancelAck.ProtoReflect.Descriptor instead.
858
+
func (*CancelAck) Descriptor() ([]byte, []int) {
859
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{11}
860
+
}
861
+
862
+
func (x *CancelAck) GetLeaseId() string {
863
+
if x != nil {
864
+
return x.LeaseId
865
+
}
866
+
return ""
867
+
}
868
+
869
+
type StatusEvent struct {
870
+
state protoimpl.MessageState `protogen:"open.v1"`
871
+
Status NonterminalStatus `protobuf:"varint,1,opt,name=status,proto3,enum=spindle.mill.v1.NonterminalStatus" json:"status,omitempty"`
872
+
Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
873
+
ExitCode int64 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"`
874
+
unknownFields protoimpl.UnknownFields
875
+
sizeCache protoimpl.SizeCache
876
+
}
877
+
878
+
func (x *StatusEvent) Reset() {
879
+
*x = StatusEvent{}
880
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[12]
881
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
882
+
ms.StoreMessageInfo(mi)
883
+
}
884
+
885
+
func (x *StatusEvent) String() string {
886
+
return protoimpl.X.MessageStringOf(x)
887
+
}
888
+
889
+
func (*StatusEvent) ProtoMessage() {}
890
+
891
+
func (x *StatusEvent) ProtoReflect() protoreflect.Message {
892
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[12]
893
+
if x != nil {
894
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
895
+
if ms.LoadMessageInfo() == nil {
896
+
ms.StoreMessageInfo(mi)
897
+
}
898
+
return ms
899
+
}
900
+
return mi.MessageOf(x)
901
+
}
902
+
903
+
// Deprecated: Use StatusEvent.ProtoReflect.Descriptor instead.
904
+
func (*StatusEvent) Descriptor() ([]byte, []int) {
905
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{12}
906
+
}
907
+
908
+
func (x *StatusEvent) GetStatus() NonterminalStatus {
909
+
if x != nil {
910
+
return x.Status
911
+
}
912
+
return NonterminalStatus_NONTERMINAL_STATUS_UNSPECIFIED
913
+
}
914
+
915
+
func (x *StatusEvent) GetError() string {
916
+
if x != nil {
917
+
return x.Error
918
+
}
919
+
return ""
920
+
}
921
+
922
+
func (x *StatusEvent) GetExitCode() int64 {
923
+
if x != nil {
924
+
return x.ExitCode
925
+
}
926
+
return 0
927
+
}
928
+
929
+
// one already-encoded models.LogLine JSON line
930
+
type LogLine struct {
931
+
state protoimpl.MessageState `protogen:"open.v1"`
932
+
RawJson []byte `protobuf:"bytes,1,opt,name=raw_json,json=rawJson,proto3" json:"raw_json,omitempty"`
933
+
unknownFields protoimpl.UnknownFields
934
+
sizeCache protoimpl.SizeCache
935
+
}
936
+
937
+
func (x *LogLine) Reset() {
938
+
*x = LogLine{}
939
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[13]
940
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
941
+
ms.StoreMessageInfo(mi)
942
+
}
943
+
944
+
func (x *LogLine) String() string {
945
+
return protoimpl.X.MessageStringOf(x)
946
+
}
947
+
948
+
func (*LogLine) ProtoMessage() {}
949
+
950
+
func (x *LogLine) ProtoReflect() protoreflect.Message {
951
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[13]
952
+
if x != nil {
953
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
954
+
if ms.LoadMessageInfo() == nil {
955
+
ms.StoreMessageInfo(mi)
956
+
}
957
+
return ms
958
+
}
959
+
return mi.MessageOf(x)
960
+
}
961
+
962
+
// Deprecated: Use LogLine.ProtoReflect.Descriptor instead.
963
+
func (*LogLine) Descriptor() ([]byte, []int) {
964
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{13}
965
+
}
966
+
967
+
func (x *LogLine) GetRawJson() []byte {
968
+
if x != nil {
969
+
return x.RawJson
970
+
}
971
+
return nil
972
+
}
973
+
974
+
// terminal outcome of an attempt
975
+
type AttemptResult struct {
976
+
state protoimpl.MessageState `protogen:"open.v1"`
977
+
Status TerminalStatus `protobuf:"varint,1,opt,name=status,proto3,enum=spindle.mill.v1.TerminalStatus" json:"status,omitempty"`
978
+
Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
979
+
ExitCode int64 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"`
980
+
unknownFields protoimpl.UnknownFields
981
+
sizeCache protoimpl.SizeCache
982
+
}
983
+
984
+
func (x *AttemptResult) Reset() {
985
+
*x = AttemptResult{}
986
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[14]
987
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
988
+
ms.StoreMessageInfo(mi)
989
+
}
990
+
991
+
func (x *AttemptResult) String() string {
992
+
return protoimpl.X.MessageStringOf(x)
993
+
}
994
+
995
+
func (*AttemptResult) ProtoMessage() {}
996
+
997
+
func (x *AttemptResult) ProtoReflect() protoreflect.Message {
998
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[14]
999
+
if x != nil {
1000
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
1001
+
if ms.LoadMessageInfo() == nil {
1002
+
ms.StoreMessageInfo(mi)
1003
+
}
1004
+
return ms
1005
+
}
1006
+
return mi.MessageOf(x)
1007
+
}
1008
+
1009
+
// Deprecated: Use AttemptResult.ProtoReflect.Descriptor instead.
1010
+
func (*AttemptResult) Descriptor() ([]byte, []int) {
1011
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{14}
1012
+
}
1013
+
1014
+
func (x *AttemptResult) GetStatus() TerminalStatus {
1015
+
if x != nil {
1016
+
return x.Status
1017
+
}
1018
+
return TerminalStatus_TERMINAL_STATUS_UNSPECIFIED
1019
+
}
1020
+
1021
+
func (x *AttemptResult) GetError() string {
1022
+
if x != nil {
1023
+
return x.Error
1024
+
}
1025
+
return ""
1026
+
}
1027
+
1028
+
func (x *AttemptResult) GetExitCode() int64 {
1029
+
if x != nil {
1030
+
return x.ExitCode
1031
+
}
1032
+
return 0
1033
+
}
1034
+
1035
+
// one event in the executor's stream back to the mill
1036
+
type Event struct {
1037
+
state protoimpl.MessageState `protogen:"open.v1"`
1038
+
Seqno uint64 `protobuf:"varint,1,opt,name=seqno,proto3" json:"seqno,omitempty"`
1039
+
LeaseId string `protobuf:"bytes,2,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"`
1040
+
// Types that are valid to be assigned to Payload:
1041
+
//
1042
+
// *Event_StatusEvent
1043
+
// *Event_LogLine
1044
+
// *Event_AttemptResult
1045
+
Payload isEvent_Payload `protobuf_oneof:"payload"`
1046
+
unknownFields protoimpl.UnknownFields
1047
+
sizeCache protoimpl.SizeCache
1048
+
}
1049
+
1050
+
func (x *Event) Reset() {
1051
+
*x = Event{}
1052
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[15]
1053
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
1054
+
ms.StoreMessageInfo(mi)
1055
+
}
1056
+
1057
+
func (x *Event) String() string {
1058
+
return protoimpl.X.MessageStringOf(x)
1059
+
}
1060
+
1061
+
func (*Event) ProtoMessage() {}
1062
+
1063
+
func (x *Event) ProtoReflect() protoreflect.Message {
1064
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[15]
1065
+
if x != nil {
1066
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
1067
+
if ms.LoadMessageInfo() == nil {
1068
+
ms.StoreMessageInfo(mi)
1069
+
}
1070
+
return ms
1071
+
}
1072
+
return mi.MessageOf(x)
1073
+
}
1074
+
1075
+
// Deprecated: Use Event.ProtoReflect.Descriptor instead.
1076
+
func (*Event) Descriptor() ([]byte, []int) {
1077
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{15}
1078
+
}
1079
+
1080
+
func (x *Event) GetSeqno() uint64 {
1081
+
if x != nil {
1082
+
return x.Seqno
1083
+
}
1084
+
return 0
1085
+
}
1086
+
1087
+
func (x *Event) GetLeaseId() string {
1088
+
if x != nil {
1089
+
return x.LeaseId
1090
+
}
1091
+
return ""
1092
+
}
1093
+
1094
+
func (x *Event) GetPayload() isEvent_Payload {
1095
+
if x != nil {
1096
+
return x.Payload
1097
+
}
1098
+
return nil
1099
+
}
1100
+
1101
+
func (x *Event) GetStatusEvent() *StatusEvent {
1102
+
if x != nil {
1103
+
if x, ok := x.Payload.(*Event_StatusEvent); ok {
1104
+
return x.StatusEvent
1105
+
}
1106
+
}
1107
+
return nil
1108
+
}
1109
+
1110
+
func (x *Event) GetLogLine() *LogLine {
1111
+
if x != nil {
1112
+
if x, ok := x.Payload.(*Event_LogLine); ok {
1113
+
return x.LogLine
1114
+
}
1115
+
}
1116
+
return nil
1117
+
}
1118
+
1119
+
func (x *Event) GetAttemptResult() *AttemptResult {
1120
+
if x != nil {
1121
+
if x, ok := x.Payload.(*Event_AttemptResult); ok {
1122
+
return x.AttemptResult
1123
+
}
1124
+
}
1125
+
return nil
1126
+
}
1127
+
1128
+
type isEvent_Payload interface {
1129
+
isEvent_Payload()
1130
+
}
1131
+
1132
+
type Event_StatusEvent struct {
1133
+
StatusEvent *StatusEvent `protobuf:"bytes,3,opt,name=status_event,json=statusEvent,proto3,oneof"`
1134
+
}
1135
+
1136
+
type Event_LogLine struct {
1137
+
LogLine *LogLine `protobuf:"bytes,4,opt,name=log_line,json=logLine,proto3,oneof"`
1138
+
}
1139
+
1140
+
type Event_AttemptResult struct {
1141
+
AttemptResult *AttemptResult `protobuf:"bytes,5,opt,name=attempt_result,json=attemptResult,proto3,oneof"`
1142
+
}
1143
+
1144
+
func (*Event_StatusEvent) isEvent_Payload() {}
1145
+
1146
+
func (*Event_LogLine) isEvent_Payload() {}
1147
+
1148
+
func (*Event_AttemptResult) isEvent_Payload() {}
1149
+
1150
+
// a flushed bundle of events
1151
+
type EventBatch struct {
1152
+
state protoimpl.MessageState `protogen:"open.v1"`
1153
+
Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"`
1154
+
Events []*Event `protobuf:"bytes,2,rep,name=events,proto3" json:"events,omitempty"`
1155
+
unknownFields protoimpl.UnknownFields
1156
+
sizeCache protoimpl.SizeCache
1157
+
}
1158
+
1159
+
func (x *EventBatch) Reset() {
1160
+
*x = EventBatch{}
1161
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[16]
1162
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
1163
+
ms.StoreMessageInfo(mi)
1164
+
}
1165
+
1166
+
func (x *EventBatch) String() string {
1167
+
return protoimpl.X.MessageStringOf(x)
1168
+
}
1169
+
1170
+
func (*EventBatch) ProtoMessage() {}
1171
+
1172
+
func (x *EventBatch) ProtoReflect() protoreflect.Message {
1173
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[16]
1174
+
if x != nil {
1175
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
1176
+
if ms.LoadMessageInfo() == nil {
1177
+
ms.StoreMessageInfo(mi)
1178
+
}
1179
+
return ms
1180
+
}
1181
+
return mi.MessageOf(x)
1182
+
}
1183
+
1184
+
// Deprecated: Use EventBatch.ProtoReflect.Descriptor instead.
1185
+
func (*EventBatch) Descriptor() ([]byte, []int) {
1186
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16}
1187
+
}
1188
+
1189
+
func (x *EventBatch) GetEpoch() string {
1190
+
if x != nil {
1191
+
return x.Epoch
1192
+
}
1193
+
return ""
1194
+
}
1195
+
1196
+
func (x *EventBatch) GetEvents() []*Event {
1197
+
if x != nil {
1198
+
return x.Events
1199
+
}
1200
+
return nil
1201
+
}
1202
+
1203
+
// all events up to and including up_to_seqno are durably processed
1204
+
type Ack struct {
1205
+
state protoimpl.MessageState `protogen:"open.v1"`
1206
+
Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"`
1207
+
UpToSeqno uint64 `protobuf:"varint,2,opt,name=up_to_seqno,json=upToSeqno,proto3" json:"up_to_seqno,omitempty"`
1208
+
unknownFields protoimpl.UnknownFields
1209
+
sizeCache protoimpl.SizeCache
1210
+
}
1211
+
1212
+
func (x *Ack) Reset() {
1213
+
*x = Ack{}
1214
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[17]
1215
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
1216
+
ms.StoreMessageInfo(mi)
1217
+
}
1218
+
1219
+
func (x *Ack) String() string {
1220
+
return protoimpl.X.MessageStringOf(x)
1221
+
}
1222
+
1223
+
func (*Ack) ProtoMessage() {}
1224
+
1225
+
func (x *Ack) ProtoReflect() protoreflect.Message {
1226
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[17]
1227
+
if x != nil {
1228
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
1229
+
if ms.LoadMessageInfo() == nil {
1230
+
ms.StoreMessageInfo(mi)
1231
+
}
1232
+
return ms
1233
+
}
1234
+
return mi.MessageOf(x)
1235
+
}
1236
+
1237
+
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
1238
+
func (*Ack) Descriptor() ([]byte, []int) {
1239
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17}
1240
+
}
1241
+
1242
+
func (x *Ack) GetEpoch() string {
1243
+
if x != nil {
1244
+
return x.Epoch
1245
+
}
1246
+
return ""
1247
+
}
1248
+
1249
+
func (x *Ack) GetUpToSeqno() uint64 {
1250
+
if x != nil {
1251
+
return x.UpToSeqno
1252
+
}
1253
+
return 0
1254
+
}
1255
+
1256
+
type Message struct {
1257
+
state protoimpl.MessageState `protogen:"open.v1"`
1258
+
Hello *Hello `protobuf:"bytes,1,opt,name=hello,proto3" json:"hello,omitempty"`
1259
+
Resume *Resume `protobuf:"bytes,2,opt,name=resume,proto3" json:"resume,omitempty"`
1260
+
NodeSnapshot *NodeSnapshot `protobuf:"bytes,3,opt,name=node_snapshot,json=nodeSnapshot,proto3" json:"node_snapshot,omitempty"`
1261
+
ReserveSeat *ReserveSeat `protobuf:"bytes,4,opt,name=reserve_seat,json=reserveSeat,proto3" json:"reserve_seat,omitempty"`
1262
+
ReserveResult *ReserveResult `protobuf:"bytes,5,opt,name=reserve_result,json=reserveResult,proto3" json:"reserve_result,omitempty"`
1263
+
CommitLease *CommitLease `protobuf:"bytes,6,opt,name=commit_lease,json=commitLease,proto3" json:"commit_lease,omitempty"`
1264
+
Committed *Committed `protobuf:"bytes,7,opt,name=committed,proto3" json:"committed,omitempty"`
1265
+
ReleaseLease *ReleaseLease `protobuf:"bytes,8,opt,name=release_lease,json=releaseLease,proto3" json:"release_lease,omitempty"`
1266
+
CancelAttempt *CancelAttempt `protobuf:"bytes,9,opt,name=cancel_attempt,json=cancelAttempt,proto3" json:"cancel_attempt,omitempty"`
1267
+
CancelAck *CancelAck `protobuf:"bytes,10,opt,name=cancel_ack,json=cancelAck,proto3" json:"cancel_ack,omitempty"`
1268
+
EventBatch *EventBatch `protobuf:"bytes,11,opt,name=event_batch,json=eventBatch,proto3" json:"event_batch,omitempty"`
1269
+
Ack *Ack `protobuf:"bytes,12,opt,name=ack,proto3" json:"ack,omitempty"`
1270
+
unknownFields protoimpl.UnknownFields
1271
+
sizeCache protoimpl.SizeCache
1272
+
}
1273
+
1274
+
func (x *Message) Reset() {
1275
+
*x = Message{}
1276
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[18]
1277
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
1278
+
ms.StoreMessageInfo(mi)
1279
+
}
1280
+
1281
+
func (x *Message) String() string {
1282
+
return protoimpl.X.MessageStringOf(x)
1283
+
}
1284
+
1285
+
func (*Message) ProtoMessage() {}
1286
+
1287
+
func (x *Message) ProtoReflect() protoreflect.Message {
1288
+
mi := &file_spindle_mill_v1_mill_proto_msgTypes[18]
1289
+
if x != nil {
1290
+
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
1291
+
if ms.LoadMessageInfo() == nil {
1292
+
ms.StoreMessageInfo(mi)
1293
+
}
1294
+
return ms
1295
+
}
1296
+
return mi.MessageOf(x)
1297
+
}
1298
+
1299
+
// Deprecated: Use Message.ProtoReflect.Descriptor instead.
1300
+
func (*Message) Descriptor() ([]byte, []int) {
1301
+
return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18}
1302
+
}
1303
+
1304
+
func (x *Message) GetHello() *Hello {
1305
+
if x != nil {
1306
+
return x.Hello
1307
+
}
1308
+
return nil
1309
+
}
1310
+
1311
+
func (x *Message) GetResume() *Resume {
1312
+
if x != nil {
1313
+
return x.Resume
1314
+
}
1315
+
return nil
1316
+
}
1317
+
1318
+
func (x *Message) GetNodeSnapshot() *NodeSnapshot {
1319
+
if x != nil {
1320
+
return x.NodeSnapshot
1321
+
}
1322
+
return nil
1323
+
}
1324
+
1325
+
func (x *Message) GetReserveSeat() *ReserveSeat {
1326
+
if x != nil {
1327
+
return x.ReserveSeat
1328
+
}
1329
+
return nil
1330
+
}
1331
+
1332
+
func (x *Message) GetReserveResult() *ReserveResult {
1333
+
if x != nil {
1334
+
return x.ReserveResult
1335
+
}
1336
+
return nil
1337
+
}
1338
+
1339
+
func (x *Message) GetCommitLease() *CommitLease {
1340
+
if x != nil {
1341
+
return x.CommitLease
1342
+
}
1343
+
return nil
1344
+
}
1345
+
1346
+
func (x *Message) GetCommitted() *Committed {
1347
+
if x != nil {
1348
+
return x.Committed
1349
+
}
1350
+
return nil
1351
+
}
1352
+
1353
+
func (x *Message) GetReleaseLease() *ReleaseLease {
1354
+
if x != nil {
1355
+
return x.ReleaseLease
1356
+
}
1357
+
return nil
1358
+
}
1359
+
1360
+
func (x *Message) GetCancelAttempt() *CancelAttempt {
1361
+
if x != nil {
1362
+
return x.CancelAttempt
1363
+
}
1364
+
return nil
1365
+
}
1366
+
1367
+
func (x *Message) GetCancelAck() *CancelAck {
1368
+
if x != nil {
1369
+
return x.CancelAck
1370
+
}
1371
+
return nil
1372
+
}
1373
+
1374
+
func (x *Message) GetEventBatch() *EventBatch {
1375
+
if x != nil {
1376
+
return x.EventBatch
1377
+
}
1378
+
return nil
1379
+
}
1380
+
1381
+
func (x *Message) GetAck() *Ack {
1382
+
if x != nil {
1383
+
return x.Ack
1384
+
}
1385
+
return nil
1386
+
}
1387
+
1388
+
var File_spindle_mill_v1_mill_proto protoreflect.FileDescriptor
1389
+
1390
+
const file_spindle_mill_v1_mill_proto_rawDesc = "" +
1391
+
"\n" +
1392
+
"\x1aspindle/mill/v1/mill.proto\x12\x0fspindle.mill.v1\x1a\x1bbuf/validate/validate.proto\"}\n" +
1393
+
"\x05Hello\x12)\n" +
1394
+
"\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x12\x12\n" +
1395
+
"\x04arch\x18\x02 \x01(\tR\x04arch\x12\x16\n" +
1396
+
"\x06labels\x18\x03 \x03(\tR\x06labels\x12\x1d\n" +
1397
+
"\x05epoch\x18\x04 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\"D\n" +
1398
+
"\x06Resume\x12\x1d\n" +
1399
+
"\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1b\n" +
1400
+
"\tack_seqno\x18\x02 \x01(\x04R\backSeqno\"\xae\x01\n" +
1401
+
"\x12EngineAvailability\x12\x1c\n" +
1402
+
"\tavailable\x18\x01 \x01(\bR\tavailable\x12A\n" +
1403
+
"\x04load\x18\x02 \x03(\v2-.spindle.mill.v1.EngineAvailability.LoadEntryR\x04load\x1a7\n" +
1404
+
"\tLoadEntry\x12\x10\n" +
1405
+
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
1406
+
"\x05value\x18\x02 \x01(\x01R\x05value:\x028\x01\"\xfe\x01\n" +
1407
+
"\fNodeSnapshot\x12\x1d\n" +
1408
+
"\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12D\n" +
1409
+
"\aengines\x18\x02 \x03(\v2*.spindle.mill.v1.NodeSnapshot.EnginesEntryR\aengines\x12(\n" +
1410
+
"\x10active_lease_ids\x18\x03 \x03(\tR\x0eactiveLeaseIds\x1a_\n" +
1411
+
"\fEnginesEntry\x12\x10\n" +
1412
+
"\x03key\x18\x01 \x01(\tR\x03key\x129\n" +
1413
+
"\x05value\x18\x02 \x01(\v2#.spindle.mill.v1.EngineAvailabilityR\x05value:\x028\x01\"\x80\x02\n" +
1414
+
"\vReserveSeat\x12\"\n" +
1415
+
"\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12,\n" +
1416
+
"\rtarget_engine\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\ftargetEngine\x12*\n" +
1417
+
"\x11raw_pipeline_json\x18\x03 \x01(\tR\x0frawPipelineJson\x12*\n" +
1418
+
"\x11raw_workflow_json\x18\x04 \x01(\tR\x0frawWorkflowJson\x12\x12\n" +
1419
+
"\x04knot\x18\x05 \x01(\tR\x04knot\x12\x12\n" +
1420
+
"\x04rkey\x18\x06 \x01(\tR\x04rkey\x12\x1f\n" +
1421
+
"\vttl_seconds\x18\a \x01(\rR\n" +
1422
+
"ttlSeconds\"\xbf\x01\n" +
1423
+
"\rReserveResult\x12\"\n" +
1424
+
"\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x1a\n" +
1425
+
"\baccepted\x18\x02 \x01(\bR\baccepted\x12#\n" +
1426
+
"\rreject_reason\x18\x03 \x01(\tR\frejectReason\x12I\n" +
1427
+
"\freject_class\x18\x04 \x01(\x0e2\x1c.spindle.mill.v1.RejectClassB\b\xbaH\x05\x82\x01\x02\x10\x01R\vrejectClass\"0\n" +
1428
+
"\x06Secret\x12\x10\n" +
1429
+
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
1430
+
"\x05value\x18\x02 \x01(\tR\x05value\"d\n" +
1431
+
"\vCommitLease\x12\"\n" +
1432
+
"\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x121\n" +
1433
+
"\asecrets\x18\x02 \x03(\v2\x17.spindle.mill.v1.SecretR\asecrets\"/\n" +
1434
+
"\tCommitted\x12\"\n" +
1435
+
"\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"2\n" +
1436
+
"\fReleaseLease\x12\"\n" +
1437
+
"\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"K\n" +
1438
+
"\rCancelAttempt\x12\"\n" +
1439
+
"\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x16\n" +
1440
+
"\x06reason\x18\x02 \x01(\tR\x06reason\"/\n" +
1441
+
"\tCancelAck\x12\"\n" +
1442
+
"\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"\x93\x01\n" +
1443
+
"\vStatusEvent\x12F\n" +
1444
+
"\x06status\x18\x01 \x01(\x0e2\".spindle.mill.v1.NonterminalStatusB\n" +
1445
+
"\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" +
1446
+
"\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" +
1447
+
"\texit_code\x18\x03 \x01(\x03R\bexitCode\"-\n" +
1448
+
"\aLogLine\x12\"\n" +
1449
+
"\braw_json\x18\x01 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\x92\x01\n" +
1450
+
"\rAttemptResult\x12C\n" +
1451
+
"\x06status\x18\x01 \x01(\x0e2\x1f.spindle.mill.v1.TerminalStatusB\n" +
1452
+
"\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" +
1453
+
"\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" +
1454
+
"\texit_code\x18\x03 \x01(\x03R\bexitCode\"\x9f\x02\n" +
1455
+
"\x05Event\x12\x1d\n" +
1456
+
"\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12\"\n" +
1457
+
"\blease_id\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12A\n" +
1458
+
"\fstatus_event\x18\x03 \x01(\v2\x1c.spindle.mill.v1.StatusEventH\x00R\vstatusEvent\x125\n" +
1459
+
"\blog_line\x18\x04 \x01(\v2\x18.spindle.mill.v1.LogLineH\x00R\alogLine\x12G\n" +
1460
+
"\x0eattempt_result\x18\x05 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResultB\x10\n" +
1461
+
"\apayload\x12\x05\xbaH\x02\b\x01\"e\n" +
1462
+
"\n" +
1463
+
"EventBatch\x12\x1d\n" +
1464
+
"\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x128\n" +
1465
+
"\x06events\x18\x02 \x03(\v2\x16.spindle.mill.v1.EventB\b\xbaH\x05\x92\x01\x02\b\x01R\x06events\"D\n" +
1466
+
"\x03Ack\x12\x1d\n" +
1467
+
"\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1e\n" +
1468
+
"\vup_to_seqno\x18\x02 \x01(\x04R\tupToSeqno\"\xf9\x06\n" +
1469
+
"\aMessage\x12,\n" +
1470
+
"\x05hello\x18\x01 \x01(\v2\x16.spindle.mill.v1.HelloR\x05hello\x12/\n" +
1471
+
"\x06resume\x18\x02 \x01(\v2\x17.spindle.mill.v1.ResumeR\x06resume\x12B\n" +
1472
+
"\rnode_snapshot\x18\x03 \x01(\v2\x1d.spindle.mill.v1.NodeSnapshotR\fnodeSnapshot\x12?\n" +
1473
+
"\freserve_seat\x18\x04 \x01(\v2\x1c.spindle.mill.v1.ReserveSeatR\vreserveSeat\x12E\n" +
1474
+
"\x0ereserve_result\x18\x05 \x01(\v2\x1e.spindle.mill.v1.ReserveResultR\rreserveResult\x12?\n" +
1475
+
"\fcommit_lease\x18\x06 \x01(\v2\x1c.spindle.mill.v1.CommitLeaseR\vcommitLease\x128\n" +
1476
+
"\tcommitted\x18\a \x01(\v2\x1a.spindle.mill.v1.CommittedR\tcommitted\x12B\n" +
1477
+
"\rrelease_lease\x18\b \x01(\v2\x1d.spindle.mill.v1.ReleaseLeaseR\freleaseLease\x12E\n" +
1478
+
"\x0ecancel_attempt\x18\t \x01(\v2\x1e.spindle.mill.v1.CancelAttemptR\rcancelAttempt\x129\n" +
1479
+
"\n" +
1480
+
"cancel_ack\x18\n" +
1481
+
" \x01(\v2\x1a.spindle.mill.v1.CancelAckR\tcancelAck\x12<\n" +
1482
+
"\vevent_batch\x18\v \x01(\v2\x1b.spindle.mill.v1.EventBatchR\n" +
1483
+
"eventBatch\x12&\n" +
1484
+
"\x03ack\x18\f \x01(\v2\x14.spindle.mill.v1.AckR\x03ack:\x9b\x01\xbaH\x97\x01\"\x94\x01\n" +
1485
+
"\x05hello\n" +
1486
+
"\x06resume\n" +
1487
+
"\rnode_snapshot\n" +
1488
+
"\freserve_seat\n" +
1489
+
"\x0ereserve_result\n" +
1490
+
"\fcommit_lease\n" +
1491
+
"\tcommitted\n" +
1492
+
"\rrelease_lease\n" +
1493
+
"\x0ecancel_attempt\n" +
1494
+
"\n" +
1495
+
"cancel_ack\n" +
1496
+
"\vevent_batch\n" +
1497
+
"\x03ack\x10\x01*f\n" +
1498
+
"\vRejectClass\x12\x1c\n" +
1499
+
"\x18REJECT_CLASS_UNSPECIFIED\x10\x00\x12\x1a\n" +
1500
+
"\x16REJECT_CLASS_TRANSIENT\x10\x01\x12\x1d\n" +
1501
+
"\x19REJECT_CLASS_INCOMPATIBLE\x10\x02*D\n" +
1502
+
"\x11NonterminalStatus\x12\"\n" +
1503
+
"\x1eNONTERMINAL_STATUS_UNSPECIFIED\x10\x00\x12\v\n" +
1504
+
"\aRUNNING\x10\x01*f\n" +
1505
+
"\x0eTerminalStatus\x12\x1f\n" +
1506
+
"\x1bTERMINAL_STATUS_UNSPECIFIED\x10\x00\x12\v\n" +
1507
+
"\aSUCCESS\x10\x01\x12\n" +
1508
+
"\n" +
1509
+
"\x06FAILED\x10\x02\x12\v\n" +
1510
+
"\aTIMEOUT\x10\x03\x12\r\n" +
1511
+
"\tCANCELLED\x10\x04B0Z.tangled.org/core/spindle/mill/proto/gen;millv1b\x06proto3"
1512
+
1513
+
var (
1514
+
file_spindle_mill_v1_mill_proto_rawDescOnce sync.Once
1515
+
file_spindle_mill_v1_mill_proto_rawDescData []byte
1516
+
)
1517
+
1518
+
func file_spindle_mill_v1_mill_proto_rawDescGZIP() []byte {
1519
+
file_spindle_mill_v1_mill_proto_rawDescOnce.Do(func() {
1520
+
file_spindle_mill_v1_mill_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc)))
1521
+
})
1522
+
return file_spindle_mill_v1_mill_proto_rawDescData
1523
+
}
1524
+
1525
+
var file_spindle_mill_v1_mill_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
1526
+
var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 21)
1527
+
var file_spindle_mill_v1_mill_proto_goTypes = []any{
1528
+
(RejectClass)(0), // 0: spindle.mill.v1.RejectClass
1529
+
(NonterminalStatus)(0), // 1: spindle.mill.v1.NonterminalStatus
1530
+
(TerminalStatus)(0), // 2: spindle.mill.v1.TerminalStatus
1531
+
(*Hello)(nil), // 3: spindle.mill.v1.Hello
1532
+
(*Resume)(nil), // 4: spindle.mill.v1.Resume
1533
+
(*EngineAvailability)(nil), // 5: spindle.mill.v1.EngineAvailability
1534
+
(*NodeSnapshot)(nil), // 6: spindle.mill.v1.NodeSnapshot
1535
+
(*ReserveSeat)(nil), // 7: spindle.mill.v1.ReserveSeat
1536
+
(*ReserveResult)(nil), // 8: spindle.mill.v1.ReserveResult
1537
+
(*Secret)(nil), // 9: spindle.mill.v1.Secret
1538
+
(*CommitLease)(nil), // 10: spindle.mill.v1.CommitLease
1539
+
(*Committed)(nil), // 11: spindle.mill.v1.Committed
1540
+
(*ReleaseLease)(nil), // 12: spindle.mill.v1.ReleaseLease
1541
+
(*CancelAttempt)(nil), // 13: spindle.mill.v1.CancelAttempt
1542
+
(*CancelAck)(nil), // 14: spindle.mill.v1.CancelAck
1543
+
(*StatusEvent)(nil), // 15: spindle.mill.v1.StatusEvent
1544
+
(*LogLine)(nil), // 16: spindle.mill.v1.LogLine
1545
+
(*AttemptResult)(nil), // 17: spindle.mill.v1.AttemptResult
1546
+
(*Event)(nil), // 18: spindle.mill.v1.Event
1547
+
(*EventBatch)(nil), // 19: spindle.mill.v1.EventBatch
1548
+
(*Ack)(nil), // 20: spindle.mill.v1.Ack
1549
+
(*Message)(nil), // 21: spindle.mill.v1.Message
1550
+
nil, // 22: spindle.mill.v1.EngineAvailability.LoadEntry
1551
+
nil, // 23: spindle.mill.v1.NodeSnapshot.EnginesEntry
1552
+
}
1553
+
var file_spindle_mill_v1_mill_proto_depIdxs = []int32{
1554
+
22, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry
1555
+
23, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry
1556
+
0, // 2: spindle.mill.v1.ReserveResult.reject_class:type_name -> spindle.mill.v1.RejectClass
1557
+
9, // 3: spindle.mill.v1.CommitLease.secrets:type_name -> spindle.mill.v1.Secret
1558
+
1, // 4: spindle.mill.v1.StatusEvent.status:type_name -> spindle.mill.v1.NonterminalStatus
1559
+
2, // 5: spindle.mill.v1.AttemptResult.status:type_name -> spindle.mill.v1.TerminalStatus
1560
+
15, // 6: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent
1561
+
16, // 7: spindle.mill.v1.Event.log_line:type_name -> spindle.mill.v1.LogLine
1562
+
17, // 8: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult
1563
+
18, // 9: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event
1564
+
3, // 10: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello
1565
+
4, // 11: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume
1566
+
6, // 12: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot
1567
+
7, // 13: spindle.mill.v1.Message.reserve_seat:type_name -> spindle.mill.v1.ReserveSeat
1568
+
8, // 14: spindle.mill.v1.Message.reserve_result:type_name -> spindle.mill.v1.ReserveResult
1569
+
10, // 15: spindle.mill.v1.Message.commit_lease:type_name -> spindle.mill.v1.CommitLease
1570
+
11, // 16: spindle.mill.v1.Message.committed:type_name -> spindle.mill.v1.Committed
1571
+
12, // 17: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease
1572
+
13, // 18: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt
1573
+
14, // 19: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck
1574
+
19, // 20: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch
1575
+
20, // 21: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack
1576
+
5, // 22: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability
1577
+
23, // [23:23] is the sub-list for method output_type
1578
+
23, // [23:23] is the sub-list for method input_type
1579
+
23, // [23:23] is the sub-list for extension type_name
1580
+
23, // [23:23] is the sub-list for extension extendee
1581
+
0, // [0:23] is the sub-list for field type_name
1582
+
}
1583
+
1584
+
func init() { file_spindle_mill_v1_mill_proto_init() }
1585
+
func file_spindle_mill_v1_mill_proto_init() {
1586
+
if File_spindle_mill_v1_mill_proto != nil {
1587
+
return
1588
+
}
1589
+
file_spindle_mill_v1_mill_proto_msgTypes[15].OneofWrappers = []any{
1590
+
(*Event_StatusEvent)(nil),
1591
+
(*Event_LogLine)(nil),
1592
+
(*Event_AttemptResult)(nil),
1593
+
}
1594
+
type x struct{}
1595
+
out := protoimpl.TypeBuilder{
1596
+
File: protoimpl.DescBuilder{
1597
+
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
1598
+
RawDescriptor: unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc)),
1599
+
NumEnums: 3,
1600
+
NumMessages: 21,
1601
+
NumExtensions: 0,
1602
+
NumServices: 0,
1603
+
},
1604
+
GoTypes: file_spindle_mill_v1_mill_proto_goTypes,
1605
+
DependencyIndexes: file_spindle_mill_v1_mill_proto_depIdxs,
1606
+
EnumInfos: file_spindle_mill_v1_mill_proto_enumTypes,
1607
+
MessageInfos: file_spindle_mill_v1_mill_proto_msgTypes,
1608
+
}.Build()
1609
+
File_spindle_mill_v1_mill_proto = out.File
1610
+
file_spindle_mill_v1_mill_proto_goTypes = nil
1611
+
file_spindle_mill_v1_mill_proto_depIdxs = nil
1612
+
}
+103
spindle/mill/proto/protocol.go
+103
spindle/mill/proto/protocol.go
···
1
+
// package millproto carries the mill<->executor session protocol. a new message
2
+
// vocabulary over the same length-prefixed protobuf framing the spindle already
3
+
// uses to talk to the microVM guest (see spindle/agentproto). only the framing
4
+
// pattern is shared. the messages are entirely separate
5
+
package millproto
6
+
7
+
import (
8
+
"encoding/binary"
9
+
"fmt"
10
+
"io"
11
+
"sync"
12
+
13
+
"buf.build/go/protovalidate"
14
+
"google.golang.org/protobuf/proto"
15
+
16
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
17
+
)
18
+
19
+
const (
20
+
ProtocolVersion = 1
21
+
// generous vs agentproto's 1 MiB. a ReserveSeat carries the raw pipeline and
22
+
// workflow JSON, and streamed log lines can be chunky
23
+
MaxMessageBytes = 8 * 1024 * 1024
24
+
)
25
+
26
+
type Message = millv1.Message
27
+
28
+
var validator protovalidate.Validator
29
+
30
+
func init() {
31
+
var err error
32
+
validator, err = protovalidate.New()
33
+
if err != nil {
34
+
panic(fmt.Errorf("failed to initialize protovalidate validator: %w", err))
35
+
}
36
+
}
37
+
38
+
type Encoder struct {
39
+
mu sync.Mutex
40
+
w io.Writer
41
+
}
42
+
43
+
func NewEncoder(w io.Writer) *Encoder {
44
+
return &Encoder{w: w}
45
+
}
46
+
47
+
func (e *Encoder) Encode(msg *Message) error {
48
+
if err := validator.Validate(msg); err != nil {
49
+
return fmt.Errorf("validate fleet message: %w", err)
50
+
}
51
+
52
+
data, err := proto.Marshal(msg)
53
+
if err != nil {
54
+
return fmt.Errorf("marshal fleet message: %w", err)
55
+
}
56
+
if len(data) > MaxMessageBytes {
57
+
return fmt.Errorf("fleet message exceeded %d bytes", MaxMessageBytes)
58
+
}
59
+
60
+
// single write of header and payload maps to exactly one websocket binary
61
+
// frame when the writer is a ws stream
62
+
frame := make([]byte, 4+len(data))
63
+
binary.BigEndian.PutUint32(frame[:4], uint32(len(data)))
64
+
copy(frame[4:], data)
65
+
66
+
e.mu.Lock()
67
+
defer e.mu.Unlock()
68
+
_, err = e.w.Write(frame)
69
+
return err
70
+
}
71
+
72
+
type Decoder struct {
73
+
r io.Reader
74
+
}
75
+
76
+
func NewDecoder(r io.Reader) *Decoder {
77
+
return &Decoder{r: r}
78
+
}
79
+
80
+
func (d *Decoder) Decode() (*Message, error) {
81
+
msg := &Message{}
82
+
var header [4]byte
83
+
if _, err := io.ReadFull(d.r, header[:]); err != nil {
84
+
return msg, err
85
+
}
86
+
87
+
size := binary.BigEndian.Uint32(header[:])
88
+
if size > MaxMessageBytes {
89
+
return msg, fmt.Errorf("fleet message exceeded %d bytes", MaxMessageBytes)
90
+
}
91
+
92
+
data := make([]byte, size)
93
+
if _, err := io.ReadFull(d.r, data); err != nil {
94
+
return msg, err
95
+
}
96
+
if err := proto.Unmarshal(data, msg); err != nil {
97
+
return msg, fmt.Errorf("parse fleet message: %w", err)
98
+
}
99
+
if err := validator.Validate(msg); err != nil {
100
+
return msg, fmt.Errorf("validate fleet message: %w", err)
101
+
}
102
+
return msg, nil
103
+
}
+227
spindle/mill/proto/protocol_test.go
+227
spindle/mill/proto/protocol_test.go
···
1
+
package millproto
2
+
3
+
import (
4
+
"bytes"
5
+
"encoding/binary"
6
+
"testing"
7
+
8
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
9
+
)
10
+
11
+
func TestEncodeDecodeRoundTrip(t *testing.T) {
12
+
var buf bytes.Buffer
13
+
enc := NewEncoder(&buf)
14
+
15
+
want := &Message{
16
+
ReserveSeat: &millv1.ReserveSeat{
17
+
LeaseId: "lease-1",
18
+
TargetEngine: "microvm",
19
+
RawWorkflowJson: `{"name":"build"}`,
20
+
Knot: "knot.example",
21
+
Rkey: "abc123",
22
+
TtlSeconds: 30,
23
+
},
24
+
}
25
+
if err := enc.Encode(want); err != nil {
26
+
t.Fatalf("Encode() error = %v", err)
27
+
}
28
+
29
+
got, err := NewDecoder(&buf).Decode()
30
+
if err != nil {
31
+
t.Fatalf("Decode() error = %v", err)
32
+
}
33
+
rs := got.GetReserveSeat()
34
+
if rs == nil {
35
+
t.Fatal("decoded message missing reserve_seat")
36
+
}
37
+
if rs.LeaseId != "lease-1" || rs.TargetEngine != "microvm" || rs.TtlSeconds != 30 {
38
+
t.Fatalf("round-trip mismatch: %+v", rs)
39
+
}
40
+
}
41
+
42
+
func TestDecoderRejectsOversizedMessage(t *testing.T) {
43
+
var tooLarge bytes.Buffer
44
+
var header [4]byte
45
+
binary.BigEndian.PutUint32(header[:], MaxMessageBytes+1)
46
+
tooLarge.Write(header[:])
47
+
48
+
if _, err := NewDecoder(&tooLarge).Decode(); err == nil {
49
+
t.Fatal("expected oversized message error")
50
+
}
51
+
}
52
+
53
+
func TestValidationRules(t *testing.T) {
54
+
tests := []struct {
55
+
name string
56
+
msg *Message
57
+
wantErr bool
58
+
}{
59
+
{
60
+
name: "valid ack message",
61
+
msg: &Message{
62
+
Ack: &millv1.Ack{
63
+
Epoch: "inc-1",
64
+
UpToSeqno: 5,
65
+
},
66
+
},
67
+
wantErr: false,
68
+
},
69
+
{
70
+
name: "valid hello message",
71
+
msg: &Message{
72
+
Hello: &millv1.Hello{
73
+
ProtocolVersion: 1,
74
+
Arch: "amd64",
75
+
Labels: []string{"linux"},
76
+
Epoch: "inc-1",
77
+
},
78
+
},
79
+
wantErr: false,
80
+
},
81
+
{
82
+
name: "invalid message with zero payloads",
83
+
msg: &Message{},
84
+
wantErr: true,
85
+
},
86
+
{
87
+
name: "invalid message with multiple payloads",
88
+
msg: &Message{
89
+
Ack: &millv1.Ack{Epoch: "inc-1", UpToSeqno: 5},
90
+
Committed: &millv1.Committed{LeaseId: "x"},
91
+
},
92
+
wantErr: true,
93
+
},
94
+
{
95
+
name: "invalid ack message missing epoch",
96
+
msg: &Message{
97
+
Ack: &millv1.Ack{
98
+
UpToSeqno: 5,
99
+
},
100
+
},
101
+
wantErr: true,
102
+
},
103
+
{
104
+
name: "invalid node snapshot with zero seqno",
105
+
msg: &Message{
106
+
NodeSnapshot: &millv1.NodeSnapshot{
107
+
Seqno: 0,
108
+
},
109
+
},
110
+
wantErr: true,
111
+
},
112
+
{
113
+
name: "valid node snapshot with positive seqno",
114
+
msg: &Message{
115
+
NodeSnapshot: &millv1.NodeSnapshot{
116
+
Seqno: 1,
117
+
},
118
+
},
119
+
wantErr: false,
120
+
},
121
+
{
122
+
name: "invalid stream batch with zero seqno entry",
123
+
msg: &Message{
124
+
EventBatch: &millv1.EventBatch{
125
+
Epoch: "inc-1",
126
+
Events: []*millv1.Event{
127
+
{
128
+
Seqno: 0,
129
+
LeaseId: "lease-1",
130
+
Payload: &millv1.Event_StatusEvent{
131
+
StatusEvent: &millv1.StatusEvent{
132
+
Status: millv1.NonterminalStatus_RUNNING,
133
+
},
134
+
},
135
+
},
136
+
},
137
+
},
138
+
},
139
+
wantErr: true,
140
+
},
141
+
{
142
+
name: "invalid stream batch with empty entries",
143
+
msg: &Message{
144
+
EventBatch: &millv1.EventBatch{
145
+
Epoch: "inc-1",
146
+
Events: []*millv1.Event{},
147
+
},
148
+
},
149
+
wantErr: true,
150
+
},
151
+
{
152
+
name: "invalid reserve result with unknown enum",
153
+
msg: &Message{
154
+
ReserveResult: &millv1.ReserveResult{
155
+
LeaseId: "lease-1",
156
+
RejectClass: millv1.RejectClass(99),
157
+
},
158
+
},
159
+
wantErr: true,
160
+
},
161
+
{
162
+
name: "invalid stream entry with malformed oneof (empty payload)",
163
+
msg: &Message{
164
+
EventBatch: &millv1.EventBatch{
165
+
Epoch: "inc-1",
166
+
Events: []*millv1.Event{
167
+
{
168
+
Seqno: 1,
169
+
LeaseId: "lease-1",
170
+
Payload: nil,
171
+
},
172
+
},
173
+
},
174
+
},
175
+
wantErr: true,
176
+
},
177
+
{
178
+
name: "valid stream batch status event",
179
+
msg: &Message{
180
+
EventBatch: &millv1.EventBatch{
181
+
Epoch: "inc-1",
182
+
Events: []*millv1.Event{
183
+
{
184
+
Seqno: 1,
185
+
LeaseId: "lease-1",
186
+
Payload: &millv1.Event_StatusEvent{
187
+
StatusEvent: &millv1.StatusEvent{
188
+
Status: millv1.NonterminalStatus_RUNNING,
189
+
},
190
+
},
191
+
},
192
+
},
193
+
},
194
+
},
195
+
wantErr: false,
196
+
},
197
+
{
198
+
name: "valid stream batch attempt result",
199
+
msg: &Message{
200
+
EventBatch: &millv1.EventBatch{
201
+
Epoch: "inc-1",
202
+
Events: []*millv1.Event{
203
+
{
204
+
Seqno: 1,
205
+
LeaseId: "lease-1",
206
+
Payload: &millv1.Event_AttemptResult{
207
+
AttemptResult: &millv1.AttemptResult{
208
+
Status: millv1.TerminalStatus_SUCCESS,
209
+
},
210
+
},
211
+
},
212
+
},
213
+
},
214
+
},
215
+
wantErr: false,
216
+
},
217
+
}
218
+
219
+
for _, tc := range tests {
220
+
t.Run(tc.name, func(t *testing.T) {
221
+
err := validator.Validate(tc.msg)
222
+
if (err != nil) != tc.wantErr {
223
+
t.Fatalf("Validate() error = %v, wantErr = %v", err, tc.wantErr)
224
+
}
225
+
})
226
+
}
227
+
}
+180
spindle/mill/proto/spindle/mill/v1/mill.proto
+180
spindle/mill/proto/spindle/mill/v1/mill.proto
···
1
+
syntax = "proto3";
2
+
3
+
package spindle.mill.v1;
4
+
5
+
import "buf/validate/validate.proto";
6
+
7
+
option go_package = "tangled.org/core/spindle/mill/proto/gen;millv1";
8
+
9
+
// executor identity, sent on connect
10
+
message Hello {
11
+
uint32 protocol_version = 1;
12
+
// GOARCH of the node, informational only
13
+
string arch = 2;
14
+
// operator-defined labels, matched against runs_on
15
+
repeated string labels = 3;
16
+
string epoch = 4 [(buf.validate.field).string.min_len = 1];
17
+
}
18
+
19
+
// reconnect state for an existing epoch
20
+
message Resume {
21
+
string epoch = 1 [(buf.validate.field).string.min_len = 1];
22
+
uint64 ack_seqno = 2;
23
+
}
24
+
25
+
// per-engine state of a node
26
+
message EngineAvailability {
27
+
bool available = 1;
28
+
// opaque engine-defined load metrics, higher means more loaded
29
+
map<string, double> load = 2;
30
+
}
31
+
32
+
// full node state
33
+
message NodeSnapshot {
34
+
uint64 seqno = 1 [(buf.validate.field).uint64.gt = 0];
35
+
map<string, EngineAvailability> engines = 2;
36
+
// every lease the executor currently holds, reserved or running
37
+
repeated string active_lease_ids = 3;
38
+
}
39
+
40
+
// a seat reservation, carries no secrets
41
+
message ReserveSeat {
42
+
string lease_id = 1 [(buf.validate.field).string.min_len = 1];
43
+
string target_engine = 2 [(buf.validate.field).string.min_len = 1];
44
+
string raw_pipeline_json = 3;
45
+
string raw_workflow_json = 4;
46
+
// pipeline id, the executor reconstructs the exact WorkflowId from it
47
+
string knot = 5;
48
+
string rkey = 6;
49
+
uint32 ttl_seconds = 7;
50
+
}
51
+
52
+
enum RejectClass {
53
+
REJECT_CLASS_UNSPECIFIED = 0;
54
+
REJECT_CLASS_TRANSIENT = 1;
55
+
REJECT_CLASS_INCOMPATIBLE = 2;
56
+
}
57
+
58
+
message ReserveResult {
59
+
string lease_id = 1 [(buf.validate.field).string.min_len = 1];
60
+
bool accepted = 2;
61
+
string reject_reason = 3;
62
+
RejectClass reject_class = 4 [(buf.validate.field).enum.defined_only = true];
63
+
}
64
+
65
+
// a single unlocked secret
66
+
message Secret {
67
+
string key = 1;
68
+
string value = 2;
69
+
}
70
+
71
+
// promotes a reservation to a running job and hands over the secrets
72
+
message CommitLease {
73
+
string lease_id = 1 [(buf.validate.field).string.min_len = 1];
74
+
repeated Secret secrets = 2;
75
+
}
76
+
77
+
message Committed {
78
+
string lease_id = 1 [(buf.validate.field).string.min_len = 1];
79
+
}
80
+
81
+
// drops a reservation that was never committed
82
+
message ReleaseLease {
83
+
string lease_id = 1 [(buf.validate.field).string.min_len = 1];
84
+
}
85
+
86
+
// cancels a running attempt
87
+
message CancelAttempt {
88
+
string lease_id = 1 [(buf.validate.field).string.min_len = 1];
89
+
string reason = 2;
90
+
}
91
+
92
+
message CancelAck {
93
+
string lease_id = 1 [(buf.validate.field).string.min_len = 1];
94
+
}
95
+
96
+
enum NonterminalStatus {
97
+
NONTERMINAL_STATUS_UNSPECIFIED = 0;
98
+
RUNNING = 1;
99
+
}
100
+
101
+
enum TerminalStatus {
102
+
TERMINAL_STATUS_UNSPECIFIED = 0;
103
+
SUCCESS = 1;
104
+
FAILED = 2;
105
+
TIMEOUT = 3;
106
+
CANCELLED = 4;
107
+
}
108
+
109
+
message StatusEvent {
110
+
NonterminalStatus status = 1 [(buf.validate.field).enum = {
111
+
defined_only: true
112
+
not_in: 0
113
+
}];
114
+
string error = 2 [(buf.validate.field).string.max_bytes = 65536];
115
+
int64 exit_code = 3;
116
+
}
117
+
118
+
// one already-encoded models.LogLine JSON line
119
+
message LogLine {
120
+
bytes raw_json = 1 [(buf.validate.field).bytes.min_len = 1];
121
+
}
122
+
123
+
// terminal outcome of an attempt
124
+
message AttemptResult {
125
+
TerminalStatus status = 1 [(buf.validate.field).enum = {
126
+
defined_only: true
127
+
not_in: 0
128
+
}];
129
+
string error = 2 [(buf.validate.field).string.max_bytes = 65536];
130
+
int64 exit_code = 3;
131
+
}
132
+
133
+
// one event in the executor's stream back to the mill
134
+
message Event {
135
+
uint64 seqno = 1 [(buf.validate.field).uint64.gt = 0];
136
+
string lease_id = 2 [(buf.validate.field).string.min_len = 1];
137
+
138
+
oneof payload {
139
+
option (buf.validate.oneof).required = true;
140
+
StatusEvent status_event = 3;
141
+
LogLine log_line = 4;
142
+
AttemptResult attempt_result = 5;
143
+
}
144
+
}
145
+
146
+
// a flushed bundle of events
147
+
message EventBatch {
148
+
string epoch = 1 [(buf.validate.field).string.min_len = 1];
149
+
repeated Event events = 2 [(buf.validate.field).repeated.min_items = 1];
150
+
}
151
+
152
+
// all events up to and including up_to_seqno are durably processed
153
+
message Ack {
154
+
string epoch = 1 [(buf.validate.field).string.min_len = 1];
155
+
uint64 up_to_seqno = 2;
156
+
}
157
+
158
+
message Message {
159
+
option (buf.validate.message).oneof = {
160
+
fields: [
161
+
"hello", "resume", "node_snapshot", "reserve_seat", "reserve_result",
162
+
"commit_lease", "committed", "release_lease", "cancel_attempt",
163
+
"cancel_ack", "event_batch", "ack"
164
+
],
165
+
required: true
166
+
};
167
+
168
+
Hello hello = 1;
169
+
Resume resume = 2;
170
+
NodeSnapshot node_snapshot = 3;
171
+
ReserveSeat reserve_seat = 4;
172
+
ReserveResult reserve_result = 5;
173
+
CommitLease commit_lease = 6;
174
+
Committed committed = 7;
175
+
ReleaseLease release_lease = 8;
176
+
CancelAttempt cancel_attempt = 9;
177
+
CancelAck cancel_ack = 10;
178
+
EventBatch event_batch = 11;
179
+
Ack ack = 12;
180
+
}
+58
spindle/mill/proto/ws.go
+58
spindle/mill/proto/ws.go
···
1
+
package millproto
2
+
3
+
import (
4
+
"io"
5
+
"sync"
6
+
7
+
"github.com/gorilla/websocket"
8
+
)
9
+
10
+
// adapts a gorilla websocket connection to an io.ReadWriteCloser so the
11
+
// length-prefixed fleet framing rides over it. each Encode produces exactly one
12
+
// binary frame. the reader reassembles the byte stream across frames
13
+
type WSStream struct {
14
+
conn *websocket.Conn
15
+
16
+
rmu sync.Mutex
17
+
r io.Reader // current message reader, advanced as frames are consumed
18
+
19
+
wmu sync.Mutex
20
+
}
21
+
22
+
func NewWSStream(conn *websocket.Conn) *WSStream {
23
+
return &WSStream{conn: conn}
24
+
}
25
+
26
+
func (s *WSStream) Read(p []byte) (int, error) {
27
+
s.rmu.Lock()
28
+
defer s.rmu.Unlock()
29
+
for {
30
+
if s.r == nil {
31
+
_, r, err := s.conn.NextReader()
32
+
if err != nil {
33
+
return 0, err
34
+
}
35
+
s.r = r
36
+
}
37
+
n, err := s.r.Read(p)
38
+
if err == io.EOF {
39
+
s.r = nil
40
+
if n > 0 {
41
+
return n, nil
42
+
}
43
+
continue
44
+
}
45
+
return n, err
46
+
}
47
+
}
48
+
49
+
func (s *WSStream) Write(p []byte) (int, error) {
50
+
s.wmu.Lock()
51
+
defer s.wmu.Unlock()
52
+
if err := s.conn.WriteMessage(websocket.BinaryMessage, p); err != nil {
53
+
return 0, err
54
+
}
55
+
return len(p), nil
56
+
}
57
+
58
+
func (s *WSStream) Close() error { return s.conn.Close() }
+326
spindle/mill/restore.go
+326
spindle/mill/restore.go
···
1
+
package mill
2
+
3
+
import (
4
+
"fmt"
5
+
"os"
6
+
"path/filepath"
7
+
"time"
8
+
9
+
"tangled.org/core/spindle/db"
10
+
millproto "tangled.org/core/spindle/mill/proto"
11
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
12
+
"tangled.org/core/spindle/models"
13
+
)
14
+
15
+
const (
16
+
leaseRowReserved = "reserved"
17
+
leaseRowRunning = "running"
18
+
)
19
+
20
+
func (m *Mill) persistLease(lease *RemoteLease, state string) error {
21
+
if m.db == nil {
22
+
return nil
23
+
}
24
+
return m.db.SaveMillLease(db.MillLease{
25
+
LeaseID: lease.id,
26
+
NodeID: lease.nodeID,
27
+
Epoch: lease.epoch,
28
+
Engine: lease.engine,
29
+
Knot: lease.wid.Knot,
30
+
Rkey: lease.wid.Rkey,
31
+
Workflow: lease.wid.Name,
32
+
State: state,
33
+
})
34
+
}
35
+
36
+
func (m *Mill) RestoreState() error {
37
+
if m.db == nil {
38
+
return nil
39
+
}
40
+
cursors, err := m.db.ListExecutorCursors()
41
+
if err != nil {
42
+
return err
43
+
}
44
+
rows, err := m.db.ListMillLeases()
45
+
if err != nil {
46
+
return err
47
+
}
48
+
49
+
m.mu.Lock()
50
+
for _, c := range cursors {
51
+
m.nodeSeqno[c.NodeID+"/"+c.Epoch] = c.AckedSeqno
52
+
}
53
+
for _, r := range rows {
54
+
lease := newLease(r.LeaseID, r.NodeID, r.Epoch, r.Engine)
55
+
lease.wid = models.WorkflowId{
56
+
PipelineId: models.PipelineId{Knot: r.Knot, Rkey: r.Rkey},
57
+
Name: r.Workflow,
58
+
}
59
+
// restored leases start as orphans, an executor must reclaim it via
60
+
// its first snapshot, or the sweep will fail it
61
+
lease.orphaned = true
62
+
lease.claimed = false
63
+
if r.State == leaseRowRunning {
64
+
lease.state = leaseRunning
65
+
}
66
+
m.leases[r.LeaseID] = lease
67
+
}
68
+
restored := len(rows)
69
+
m.mu.Unlock()
70
+
71
+
if restored > 0 {
72
+
m.l.Info("restored mill leases from previous run", "leases", restored, "cursors", len(cursors))
73
+
// executors get a grace window to reconnect and claim their leases
74
+
time.AfterFunc(m.cfg.ReconnectGrace, m.sweepUnclaimedOrphans)
75
+
}
76
+
// crash between terminal and cleanup leaves rows with no owner to
77
+
// flush them, do it now
78
+
return m.flushOrphanedCanonicalLogs()
79
+
}
80
+
81
+
// flushes from db to a file since db is only used while wf is live
82
+
func (m *Mill) flushCanonicalLog(workflow string) error {
83
+
logs, err := m.db.ListCanonicalLogsFor(workflow)
84
+
if err != nil {
85
+
return fmt.Errorf("list canonical logs for %q: %w", workflow, err)
86
+
}
87
+
payloads := make([][]byte, 0, len(logs))
88
+
for _, log := range logs {
89
+
payloads = append(payloads, log.Payload)
90
+
}
91
+
// no lines streamed means no archive file
92
+
if len(payloads) > 0 {
93
+
if err := writeCanonicalFile(workflow, payloads); err != nil {
94
+
return err
95
+
}
96
+
}
97
+
// delete once the file is durable, a retry just rewrites the same content
98
+
return m.db.DeleteCanonicalLogs(workflow)
99
+
}
100
+
101
+
// flushes rows whose lease row is gone
102
+
func (m *Mill) flushOrphanedCanonicalLogs() error {
103
+
logs, err := m.db.ListCanonicalLogs()
104
+
if err != nil {
105
+
return fmt.Errorf("list canonical logs: %w", err)
106
+
}
107
+
leases, err := m.db.ListMillLeases()
108
+
if err != nil {
109
+
return err
110
+
}
111
+
live := make(map[string]struct{}, len(leases))
112
+
for _, l := range leases {
113
+
path := models.LogFilePath(m.cfg.LogDir, models.WorkflowId{
114
+
PipelineId: models.PipelineId{Knot: l.Knot, Rkey: l.Rkey},
115
+
Name: l.Workflow,
116
+
})
117
+
live[path] = struct{}{}
118
+
}
119
+
flushed := make(map[string]struct{})
120
+
for _, log := range logs {
121
+
if _, ok := live[log.Workflow]; ok {
122
+
continue
123
+
}
124
+
if _, done := flushed[log.Workflow]; done {
125
+
continue
126
+
}
127
+
if err := m.flushCanonicalLog(log.Workflow); err != nil {
128
+
return fmt.Errorf("flush orphaned canonical log %q: %w", log.Workflow, err)
129
+
}
130
+
flushed[log.Workflow] = struct{}{}
131
+
}
132
+
return nil
133
+
}
134
+
135
+
// atomically replaces path with the payloads to prevent half-written reads
136
+
func writeCanonicalFile(path string, payloads [][]byte) error {
137
+
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
138
+
return fmt.Errorf("create canonical log directory %q: %w", path, err)
139
+
}
140
+
file, err := os.CreateTemp(filepath.Dir(path), ".mill-log-*")
141
+
if err != nil {
142
+
return fmt.Errorf("create canonical log %q: %w", path, err)
143
+
}
144
+
tmpPath := file.Name()
145
+
defer func() {
146
+
_ = file.Close()
147
+
_ = os.Remove(tmpPath)
148
+
}()
149
+
for _, p := range payloads {
150
+
if _, err := file.Write(p); err != nil {
151
+
return fmt.Errorf("rebuild canonical log %q: %w", path, err)
152
+
}
153
+
}
154
+
if err := file.Close(); err != nil {
155
+
return err
156
+
}
157
+
if err := os.Chmod(tmpPath, 0644); err != nil {
158
+
return err
159
+
}
160
+
if err := os.Rename(tmpPath, path); err != nil {
161
+
return err
162
+
}
163
+
return nil
164
+
}
165
+
166
+
func (m *Mill) sweepUnclaimedOrphans() {
167
+
m.mu.Lock()
168
+
var unclaimed []*RemoteLease
169
+
for _, lease := range m.leases {
170
+
if !lease.orphaned {
171
+
continue
172
+
}
173
+
if lease.claimed {
174
+
continue
175
+
}
176
+
if lease.getState() == leaseDone {
177
+
continue
178
+
}
179
+
if sess := m.sessions[lease.nodeID]; sess == nil || sess.disconnected {
180
+
unclaimed = append(unclaimed, lease)
181
+
}
182
+
}
183
+
m.mu.Unlock()
184
+
185
+
retry := false
186
+
reason := "executor did not reconnect after mill restart"
187
+
for _, lease := range unclaimed {
188
+
m.l.Warn("failing unclaimed restored lease", "lease", lease.id, "node", lease.nodeID)
189
+
if err := m.finishOrphan(lease, string(models.StatusKindFailed), &reason, nil); err != nil {
190
+
m.l.Error("finish unclaimed restored lease", "lease", lease.id, "err", err)
191
+
retry = true
192
+
}
193
+
}
194
+
if retry {
195
+
time.AfterFunc(5*time.Second, m.sweepUnclaimedOrphans)
196
+
}
197
+
}
198
+
199
+
func (m *Mill) reconcileLeases(sess *millSession, activeLeaseIDs []string) error {
200
+
active := make(map[string]struct{}, len(activeLeaseIDs))
201
+
for _, id := range activeLeaseIDs {
202
+
active[id] = struct{}{}
203
+
}
204
+
205
+
known := make(map[string]struct{})
206
+
m.mu.Lock()
207
+
var gone []*RemoteLease
208
+
for _, lease := range m.leases {
209
+
if lease.nodeID == sess.nodeID {
210
+
known[lease.id] = struct{}{}
211
+
if lease.epoch != sess.epoch {
212
+
gone = append(gone, lease)
213
+
} else if _, ok := active[lease.id]; !ok {
214
+
gone = append(gone, lease)
215
+
}
216
+
}
217
+
}
218
+
for _, lease := range m.reservations {
219
+
if lease.nodeID == sess.nodeID && lease.epoch == sess.epoch {
220
+
known[lease.id] = struct{}{}
221
+
}
222
+
}
223
+
m.mu.Unlock()
224
+
var unknown []string
225
+
for id := range active {
226
+
if _, ok := known[id]; !ok {
227
+
unknown = append(unknown, id)
228
+
}
229
+
}
230
+
231
+
for _, lease := range gone {
232
+
status := string(models.StatusKindFailed)
233
+
reason := "executor no longer holds lease"
234
+
if cancelled, cancelReason := lease.cancelRequested(); cancelled {
235
+
status = string(models.StatusKindCancelled)
236
+
reason = cancelReason
237
+
}
238
+
m.l.Warn("finishing reconciled lease", "lease", lease.id, "node", sess.nodeID, "leaseInc", lease.epoch, "sessInc", sess.epoch)
239
+
if lease.orphaned {
240
+
if err := m.finishOrphan(lease, status, &reason, nil); err != nil {
241
+
return err
242
+
}
243
+
} else if err := m.finishLiveLease(lease, status, reason); err != nil {
244
+
return err
245
+
}
246
+
}
247
+
for _, id := range unknown {
248
+
if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{LeaseId: id, Reason: "lease is not owned by this mill"}}); err != nil {
249
+
return fmt.Errorf("cancel unknown executor lease %q: %w", id, err)
250
+
}
251
+
}
252
+
return nil
253
+
}
254
+
255
+
func (m *Mill) completeLeaseRow(lease *RemoteLease, status string, errMsg *string, exitCode *int64) error {
256
+
if m.db == nil {
257
+
return nil
258
+
}
259
+
return m.db.CompleteMillLease(
260
+
lease.id,
261
+
string(lease.wid.PipelineId.AtUri()),
262
+
lease.wid.Name,
263
+
status,
264
+
errMsg,
265
+
exitCode,
266
+
m.n,
267
+
)
268
+
}
269
+
270
+
func (m *Mill) finishLiveLease(lease *RemoteLease, status, reason string) error {
271
+
lease.finishMu.Lock()
272
+
defer lease.finishMu.Unlock()
273
+
if lease.getState() == leaseDone {
274
+
return nil
275
+
}
276
+
if err := m.completeLeaseRow(lease, status, &reason, nil); err != nil {
277
+
return err
278
+
}
279
+
lease.deliverTerminal(&millv1.AttemptResult{
280
+
Status: mapTerminalStatusString(status),
281
+
Error: reason,
282
+
})
283
+
return m.cleanupLeaseLocked(lease)
284
+
}
285
+
286
+
func (m *Mill) finishOrphan(lease *RemoteLease, status string, errMsg *string, exitCode *int64) error {
287
+
lease.finishMu.Lock()
288
+
defer lease.finishMu.Unlock()
289
+
if lease.getState() == leaseDone {
290
+
return nil
291
+
}
292
+
if err := m.completeLeaseRow(lease, status, errMsg, exitCode); err != nil {
293
+
return err
294
+
}
295
+
lease.markDone()
296
+
return m.cleanupLeaseLocked(lease)
297
+
}
298
+
299
+
func mapTerminalStatusString(s string) millv1.TerminalStatus {
300
+
switch s {
301
+
case "success":
302
+
return millv1.TerminalStatus_SUCCESS
303
+
case "failed":
304
+
return millv1.TerminalStatus_FAILED
305
+
case "timeout":
306
+
return millv1.TerminalStatus_TIMEOUT
307
+
case "cancelled":
308
+
return millv1.TerminalStatus_CANCELLED
309
+
default:
310
+
return millv1.TerminalStatus_TERMINAL_STATUS_UNSPECIFIED
311
+
}
312
+
}
313
+
314
+
func derefString(p *string) string {
315
+
if p == nil {
316
+
return ""
317
+
}
318
+
return *p
319
+
}
320
+
321
+
func derefInt64(p *int64) int64 {
322
+
if p == nil {
323
+
return 0
324
+
}
325
+
return *p
326
+
}
+546
spindle/mill/restore_test.go
+546
spindle/mill/restore_test.go
···
1
+
package mill
2
+
3
+
import (
4
+
"context"
5
+
"os"
6
+
"path/filepath"
7
+
"testing"
8
+
"time"
9
+
10
+
"tangled.org/core/notifier"
11
+
"tangled.org/core/spindle/db"
12
+
millproto "tangled.org/core/spindle/mill/proto"
13
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
14
+
"tangled.org/core/spindle/models"
15
+
)
16
+
17
+
func restoreTestMill(t *testing.T, cfg Config) (*Mill, *db.DB) {
18
+
t.Helper()
19
+
bdb, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "mill.db"))
20
+
if err != nil {
21
+
t.Fatalf("db.Make: %v", err)
22
+
}
23
+
t.Cleanup(func() { bdb.Close() })
24
+
n := notifier.New()
25
+
m := New(discardLogger(), cfg)
26
+
m.Attach(bdb, &n)
27
+
return m, bdb
28
+
}
29
+
30
+
func restoredMill(t *testing.T, bdb *db.DB, cfg Config) *Mill {
31
+
t.Helper()
32
+
n := notifier.New()
33
+
m := New(discardLogger(), cfg)
34
+
m.Attach(bdb, &n)
35
+
if err := m.RestoreState(); err != nil {
36
+
t.Fatalf("RestoreState: %v", err)
37
+
}
38
+
return m
39
+
}
40
+
41
+
func TestRestoreStateRebuildsLeasesAndCursors(t *testing.T) {
42
+
_, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
43
+
44
+
if err := bdb.SaveMillLease(db.MillLease{
45
+
LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy",
46
+
Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning,
47
+
}); err != nil {
48
+
t.Fatalf("SaveMillLease: %v", err)
49
+
}
50
+
if err := bdb.SetExecutorCursor("node-1", "inc-1", 7); err != nil {
51
+
t.Fatalf("SetExecutorCursor: %v", err)
52
+
}
53
+
54
+
m2 := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute})
55
+
56
+
m2.mu.Lock()
57
+
lease := m2.leases["lease-1"]
58
+
seqno := m2.nodeSeqno["node-1/inc-1"]
59
+
m2.mu.Unlock()
60
+
61
+
if lease == nil {
62
+
t.Fatal("restored mill has no lease-1")
63
+
}
64
+
if !lease.orphaned {
65
+
t.Fatal("restored lease is not orphaned; a terminal would be delivered to a waiter that does not exist")
66
+
}
67
+
if lease.getState() != leaseRunning {
68
+
t.Fatalf("restored lease state = %v, want leaseRunning", lease.getState())
69
+
}
70
+
wantWid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"}
71
+
if lease.wid != wantWid {
72
+
t.Fatalf("restored lease wid = %+v, want %+v", lease.wid, wantWid)
73
+
}
74
+
if seqno != 7 {
75
+
t.Fatalf("restored cursor = %d, want 7", seqno)
76
+
}
77
+
}
78
+
79
+
func TestRestoreStateFlushesOrphanedCanonicalLogs(t *testing.T) {
80
+
_, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
81
+
logPath := filepath.Join(t.TempDir(), "logs", "workflow.jsonl")
82
+
if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error {
83
+
if err := tx.AddCanonicalLog("node-1", "inc-1", 1, logPath, []byte("{\"line\":1}\n")); err != nil {
84
+
return err
85
+
}
86
+
return tx.AddCanonicalLog("node-1", "inc-1", 2, logPath, []byte("{\"line\":2}\n"))
87
+
}); err != nil {
88
+
t.Fatalf("seed canonical logs: %v", err)
89
+
}
90
+
if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil {
91
+
t.Fatalf("create log directory: %v", err)
92
+
}
93
+
if err := os.WriteFile(logPath, []byte("stale\n"), 0644); err != nil {
94
+
t.Fatalf("seed stale log: %v", err)
95
+
}
96
+
97
+
// rows with no lease row left (terminal committed, crash before
98
+
// cleanup) are flushed to the cold archive at boot
99
+
restoredMill(t, bdb, Config{ReconnectGrace: time.Minute})
100
+
got, err := os.ReadFile(logPath)
101
+
if err != nil {
102
+
t.Fatalf("read flushed log: %v", err)
103
+
}
104
+
if want := "{\"line\":1}\n{\"line\":2}\n"; string(got) != want {
105
+
t.Fatalf("flushed log = %q, want %q", got, want)
106
+
}
107
+
logs, err := bdb.ListCanonicalLogs()
108
+
if err != nil {
109
+
t.Fatal(err)
110
+
}
111
+
if len(logs) != 0 {
112
+
t.Fatalf("canonical logs retained after orphan flush: %+v", logs)
113
+
}
114
+
}
115
+
116
+
func TestLiveLogStaysInTableUntilCleanup(t *testing.T) {
117
+
logDir := t.TempDir()
118
+
m, bdb := restoreTestMill(t, Config{LogDir: logDir, ReconnectGrace: time.Minute})
119
+
lease := newLease("lease-1", "node-1", "inc-1", "dummy")
120
+
lease.wid = models.WorkflowId{
121
+
PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"},
122
+
Name: "build",
123
+
}
124
+
m.mu.Lock()
125
+
m.leases[lease.id] = lease
126
+
m.mu.Unlock()
127
+
128
+
sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
129
+
m.attachSession(sess)
130
+
if err := m.onEventBatch(sess, &millv1.EventBatch{
131
+
Epoch: "inc-1",
132
+
Events: []*millv1.Event{{
133
+
Seqno: 1,
134
+
LeaseId: lease.id,
135
+
Payload: &millv1.Event_LogLine{LogLine: &millv1.LogLine{
136
+
RawJson: []byte(`{"line":1}`),
137
+
}},
138
+
}},
139
+
}); err != nil {
140
+
t.Fatalf("onEventBatch: %v", err)
141
+
}
142
+
143
+
// while the job runs, the table is the log: no file exists yet
144
+
logPath := models.LogFilePath(logDir, lease.wid)
145
+
if _, err := os.Stat(logPath); !os.IsNotExist(err) {
146
+
t.Fatalf("log file exists mid-run, table should be the only store")
147
+
}
148
+
logs, err := bdb.ListCanonicalLogsFor(logPath)
149
+
if err != nil {
150
+
t.Fatal(err)
151
+
}
152
+
if len(logs) != 1 || string(logs[0].Payload) != "{\"line\":1}\n" {
153
+
t.Fatalf("canonical rows = %+v, want the one streamed line", logs)
154
+
}
155
+
156
+
// cleanup materializes the cold archive file and drops the hot rows
157
+
if err := m.cleanupLease(lease); err != nil {
158
+
t.Fatalf("cleanupLease: %v", err)
159
+
}
160
+
got, err := os.ReadFile(logPath)
161
+
if err != nil {
162
+
t.Fatal(err)
163
+
}
164
+
if want := "{\"line\":1}\n"; string(got) != want {
165
+
t.Fatalf("flushed log = %q, want %q", got, want)
166
+
}
167
+
logs, err = bdb.ListCanonicalLogs()
168
+
if err != nil {
169
+
t.Fatal(err)
170
+
}
171
+
if len(logs) != 0 {
172
+
t.Fatalf("canonical logs retained after cleanup: %+v", logs)
173
+
}
174
+
}
175
+
176
+
func TestMillSideLogCapDropsBeyondBudget(t *testing.T) {
177
+
logDir := t.TempDir()
178
+
m, bdb := restoreTestMill(t, Config{LogDir: logDir, ReconnectGrace: time.Minute, MaxLeaseLogBytes: 12})
179
+
lease := newLease("lease-1", "node-1", "inc-1", "dummy")
180
+
lease.wid = models.WorkflowId{
181
+
PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"},
182
+
Name: "build",
183
+
}
184
+
m.mu.Lock()
185
+
m.leases[lease.id] = lease
186
+
m.mu.Unlock()
187
+
188
+
sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
189
+
m.attachSession(sess)
190
+
logLine := func(seqno uint64, body string) *millv1.Event {
191
+
return &millv1.Event{
192
+
Seqno: seqno,
193
+
LeaseId: lease.id,
194
+
Payload: &millv1.Event_LogLine{LogLine: &millv1.LogLine{
195
+
RawJson: []byte(body),
196
+
}},
197
+
}
198
+
}
199
+
// each line is 11 bytes on disk; the cap of 12 admits exactly one
200
+
if err := m.onEventBatch(sess, &millv1.EventBatch{
201
+
Epoch: "inc-1",
202
+
Events: []*millv1.Event{logLine(1, `{"line":1}`), logLine(2, `{"line":2}`)},
203
+
}); err != nil {
204
+
t.Fatalf("onEventBatch: %v", err)
205
+
}
206
+
207
+
// the dropped line never reaches the table, so it can never be flushed
208
+
logPath := models.LogFilePath(logDir, lease.wid)
209
+
logs, err := bdb.ListCanonicalLogsFor(logPath)
210
+
if err != nil {
211
+
t.Fatal(err)
212
+
}
213
+
if len(logs) != 1 || string(logs[0].Payload) != "{\"line\":1}\n" {
214
+
t.Fatalf("canonical rows = %+v, want only the first line", logs)
215
+
}
216
+
if lease.droppedLogs != 1 {
217
+
t.Fatalf("droppedLogs = %d, want 1", lease.droppedLogs)
218
+
}
219
+
if err := m.cleanupLease(lease); err != nil {
220
+
t.Fatalf("cleanupLease: %v", err)
221
+
}
222
+
got, err := os.ReadFile(logPath)
223
+
if err != nil {
224
+
t.Fatal(err)
225
+
}
226
+
if want := "{\"line\":1}\n"; string(got) != want {
227
+
t.Fatalf("flushed log = %q, want only the first line %q", got, want)
228
+
}
229
+
logs, err = bdb.ListCanonicalLogs()
230
+
if err != nil {
231
+
t.Fatal(err)
232
+
}
233
+
if len(logs) != 0 {
234
+
t.Fatalf("canonical logs = %+v, want none after cleanup", logs)
235
+
}
236
+
}
237
+
238
+
func TestOrphanTerminalAuthorsStatusRow(t *testing.T) {
239
+
_, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
240
+
if err := bdb.SaveMillLease(db.MillLease{
241
+
LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy",
242
+
Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning,
243
+
}); err != nil {
244
+
t.Fatalf("SaveMillLease: %v", err)
245
+
}
246
+
if err := bdb.SetExecutorCursor("node-1", "inc-1", 3); err != nil {
247
+
t.Fatalf("SetExecutorCursor: %v", err)
248
+
}
249
+
250
+
m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute})
251
+
252
+
sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
253
+
resume, ok := m.attachSession(sess)
254
+
if !ok {
255
+
t.Fatal("attachSession rejected the reconnecting executor")
256
+
}
257
+
if resume != 3 {
258
+
t.Fatalf("attachSession resume seqno = %d, want restored cursor 3", resume)
259
+
}
260
+
261
+
_ = m.onEventBatch(sess, &millv1.EventBatch{
262
+
Epoch: sess.epoch,
263
+
Events: []*millv1.Event{
264
+
{
265
+
Seqno: 4,
266
+
LeaseId: "lease-1",
267
+
Payload: &millv1.Event_AttemptResult{
268
+
AttemptResult: &millv1.AttemptResult{
269
+
Status: millv1.TerminalStatus_SUCCESS,
270
+
},
271
+
},
272
+
},
273
+
},
274
+
})
275
+
276
+
wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"}
277
+
st, err := bdb.GetStatus(wid)
278
+
if err != nil {
279
+
t.Fatalf("GetStatus after orphan terminal: %v", err)
280
+
}
281
+
if st.Status != string(models.StatusKindSuccess) {
282
+
t.Fatalf("orphan terminal authored status %q, want success", st.Status)
283
+
}
284
+
285
+
m.mu.Lock()
286
+
_, still := m.leases["lease-1"]
287
+
m.mu.Unlock()
288
+
if still {
289
+
t.Fatal("finished orphan still in the lease map")
290
+
}
291
+
if rows, _ := bdb.ListMillLeases(); len(rows) != 0 {
292
+
t.Fatalf("finished orphan still persisted: %+v", rows)
293
+
}
294
+
}
295
+
296
+
func TestSnapshotReconciliationFailsDroppedOrphans(t *testing.T) {
297
+
_, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
298
+
for _, l := range []db.MillLease{
299
+
{LeaseID: "lease-kept", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", Knot: "k", Rkey: "r1", Workflow: "w", State: leaseRowRunning},
300
+
{LeaseID: "lease-gone", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", Knot: "k", Rkey: "r2", Workflow: "w", State: leaseRowRunning},
301
+
} {
302
+
if err := bdb.SaveMillLease(l); err != nil {
303
+
t.Fatalf("SaveMillLease(%s): %v", l.LeaseID, err)
304
+
}
305
+
}
306
+
307
+
m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute})
308
+
309
+
sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
310
+
m.attachSession(sess)
311
+
312
+
m.onSnapshot(sess, &millv1.NodeSnapshot{
313
+
Seqno: 1,
314
+
ActiveLeaseIds: []string{"lease-kept"},
315
+
})
316
+
317
+
m.mu.Lock()
318
+
_, kept := m.leases["lease-kept"]
319
+
_, gone := m.leases["lease-gone"]
320
+
m.mu.Unlock()
321
+
if !kept {
322
+
t.Fatal("reconciliation dropped a lease the executor still holds")
323
+
}
324
+
if gone {
325
+
t.Fatal("reconciliation kept a lease the executor no longer holds")
326
+
}
327
+
328
+
st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r2"}, Name: "w"})
329
+
if err != nil {
330
+
t.Fatalf("GetStatus for dropped orphan: %v", err)
331
+
}
332
+
if st.Status != string(models.StatusKindFailed) {
333
+
t.Fatalf("dropped orphan authored status %q, want failed", st.Status)
334
+
}
335
+
}
336
+
func TestSnapshotReconciliationPreservesRequestedCancellation(t *testing.T) {
337
+
m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
338
+
lease := newLease("lease-1", "node-1", "inc-old", "dummy")
339
+
lease.wid = models.WorkflowId{
340
+
PipelineId: models.PipelineId{Knot: "k", Rkey: "r1"},
341
+
Name: "w",
342
+
}
343
+
lease.setState(leaseRunning)
344
+
lease.requestCancel("workflow destroyed")
345
+
if err := m.persistLease(lease, leaseRowRunning); err != nil {
346
+
t.Fatalf("persistLease: %v", err)
347
+
}
348
+
m.mu.Lock()
349
+
m.leases[lease.id] = lease
350
+
m.mu.Unlock()
351
+
352
+
sess := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger())
353
+
m.attachSession(sess)
354
+
if err := m.onSnapshot(sess, &millv1.NodeSnapshot{Seqno: 1}); err != nil {
355
+
t.Fatalf("onSnapshot: %v", err)
356
+
}
357
+
358
+
st, err := bdb.GetStatus(lease.wid)
359
+
if err != nil {
360
+
t.Fatalf("GetStatus: %v", err)
361
+
}
362
+
if st.Status != string(models.StatusKindCancelled) {
363
+
t.Fatalf("reconciled status = %q, want cancelled", st.Status)
364
+
}
365
+
}
366
+
367
+
func TestSnapshotReconciliationCancelsUnknownExecutorLease(t *testing.T) {
368
+
m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
369
+
cancelled := make(chan string, 1)
370
+
sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error {
371
+
if cancel := msg.GetCancelAttempt(); cancel != nil {
372
+
cancelled <- cancel.GetLeaseId()
373
+
}
374
+
return nil
375
+
}), discardLogger())
376
+
m.attachSession(sess)
377
+
378
+
if err := m.onSnapshot(sess, &millv1.NodeSnapshot{
379
+
Seqno: 1,
380
+
ActiveLeaseIds: []string{"executor-only"},
381
+
}); err != nil {
382
+
t.Fatalf("onSnapshot: %v", err)
383
+
}
384
+
select {
385
+
case id := <-cancelled:
386
+
if id != "executor-only" {
387
+
t.Fatalf("cancelled lease = %q, want executor-only", id)
388
+
}
389
+
default:
390
+
t.Fatal("snapshot reconciliation left an executor-only lease running")
391
+
}
392
+
}
393
+
394
+
func TestSweepFailsOrphansOfAbsentExecutors(t *testing.T) {
395
+
_, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
396
+
if err := bdb.SaveMillLease(db.MillLease{
397
+
LeaseID: "lease-1", NodeID: "node-absent", Epoch: "inc-absent", Engine: "dummy",
398
+
Knot: "k", Rkey: "r1", Workflow: "w", State: leaseRowReserved,
399
+
}); err != nil {
400
+
t.Fatalf("SaveMillLease: %v", err)
401
+
}
402
+
403
+
m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute})
404
+
m.sweepUnclaimedOrphans()
405
+
406
+
m.mu.Lock()
407
+
_, still := m.leases["lease-1"]
408
+
m.mu.Unlock()
409
+
if still {
410
+
t.Fatal("sweep kept an orphan whose executor never reconnected")
411
+
}
412
+
st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r1"}, Name: "w"})
413
+
if err != nil {
414
+
t.Fatalf("GetStatus after sweep: %v", err)
415
+
}
416
+
if st.Status != string(models.StatusKindFailed) {
417
+
t.Fatalf("sweep authored status %q, want failed", st.Status)
418
+
}
419
+
if rows, _ := bdb.ListMillLeases(); len(rows) != 0 {
420
+
t.Fatalf("swept orphan still persisted: %+v", rows)
421
+
}
422
+
}
423
+
424
+
func TestAckSeqnoPersistsCursor(t *testing.T) {
425
+
m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
426
+
sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
427
+
m.attachSession(sess)
428
+
429
+
owned := newLease("lease-1", "node-1", "inc-1", "dummy")
430
+
owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}
431
+
m.mu.Lock()
432
+
m.leases[owned.id] = owned
433
+
m.mu.Unlock()
434
+
435
+
err := m.onEventBatch(sess, &millv1.EventBatch{
436
+
Epoch: sess.epoch,
437
+
Events: []*millv1.Event{
438
+
{
439
+
Seqno: 1,
440
+
LeaseId: owned.id,
441
+
Payload: &millv1.Event_StatusEvent{
442
+
StatusEvent: &millv1.StatusEvent{
443
+
Status: millv1.NonterminalStatus_RUNNING,
444
+
},
445
+
},
446
+
},
447
+
},
448
+
})
449
+
if err != nil {
450
+
t.Fatalf("onEventBatch: %v", err)
451
+
}
452
+
453
+
cursors, err := bdb.ListExecutorCursors()
454
+
if err != nil {
455
+
t.Fatalf("ListExecutorCursors: %v", err)
456
+
}
457
+
if len(cursors) != 1 || cursors[0].AckedSeqno != 1 {
458
+
t.Fatalf("persisted cursor = %+v, want seqno 1", cursors)
459
+
}
460
+
}
461
+
462
+
func TestOrphanTerminalFailureKeepsLeaseAndSeqnoRetryable(t *testing.T) {
463
+
_, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
464
+
if err := bdb.SaveMillLease(db.MillLease{
465
+
LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy",
466
+
Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning,
467
+
}); err != nil {
468
+
t.Fatalf("SaveMillLease: %v", err)
469
+
}
470
+
if err := bdb.SetExecutorCursor("node-1", "inc-1", 3); err != nil {
471
+
t.Fatalf("SetExecutorCursor: %v", err)
472
+
}
473
+
if _, err := bdb.Exec(`
474
+
create trigger reject_orphan_lease_delete
475
+
before delete on mill_leases
476
+
begin
477
+
select raise(abort, 'forced delete failure');
478
+
end
479
+
`); err != nil {
480
+
t.Fatalf("create failure trigger: %v", err)
481
+
}
482
+
483
+
m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute})
484
+
sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
485
+
if _, ok := m.attachSession(sess); !ok {
486
+
t.Fatal("attachSession rejected reconnect")
487
+
}
488
+
batch := &millv1.EventBatch{
489
+
Epoch: sess.epoch,
490
+
Events: []*millv1.Event{
491
+
{
492
+
Seqno: 4,
493
+
LeaseId: "lease-1",
494
+
Payload: &millv1.Event_AttemptResult{
495
+
AttemptResult: &millv1.AttemptResult{
496
+
Status: millv1.TerminalStatus_SUCCESS,
497
+
},
498
+
},
499
+
},
500
+
},
501
+
}
502
+
if err := m.onEventBatch(sess, batch); err == nil {
503
+
t.Fatal("orphan terminal stream succeeded despite forced transaction failure")
504
+
}
505
+
506
+
m.mu.Lock()
507
+
lease := m.leases["lease-1"]
508
+
seqno := m.nodeSeqno["node-1/inc-1"]
509
+
m.mu.Unlock()
510
+
if lease == nil || lease.getState() == leaseDone {
511
+
t.Fatal("failed orphan completion made the in-memory lease unretryable")
512
+
}
513
+
if seqno != 3 {
514
+
t.Fatalf("in-memory stream seqno = %d, want 3", seqno)
515
+
}
516
+
if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 1 {
517
+
t.Fatalf("durable leases after transaction rollback = %+v, err = %v; want retained lease", rows, err)
518
+
}
519
+
var events int
520
+
if err := bdb.QueryRow(`select count(*) from events`).Scan(&events); err != nil {
521
+
t.Fatalf("count events: %v", err)
522
+
}
523
+
if events != 0 {
524
+
t.Fatalf("terminal events after transaction rollback = %d, want 0", events)
525
+
}
526
+
527
+
if _, err := bdb.Exec(`drop trigger reject_orphan_lease_delete`); err != nil {
528
+
t.Fatalf("drop failure trigger: %v", err)
529
+
}
530
+
if err := m.onEventBatch(sess, batch); err != nil {
531
+
t.Fatalf("retry orphan terminal: %v", err)
532
+
}
533
+
m.mu.Lock()
534
+
_, still := m.leases["lease-1"]
535
+
seqno = m.nodeSeqno["node-1/inc-1"]
536
+
m.mu.Unlock()
537
+
if still {
538
+
t.Fatal("successful orphan completion retained in-memory lease")
539
+
}
540
+
if seqno != 4 {
541
+
t.Fatalf("in-memory stream seqno after retry = %d, want 4", seqno)
542
+
}
543
+
if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 0 {
544
+
t.Fatalf("durable leases after successful retry = %+v, err = %v; want none", rows, err)
545
+
}
546
+
}
+158
spindle/mill/session.go
+158
spindle/mill/session.go
···
1
+
package mill
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"log/slog"
7
+
"sync"
8
+
"time"
9
+
10
+
millproto "tangled.org/core/spindle/mill/proto"
11
+
millv1 "tangled.org/core/spindle/mill/proto/gen"
12
+
)
13
+
14
+
var errSessionClosed = errors.New("mill: executor session closed")
15
+
16
+
// one live websocket to an executor. many leases and async streams share
17
+
// it, so one reader goroutine demuxes by message type and correlates by
18
+
// lease ID. never hold locks across decodes
19
+
type millSession struct {
20
+
nodeID string
21
+
epoch string
22
+
labels []string
23
+
enc messageEncoder
24
+
l *slog.Logger
25
+
closeTransport func() error
26
+
27
+
// snapshot, disconnected and graceTimer are guarded by Mill.mu, the
28
+
// fleet ranks across sessions under its own lock
29
+
snapshot *millv1.NodeSnapshot
30
+
disconnected bool
31
+
graceTimer *time.Timer
32
+
lastSeen time.Time
33
+
34
+
mu sync.Mutex
35
+
pending map[string]chan *millproto.Message // maps lease ID to response waiter
36
+
37
+
closeOnce sync.Once
38
+
closed chan struct{}
39
+
}
40
+
41
+
type messageEncoder interface {
42
+
Encode(*millproto.Message) error
43
+
}
44
+
45
+
func newSession(nodeID string, epoch string, labels []string, enc messageEncoder, l *slog.Logger) *millSession {
46
+
return &millSession{
47
+
nodeID: nodeID,
48
+
epoch: epoch,
49
+
labels: labels,
50
+
enc: enc,
51
+
l: l,
52
+
pending: make(map[string]chan *millproto.Message),
53
+
closed: make(chan struct{}),
54
+
lastSeen: time.Now(),
55
+
}
56
+
}
57
+
58
+
// caller holds Mill.mu
59
+
func (s *millSession) live(grace time.Duration) bool {
60
+
return !s.disconnected && time.Since(s.lastSeen) <= grace
61
+
}
62
+
63
+
func (s *millSession) send(msg *millproto.Message) error {
64
+
return s.enc.Encode(msg)
65
+
}
66
+
67
+
func (s *millSession) close() {
68
+
s.closeOnce.Do(func() {
69
+
close(s.closed)
70
+
if s.closeTransport != nil {
71
+
_ = s.closeTransport()
72
+
}
73
+
})
74
+
}
75
+
76
+
// one-shot waiter for the next response on the lease, cancel unregisters it
77
+
func (s *millSession) await(leaseID string) (<-chan *millproto.Message, func()) {
78
+
ch := make(chan *millproto.Message, 1)
79
+
s.mu.Lock()
80
+
s.pending[leaseID] = ch
81
+
s.mu.Unlock()
82
+
return ch, func() {
83
+
s.mu.Lock()
84
+
if s.pending[leaseID] == ch {
85
+
delete(s.pending, leaseID)
86
+
}
87
+
s.mu.Unlock()
88
+
}
89
+
}
90
+
91
+
func (s *millSession) deliver(leaseID string, msg *millproto.Message) {
92
+
s.mu.Lock()
93
+
ch := s.pending[leaseID]
94
+
delete(s.pending, leaseID)
95
+
s.mu.Unlock()
96
+
if ch != nil {
97
+
select {
98
+
case ch <- msg:
99
+
default:
100
+
}
101
+
}
102
+
}
103
+
104
+
// sends a message and waits for its response, respecting ctx and session closure
105
+
func (s *millSession) request(ctx context.Context, leaseID string, msg *millproto.Message) (*millproto.Message, error) {
106
+
if err := ctx.Err(); err != nil {
107
+
return nil, err
108
+
}
109
+
ch, cancel := s.await(leaseID)
110
+
defer cancel()
111
+
112
+
if err := s.send(msg); err != nil {
113
+
return nil, err
114
+
}
115
+
116
+
select {
117
+
case resp := <-ch:
118
+
return resp, nil
119
+
case <-ctx.Done():
120
+
return nil, ctx.Err()
121
+
case <-s.closed:
122
+
return nil, errSessionClosed
123
+
}
124
+
}
125
+
126
+
// demuxes frames until the decoder errors (connection gone)
127
+
func (s *millSession) readLoop(m *Mill, dec *millproto.Decoder) error {
128
+
for {
129
+
msg, err := dec.Decode()
130
+
if err != nil {
131
+
return err
132
+
}
133
+
if err := s.dispatch(m, msg); err != nil {
134
+
return err
135
+
}
136
+
}
137
+
}
138
+
139
+
func (s *millSession) dispatch(m *Mill, msg *millproto.Message) error {
140
+
if !m.touchSession(s) {
141
+
return errSessionClosed
142
+
}
143
+
switch {
144
+
case msg.GetNodeSnapshot() != nil:
145
+
return m.onSnapshot(s, msg.GetNodeSnapshot())
146
+
case msg.GetReserveResult() != nil:
147
+
s.deliver(msg.GetReserveResult().GetLeaseId(), msg)
148
+
case msg.GetCommitted() != nil:
149
+
s.deliver(msg.GetCommitted().GetLeaseId(), msg)
150
+
case msg.GetEventBatch() != nil:
151
+
return m.onEventBatch(s, msg.GetEventBatch())
152
+
case msg.GetCancelAck() != nil:
153
+
m.onCancelAck(s, msg.GetCancelAck())
154
+
default:
155
+
s.l.Warn("session received unexpected message", "node", s.nodeID)
156
+
}
157
+
return nil
158
+
}
+23
spindle/mill/token.go
+23
spindle/mill/token.go
···
1
+
package mill
2
+
3
+
import (
4
+
"crypto/rand"
5
+
"crypto/sha256"
6
+
"encoding/base64"
7
+
"encoding/hex"
8
+
)
9
+
10
+
func GenerateToken() (string, error) {
11
+
var b [32]byte
12
+
if _, err := rand.Read(b[:]); err != nil {
13
+
return "", err
14
+
}
15
+
return base64.RawURLEncoding.EncodeToString(b[:]), nil
16
+
}
17
+
18
+
// hashes the token for persistence and comparison. a database leak never
19
+
// exposes a usable token, and lookup avoids a secret-dependent comparison
20
+
func HashToken(token string) string {
21
+
sum := sha256.Sum256([]byte(token))
22
+
return hex.EncodeToString(sum[:])
23
+
}
+154
-94
spindle/server.go
+154
-94
spindle/server.go
···
38
38
"tangled.org/core/spindle/engines/dummy"
39
39
"tangled.org/core/spindle/engines/nixery"
40
40
"tangled.org/core/spindle/git"
41
+
"tangled.org/core/spindle/mill"
42
+
"tangled.org/core/spindle/mill/executor"
41
43
"tangled.org/core/spindle/models"
42
44
"tangled.org/core/spindle/queue"
43
45
"tangled.org/core/spindle/secrets"
···
72
74
motd []byte
73
75
motdMu sync.RWMutex
74
76
rootCtx context.Context
77
+
78
+
// set only when this spindle hosts the mill or joins one as an executor
79
+
mill *mill.Mill
80
+
exec *executor.Executor
75
81
}
76
82
77
83
// New creates a new Spindle server with the provided configuration and engines.
78
84
func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) {
79
85
logger := log.FromContext(ctx)
86
+
n := notifier.New()
87
+
88
+
if cfg.Role == config.RoleExecutor {
89
+
if err := cleanupOrphanRepos(ctx, d, logger); err != nil {
90
+
return nil, fmt.Errorf("failed to run startup cleanup: %w", err)
91
+
}
92
+
} else if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil {
93
+
return nil, fmt.Errorf("failed to run startup migrations: %w", err)
94
+
}
95
+
96
+
spindle := &Spindle{
97
+
db: d,
98
+
l: logger,
99
+
n: &n,
100
+
engs: engines,
101
+
cfg: cfg,
102
+
motd: defaultMotd,
103
+
rootCtx: ctx,
104
+
}
105
+
if cfg.Role == config.RoleExecutor {
106
+
return spindle, nil
107
+
}
80
108
81
109
e, err := rbac.NewEnforcer(cfg.Server.DBPath)
82
110
if err != nil {
83
111
return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err)
84
112
}
85
113
e.E.EnableAutoSave(true)
86
-
87
-
n := notifier.New()
114
+
spindle.e = e
88
115
89
-
var vault secrets.Manager
90
116
switch cfg.Server.Secrets.Provider {
91
117
case "openbao":
92
118
if cfg.Server.Secrets.OpenBao.ProxyAddr == "" {
93
119
return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider")
94
120
}
95
-
vault, err = secrets.NewOpenBaoManager(
121
+
spindle.vault, err = secrets.NewOpenBaoManager(
96
122
cfg.Server.Secrets.OpenBao.ProxyAddr,
97
123
logger,
98
124
secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount),
···
102
128
}
103
129
logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount)
104
130
case "sqlite", "":
105
-
vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets"))
131
+
spindle.vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets"))
106
132
if err != nil {
107
133
return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err)
108
134
}
···
111
137
return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider)
112
138
}
113
139
114
-
if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil {
115
-
return nil, fmt.Errorf("failed to run startup migrations: %w", err)
116
-
}
117
-
118
140
jq := queue.NewQueue(cfg.Server.QueueSize, cfg.Server.MaxJobCount)
141
+
spindle.jq = jq
119
142
logger.Info("initialized queue", "queueSize", cfg.Server.QueueSize, "numWorkers", cfg.Server.MaxJobCount)
120
143
121
144
collections := []string{
···
128
151
if err != nil {
129
152
return nil, fmt.Errorf("failed to setup jetstream client: %w", err)
130
153
}
154
+
spindle.jc = jc
131
155
jc.AddDid(cfg.Server.Owner)
132
156
// pull records are created by arbitrary users too, same hack as in tap
133
157
jc.ExemptCollection(tangled.RepoPullNSID)
···
151
175
}
152
176
}
153
177
154
-
resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl)
155
-
156
-
spindle := &Spindle{
157
-
jc: jc,
158
-
e: e,
159
-
db: d,
160
-
l: logger,
161
-
n: &n,
162
-
engs: engines,
163
-
jq: jq,
164
-
cfg: cfg,
165
-
res: resolver,
166
-
verify: repoverify.New(resolver, cfg.Server.Dev),
167
-
vault: vault,
168
-
motd: defaultMotd,
169
-
rootCtx: ctx,
170
-
}
178
+
spindle.res = idresolver.DefaultResolver(cfg.Server.PlcUrl)
179
+
spindle.verify = repoverify.New(spindle.res, cfg.Server.Dev)
171
180
172
181
err = e.AddSpindle(rbacDomain)
173
182
if err != nil {
···
272
281
return s.motd
273
282
}
274
283
275
-
// Start starts the Spindle server (blocking).
284
+
// runs the server. blocks
276
285
func (s *Spindle) Start(ctx context.Context) error {
277
-
// starts a job queue runner in the background
278
-
s.jq.Start()
279
-
defer s.jq.Stop()
286
+
// only standalone runs the local queue. mill hosts place directly onto
287
+
// executors, and executors only run jobs explicitly assigned by a mill
288
+
if s.cfg.Role == config.RoleStandalone {
289
+
s.jq.Start()
290
+
defer s.jq.Stop()
291
+
}
280
292
281
-
// Stop vault token renewal if it implements Stopper
293
+
// an executor dials out to its mill and takes work from it
294
+
if s.exec != nil {
295
+
go s.exec.Connect(ctx)
296
+
}
297
+
282
298
if stopper, ok := s.vault.(secrets.Stopper); ok {
283
299
defer stopper.Stop()
284
300
}
285
301
286
-
tapCtx, tapCancel := context.WithCancel(ctx)
302
+
if s.cfg.Role != config.RoleExecutor {
303
+
tapCtx, tapCancel := context.WithCancel(ctx)
287
304
288
-
if s.cfg.Server.Tap.Embed {
289
-
emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap"))
290
-
if err != nil {
291
-
tapCancel()
292
-
return fmt.Errorf("starting embedded tap: %w", err)
305
+
if s.cfg.Server.Tap.Embed {
306
+
emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap"))
307
+
if err != nil {
308
+
tapCancel()
309
+
return fmt.Errorf("starting embedded tap: %w", err)
310
+
}
311
+
s.embedTap = emb
312
+
defer func() {
313
+
tapCancel()
314
+
s.embedTap.Shutdown()
315
+
}()
316
+
317
+
go s.watchTapDrain(tapCtx, tapCancel)
318
+
} else {
319
+
defer tapCancel()
293
320
}
294
-
s.embedTap = emb
295
-
defer func() {
296
-
tapCancel()
297
-
s.embedTap.Shutdown()
321
+
322
+
go func() {
323
+
s.l.Info("starting knot event consumer")
324
+
s.ks.Start(ctx)
298
325
}()
299
326
300
-
go s.watchTapDrain(tapCtx, tapCancel)
301
-
} else {
302
-
defer tapCancel()
327
+
s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url)
328
+
s.tap.Start(tapCtx)
303
329
}
304
-
305
-
go func() {
306
-
s.l.Info("starting knot event consumer")
307
-
s.ks.Start(ctx)
308
-
}()
309
-
310
-
s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url)
311
-
s.tap.Start(tapCtx)
312
330
313
331
s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr)
314
332
return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router())
···
354
372
return fmt.Errorf("failed to setup db: %w", err)
355
373
}
356
374
357
-
nixeryEng, err := nixery.New(ctx, cfg)
358
-
if err != nil {
359
-
return err
375
+
logger := log.FromContext(ctx)
376
+
377
+
var engines map[string]models.Engine
378
+
var m *mill.Mill
379
+
380
+
if cfg.Role == config.RoleMill {
381
+
// mill host: register engines that place jobs on executors instead of
382
+
// running them. all names share one Mill
383
+
m = mill.New(log.SubLogger(logger, "mill"), mill.Config{
384
+
MaxPending: cfg.Mill.MaxPending,
385
+
ReconnectGrace: cfg.Mill.ReconnectGrace,
386
+
LogDir: cfg.Server.LogDir,
387
+
})
388
+
engines = map[string]models.Engine{
389
+
"nixery": mill.NewEngine("nixery", m),
390
+
"microvm": mill.NewEngine("microvm", m),
391
+
"dummy": mill.NewEngine("dummy", m),
392
+
}
393
+
} else {
394
+
// standalone and executor both run real engines locally.
395
+
nixeryEng, err := nixery.New(ctx, cfg)
396
+
if err != nil {
397
+
return err
398
+
}
399
+
microvmEng, err := newMicrovmEngine(ctx, cfg, d)
400
+
if err != nil {
401
+
return err
402
+
}
403
+
engines = map[string]models.Engine{
404
+
"nixery": nixeryEng,
405
+
"microvm": microvmEng,
406
+
"dummy": dummy.New(logger),
407
+
}
360
408
}
361
409
362
-
microvmEng, err := newMicrovmEngine(ctx, cfg, d)
410
+
s, err := New(ctx, cfg, d, engines)
363
411
if err != nil {
364
412
return err
365
413
}
366
414
367
-
s, err := New(ctx, cfg, d, map[string]models.Engine{
368
-
"nixery": nixeryEng,
369
-
"microvm": microvmEng,
370
-
"dummy": dummy.New(log.FromContext(ctx)),
371
-
})
372
-
if err != nil {
373
-
return err
415
+
if m != nil {
416
+
// resolve the chicken-and-egg: the engines (built above) hold the mill,
417
+
// but the mill's db/notifier are created inside New
418
+
m.Attach(s.DB(), s.Notifier())
419
+
s.mill = m
420
+
if err := m.RestoreState(); err != nil {
421
+
return fmt.Errorf("restoring mill state: %w", err)
422
+
}
423
+
}
424
+
if cfg.Role == config.RoleExecutor {
425
+
s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"))
426
+
if err != nil {
427
+
return err
428
+
}
374
429
}
375
430
376
431
return s.Start(ctx)
···
382
437
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
383
438
w.Write(s.GetMotdContent())
384
439
})
440
+
if s.cfg.Role == config.RoleExecutor {
441
+
return mux
442
+
}
443
+
385
444
mux.HandleFunc("/events", s.Events)
386
445
mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs)
446
+
447
+
// mill host: executors dial in here (plain ws, shared-secret auth)
448
+
if s.mill != nil {
449
+
mux.HandleFunc("/mill", s.mill.HandleExecutorConn)
450
+
}
387
451
388
452
mux.Mount("/xrpc", s.XrpcRouter())
389
453
return mux
···
822
886
workflows[eng] = append(workflows[eng], *ewf)
823
887
}
824
888
825
-
// enqueue pipeline
826
-
ok := s.jq.Enqueue(repoDid, queue.Job{
827
-
Run: func() error {
828
-
engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, &models.Pipeline{
829
-
RepoDid: repoDid,
830
-
Workflows: workflows,
831
-
TrustedSource: trustedSource,
832
-
}, pipelineId)
833
-
return nil
834
-
},
835
-
OnFail: func(jobError error) {
836
-
s.l.Error("pipeline run failed", "error", jobError)
837
-
},
838
-
})
839
-
if !ok {
840
-
return fmt.Errorf("failed to enqueue pipeline: queue is full")
889
+
pipeline := &models.Pipeline{
890
+
RepoDid: repoDid,
891
+
Workflows: workflows,
892
+
TrustedSource: trustedSource,
841
893
}
842
-
s.l.Info("pipeline enqueued successfully", "id", pipelineId)
894
+
895
+
if s.mill != nil {
896
+
// mill host: no bounded pool. each job blocks in placement
897
+
// (AcquireWorkflowSlot) which the user sees as pending. the only bound
898
+
// is the mill's maxPending. rootCtx is the long-lived consumer context,
899
+
// so the goroutine safely outlives this call
900
+
go engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, pipeline, pipelineId)
901
+
s.l.Info("pipeline handed to mill placement", "id", pipelineId)
902
+
} else {
903
+
ok := s.jq.Enqueue(repoDid, queue.Job{
904
+
Run: func() error {
905
+
engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, pipeline, pipelineId)
906
+
return nil
907
+
},
908
+
OnFail: func(jobError error) {
909
+
s.l.Error("pipeline run failed", "error", jobError)
910
+
},
911
+
})
912
+
if !ok {
913
+
return fmt.Errorf("failed to enqueue pipeline: queue is full")
914
+
}
915
+
s.l.Info("pipeline enqueued successfully", "id", pipelineId)
916
+
}
843
917
844
918
// after successful enqueue, emit StatusPending for all workflows
845
919
for _, ewfs := range workflows {
···
881
955
return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion)
882
956
}
883
957
return nil
884
-
}
885
-
886
-
func (s *Spindle) resolvePipelineRepoDid(repo *tangled.Pipeline_TriggerRepo) (syntax.DID, error) {
887
-
if repo.RepoDid == nil || *repo.RepoDid == "" {
888
-
return "", fmt.Errorf("pipeline trigger missing repoDid")
889
-
}
890
-
repoDid, err := syntax.ParseDID(*repo.RepoDid)
891
-
if err != nil {
892
-
return "", fmt.Errorf("parse repoDid %s: %w", *repo.RepoDid, err)
893
-
}
894
-
if _, err := s.db.GetRepoByDid(repoDid); err != nil {
895
-
return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err)
896
-
}
897
-
return repoDid, nil
898
958
}
899
959
900
960
func (s *Spindle) configureOwner() error {
+44
spindle/server_test.go
+44
spindle/server_test.go
···
1
1
package spindle
2
2
3
3
import (
4
+
"context"
5
+
"net/http"
6
+
"net/http/httptest"
7
+
"path/filepath"
4
8
"testing"
5
9
6
10
kgit "tangled.org/core/knotserver/git"
11
+
"tangled.org/core/spindle/config"
12
+
"tangled.org/core/spindle/db"
13
+
"tangled.org/core/spindle/models"
7
14
)
8
15
9
16
func TestHasSkipCIPushOption(t *testing.T) {
···
53
60
})
54
61
}
55
62
}
63
+
64
+
func TestExecutorRoleBuildsMinimalSpindle(t *testing.T) {
65
+
ctx := context.Background()
66
+
dbPath := filepath.Join(t.TempDir(), "spindle.db")
67
+
d, err := db.Make(ctx, dbPath)
68
+
if err != nil {
69
+
t.Fatalf("db.Make() error = %v", err)
70
+
}
71
+
72
+
cfg := &config.Config{Role: config.RoleExecutor}
73
+
cfg.Server.DBPath = dbPath
74
+
cfg.Server.Hostname = "executor.test"
75
+
cfg.Server.Tap.Embed = true
76
+
77
+
s, err := New(ctx, cfg, d, map[string]models.Engine{})
78
+
if err != nil {
79
+
t.Fatalf("New() error = %v", err)
80
+
}
81
+
82
+
if s.jc != nil || s.tap != nil || s.e != nil || s.jq != nil || s.ks != nil || s.res != nil || s.vault != nil {
83
+
t.Fatal("executor role built coordinator-only spindle dependencies")
84
+
}
85
+
86
+
rr := httptest.NewRecorder()
87
+
req := httptest.NewRequest(http.MethodGet, "/", nil)
88
+
s.Router().ServeHTTP(rr, req)
89
+
if rr.Code != http.StatusOK {
90
+
t.Fatalf("root status = %d, want %d", rr.Code, http.StatusOK)
91
+
}
92
+
93
+
rr = httptest.NewRecorder()
94
+
req = httptest.NewRequest(http.MethodGet, "/xrpc/_health", nil)
95
+
s.Router().ServeHTTP(rr, req)
96
+
if rr.Code != http.StatusNotFound {
97
+
t.Fatalf("executor xrpc status = %d, want %d", rr.Code, http.StatusNotFound)
98
+
}
99
+
}
+13
-25
spindle/stream.go
+13
-25
spindle/stream.go
···
4
4
"context"
5
5
"errors"
6
6
"fmt"
7
-
"io"
8
7
"net/http"
9
8
"time"
10
9
11
10
"tangled.org/core/eventstream"
12
11
"tangled.org/core/log"
12
+
"tangled.org/core/spindle/config"
13
+
"tangled.org/core/spindle/logview"
13
14
"tangled.org/core/spindle/models"
14
15
15
16
"github.com/go-chi/chi/v5"
16
17
"github.com/gorilla/websocket"
17
-
"github.com/hpcloud/tail"
18
18
)
19
19
20
20
var upgrader = websocket.Upgrader{
···
89
89
}
90
90
isFinished := models.StatusKind(status.Status).IsFinish()
91
91
92
-
filePath := models.LogFilePath(s.cfg.Server.LogDir, wid)
93
-
94
-
config := tail.Config{
95
-
Follow: !isFinished,
96
-
ReOpen: !isFinished,
97
-
MustExist: false,
98
-
Location: &tail.SeekInfo{
99
-
Offset: 0,
100
-
Whence: io.SeekStart,
101
-
},
102
-
// Logger: tail.DiscardingLogger,
103
-
}
104
-
105
-
t, err := tail.TailFile(filePath, config)
92
+
lines, stop, err := logview.Follow(ctx, s.db, s.cfg.Server.LogDir, wid, s.cfg.Role == config.RoleMill, isFinished)
106
93
if err != nil {
107
-
return fmt.Errorf("failed to tail log file: %w", err)
94
+
return fmt.Errorf("failed to follow workflow log: %w", err)
108
95
}
109
-
defer t.Stop()
96
+
defer stop()
110
97
111
98
for {
112
99
select {
113
100
case <-ctx.Done():
114
101
return ctx.Err()
115
-
case line := <-t.Lines:
116
-
if line == nil && isFinished {
117
-
return fmt.Errorf("tail completed")
102
+
case line, ok := <-lines:
103
+
if !ok && isFinished {
104
+
return fmt.Errorf("log completed")
105
+
}
106
+
if !ok {
107
+
return fmt.Errorf("log channel closed unexpectedly")
118
108
}
119
-
120
109
if line == nil {
121
-
return fmt.Errorf("tail channel closed unexpectedly")
110
+
continue
122
111
}
123
-
124
112
if line.Err != nil {
125
-
return fmt.Errorf("error tailing log file: %w", line.Err)
113
+
return fmt.Errorf("error following workflow log: %w", line.Err)
126
114
}
127
115
128
116
if err := conn.WriteMessage(websocket.TextMessage, []byte(line.Text)); err != nil {
+8
-20
spindle/xrpc/ci_pipeline_subscribe_logs.go
+8
-20
spindle/xrpc/ci_pipeline_subscribe_logs.go
···
4
4
"context"
5
5
"encoding/json"
6
6
"fmt"
7
-
"io"
8
7
"net/http"
9
8
"sync"
10
9
"time"
···
12
11
"github.com/bluesky-social/indigo/atproto/atclient"
13
12
"github.com/bluesky-social/indigo/atproto/syntax"
14
13
"github.com/gorilla/websocket"
15
-
"github.com/hpcloud/tail"
16
14
"tangled.org/core/api/tangled"
15
+
"tangled.org/core/spindle/config"
16
+
"tangled.org/core/spindle/logview"
17
17
"tangled.org/core/spindle/models"
18
18
)
19
19
···
166
166
isFinished = models.StatusKind(status.Status).IsFinish()
167
167
}
168
168
169
-
filePath := models.LogFilePath(x.Config.Server.LogDir, wid)
170
-
171
-
tailConfig := tail.Config{
172
-
Follow: !isFinished,
173
-
ReOpen: !isFinished,
174
-
MustExist: false,
175
-
Location: &tail.SeekInfo{
176
-
Offset: 0,
177
-
Whence: io.SeekStart,
178
-
},
179
-
}
180
-
181
-
t, err := tail.TailFile(filePath, tailConfig)
169
+
lines, stop, err := logview.Follow(ctx, x.Db, x.Config.Server.LogDir, wid, x.Config.Role == config.RoleMill, isFinished)
182
170
if err != nil {
183
-
l.Error("failed to tail log file", "workflow", wfName, "err", err)
171
+
l.Error("failed to follow workflow log", "workflow", wfName, "err", err)
184
172
return
185
173
}
186
-
defer t.Stop()
174
+
defer stop()
187
175
188
-
// if we are following, poll status in database to stop tailing when finished
176
+
// if we are following, poll status in database to stop when finished
189
177
if !isFinished {
190
178
go func() {
191
179
ticker := time.NewTicker(2 * time.Second)
···
197
185
case <-ticker.C:
198
186
status, err := x.Db.GetStatus(wid)
199
187
if err == nil && models.StatusKind(status.Status).IsFinish() {
200
-
t.Stop()
188
+
stop()
201
189
return
202
190
}
203
191
}
···
209
197
select {
210
198
case <-ctx.Done():
211
199
return
212
-
case line, ok := <-t.Lines:
200
+
case line, ok := <-lines:
213
201
if !ok || line == nil {
214
202
return
215
203
}
+7
-8
workflow/compile.go
+7
-8
workflow/compile.go
···
112
112
}
113
113
114
114
func (compiler *Compiler) compileWorkflow(w Workflow) *tangled.Pipeline_Workflow {
115
-
cw := &tangled.Pipeline_Workflow{}
116
-
117
115
matched, err := w.Match(compiler.Trigger, compiler.ChangedFiles)
118
116
if err != nil {
119
117
compiler.Diagnostics.AddError(
···
134
132
// validate clone options
135
133
compiler.analyzeCloneOptions(w)
136
134
137
-
cw.Name = w.Name
138
-
139
135
if w.Engine == "" {
140
136
compiler.Diagnostics.AddError(w.Name, MissingEngine)
141
137
return nil
142
138
}
143
139
144
-
cw.Engine = w.Engine
145
-
cw.Raw = w.Raw
146
-
147
140
o := w.CloneOpts.AsRecord()
148
-
cw.Clone = &o
141
+
cw := &tangled.Pipeline_Workflow{
142
+
Clone: &o,
143
+
Engine: w.Engine,
144
+
Name: w.Name,
145
+
Raw: w.Raw,
146
+
RunsOn: w.RunsOn,
147
+
}
149
148
150
149
return cw
151
150
}
+45
workflow/compile_test.go
+45
workflow/compile_test.go
···
1
1
package workflow
2
2
3
3
import (
4
+
"encoding/json"
4
5
"strings"
5
6
"testing"
6
7
···
39
40
assert.Equal(t, wf.Name, cp.Workflows[0].Name)
40
41
assert.False(t, cp.Workflows[0].Clone.Skip)
41
42
assert.False(t, c.Diagnostics.IsErr())
43
+
}
44
+
45
+
func TestCompileWorkflow_RunsOnLabelsPersistWithoutFanout(t *testing.T) {
46
+
wf := Workflow{
47
+
Name: ".tangled/workflows/arm64.yml",
48
+
Engine: "microvm",
49
+
When: when,
50
+
RunsOn: []string{"linux/arm64", "kvm"},
51
+
}
52
+
53
+
c := Compiler{Trigger: trigger}
54
+
cp := c.Compile([]Workflow{wf})
55
+
56
+
assert.Len(t, cp.Workflows, 1)
57
+
assert.Equal(t, []string{"linux/arm64", "kvm"}, cp.Workflows[0].RunsOn)
58
+
59
+
raw, err := json.Marshal(cp.Workflows[0])
60
+
assert.NoError(t, err)
61
+
62
+
var persisted map[string]any
63
+
assert.NoError(t, json.Unmarshal(raw, &persisted))
64
+
assert.Equal(t, []any{"linux/arm64", "kvm"}, persisted["runsOn"])
65
+
}
66
+
67
+
func TestCompileWorkflow_LegacyWorkflowOmitsRunsOn(t *testing.T) {
68
+
wf := Workflow{
69
+
Name: ".tangled/workflows/legacy.yml",
70
+
Engine: "microvm",
71
+
When: when,
72
+
}
73
+
74
+
c := Compiler{Trigger: trigger}
75
+
cp := c.Compile([]Workflow{wf})
76
+
77
+
assert.Len(t, cp.Workflows, 1)
78
+
assert.Empty(t, cp.Workflows[0].RunsOn)
79
+
80
+
raw, err := json.Marshal(cp.Workflows[0])
81
+
assert.NoError(t, err)
82
+
83
+
var persisted map[string]any
84
+
assert.NoError(t, json.Unmarshal(raw, &persisted))
85
+
_, ok := persisted["runsOn"]
86
+
assert.False(t, ok, "legacy workflow JSON should not gain runsOn")
42
87
}
43
88
44
89
func TestCompileWorkflow_TriggerMismatch(t *testing.T) {
+14
-7
workflow/def.go
+14
-7
workflow/def.go
···
27
27
Workflow struct {
28
28
Name string `yaml:"-"` // name of the workflow file
29
29
Engine string `yaml:"engine"`
30
+
RunsOn []string `yaml:"runs_on"`
30
31
When []Constraint `yaml:"when"`
31
32
CloneOpts CloneOpts `yaml:"clone"`
32
33
Raw string `yaml:"-"`
···
121
122
}
122
123
123
124
func (c *Constraint) Match(trigger tangled.Pipeline_TriggerMetadata, changedFiles []string) (bool, error) {
124
-
match := true
125
-
126
125
// manual triggers always pass this constraint
127
126
if trigger.Manual != nil {
128
127
return true, nil
129
128
}
130
129
131
130
// apply event constraints
132
-
match = match && c.MatchEvent(trigger.Kind)
131
+
if !c.MatchEvent(trigger.Kind) {
132
+
return false, nil
133
+
}
133
134
134
135
// apply branch constraints for PRs
135
136
if trigger.PullRequest != nil {
···
137
138
if err != nil {
138
139
return false, err
139
140
}
140
-
match = match && matched
141
+
if !matched {
142
+
return false, nil
143
+
}
141
144
}
142
145
143
146
// apply ref constraints for pushes
···
146
149
if err != nil {
147
150
return false, err
148
151
}
149
-
match = match && matched
152
+
if !matched {
153
+
return false, nil
154
+
}
150
155
}
151
156
152
157
// apply paths filter: if specified, at least one changed file must match
···
155
160
if err != nil {
156
161
return false, err
157
162
}
158
-
match = match && matched
163
+
if !matched {
164
+
return false, nil
165
+
}
159
166
}
160
167
161
-
return match, nil
168
+
return true, nil
162
169
}
163
170
164
171
// matchesAnyFile returns true if any file in files matches any of the glob patterns.
+13
workflow/def_test.go
+13
workflow/def_test.go
···
23
23
assert.False(t, wf.CloneOpts.Skip, "Skip should default to false")
24
24
}
25
25
26
+
func TestUnmarshalWorkflowWithRunsOnLabels(t *testing.T) {
27
+
yamlData := `
28
+
engine: microvm
29
+
runs_on: [linux/arm64, kvm]
30
+
when:
31
+
- event: push`
32
+
33
+
wf, err := FromFile("test.yml", []byte(yamlData))
34
+
assert.NoError(t, err)
35
+
36
+
assert.Equal(t, []string{"linux/arm64", "kvm"}, wf.RunsOn)
37
+
}
38
+
26
39
func TestUnmarshalCloneFalse(t *testing.T) {
27
40
yamlData := `
28
41
when: