tangled.org
Monorepo for Tangled
tangled.org
5.6 kB
157 lines
1# turns the primary spindle into a mill host and runs a three-executor fleet
2# against it. the executors differ on two axes so placement is testable:
3#
4# executor labels images can run
5# --------- ------------- ----------- ------------------------
6# executor-a linux, fast full image/alpine, image/nixos
7# executor-b linux, slow full image/alpine, image/nixos
8# executor-c linux, gpu alpine only image/alpine
9
10x-mill-executor: &mill-executor
11 profiles: ["linux"]
12 build:
13 context: .
14 dockerfile: localinfra/spindle.Dockerfile
15 restart: unless-stopped
16 environment: &mill-executor-env
17 SPINDLE_ROLE: executor
18 SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555
19 SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db
20 SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev
21 SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe
22 SPINDLE_SERVER_DEV: "true"
23 SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev
24 SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db
25 SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev
26 SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images
27 SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays
28 SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts
29 SPINDLE_MILL_ARTIFACT_STORE: disk
30 SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false"
31 SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501
32 SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU=
33 SPINDLE_NIX_CACHE_UPLOAD_URL: http://ncps:8501/upload
34 # dials the mill container directly using ws
35 SPINDLE_MILL_URL: ws://spindle:6555/mill
36 devices:
37 - /dev/vsock:/dev/vsock
38 - /dev/kvm:/dev/kvm
39 - /dev/vhost-vsock:/dev/vhost-vsock
40 - /dev/net/tun:/dev/net/tun
41 cap_add:
42 - NET_ADMIN
43 - SYS_ADMIN
44 security_opt:
45 - label=disable
46 - seccomp=unconfined
47 healthcheck:
48 test: ["CMD", "wget", "-qO-", "http://localhost:6555/"]
49 interval: 2s
50 timeout: 2s
51 retries: 30
52 start_period: 5s
53 depends_on: &mill-executor-deps
54 plc:
55 condition: service_started
56 jetstream:
57 condition: service_started
58 init-accounts:
59 condition: service_completed_successfully
60 ncps:
61 condition: service_started
62 spindle:
63 condition: service_healthy
64 mill-tokens:
65 condition: service_completed_successfully
66 networks: [tngl]
67
68services:
69 # configures the primary spindle container as a mill host
70 spindle:
71 environment:
72 SPINDLE_ROLE: mill
73 SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts
74 SPINDLE_MILL_ARTIFACT_STORE: disk
75 volumes:
76 - spindle-artifacts:/var/lib/spindle/artifacts
77
78 mill-tokens:
79 profiles: ["linux"]
80 build:
81 context: .
82 dockerfile: localinfra/spindle.Dockerfile
83 restart: "no"
84 entrypoint: ["/bin/sh", "-c"]
85 environment:
86 SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db
87 command:
88 - |
89 set -eu
90 seed() {
91 spindle mill executor revoke "$$1" >/dev/null 2>&1 || true
92 args=""
93 for l in $$(echo "$$2" | tr ',' ' '); do args="$$args --label $$l"; done
94 spindle mill executor add "$$1" $$args > "/shared/$$1.mill-token"
95 }
96 seed executor-a linux,fast
97 seed executor-b linux,slow
98 seed executor-c linux,gpu
99 echo "seeded executor tokens"
100 volumes:
101 - spindle-data:/var/lib/spindle
102 - init-state:/shared
103 depends_on:
104 spindle:
105 condition: service_healthy
106 networks: [tngl]
107
108 spindle-executor-a:
109 <<: *mill-executor
110 environment:
111 <<: *mill-executor-env
112 SPINDLE_SERVER_HOSTNAME: executor-a.tngl.boltless.dev
113 SPINDLE_MILL_LABELS: linux,fast
114 SPINDLE_MILL_TOKEN_FILE: /shared/executor-a.mill-token
115 SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11241"
116 volumes:
117 - spindle-executor-a-data:/var/lib/spindle
118 - spindle-artifacts:/var/lib/spindle/artifacts
119 - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro
120 - init-state:/shared:ro
121 - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro
122
123 spindle-executor-b:
124 <<: *mill-executor
125 environment:
126 <<: *mill-executor-env
127 SPINDLE_SERVER_HOSTNAME: executor-b.tngl.boltless.dev
128 SPINDLE_MILL_LABELS: linux,slow
129 SPINDLE_MILL_TOKEN_FILE: /shared/executor-b.mill-token
130 SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11242"
131 volumes:
132 - spindle-executor-b-data:/var/lib/spindle
133 - spindle-artifacts:/var/lib/spindle/artifacts
134 - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro
135 - init-state:/shared:ro
136 - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro
137
138 spindle-executor-c:
139 <<: *mill-executor
140 environment:
141 <<: *mill-executor-env
142 SPINDLE_SERVER_HOSTNAME: executor-c.tngl.boltless.dev
143 SPINDLE_MILL_LABELS: linux,gpu
144 SPINDLE_MILL_TOKEN_FILE: /shared/executor-c.mill-token
145 SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11243"
146 volumes:
147 - spindle-executor-c-data:/var/lib/spindle
148 - spindle-artifacts:/var/lib/spindle/artifacts
149 - ./out/localinfra-spindle-images-alpine:/var/lib/spindle/images:ro
150 - init-state:/shared:ro
151 - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro
152
153volumes:
154 spindle-executor-a-data:
155 spindle-executor-b-data:
156 spindle-executor-c-data:
157 spindle-artifacts: