bas.sh
forked from
tangled.org/core
Monorepo for Tangled
21 kB
599 lines
1{
2 description = "atproto github";
3
4 inputs = {
5 nixpkgs.url = "github:nixos/nixpkgs/nixos-26.05";
6 microvm = {
7 url = "github:microvm-nix/microvm.nix";
8 inputs.nixpkgs.follows = "nixpkgs";
9 };
10 fenix = {
11 url = "github:nix-community/fenix";
12 inputs.nixpkgs.follows = "nixpkgs";
13 };
14 gomod2nix = {
15 url = "github:nix-community/gomod2nix";
16 inputs.nixpkgs.follows = "nixpkgs";
17 };
18 flake-compat = {
19 url = "https://git.lix.systems/lix-project/flake-compat/archive/main.tar.gz";
20 flake = false;
21 };
22 indigo = {
23 url = "github:oppiliappan/indigo";
24 flake = false;
25 };
26 htmx-src = {
27 url = "https://unpkg.com/htmx.org@2.0.4/dist/htmx.min.js";
28 flake = false;
29 };
30 htmx-ws-src = {
31 # strange errors in console that i can't really make out
32 # url = "https://unpkg.com/htmx.org@2.0.4/dist/ext/ws.js";
33 url = "https://cdn.jsdelivr.net/npm/htmx-ext-ws@2.0.2";
34 flake = false;
35 };
36 lucide-src = {
37 url = "https://github.com/lucide-icons/lucide/releases/download/0.536.0/lucide-icons-0.536.0.zip";
38 flake = false;
39 };
40 inter-fonts-src = {
41 url = "https://github.com/rsms/inter/releases/download/v4.1/Inter-4.1.zip";
42 flake = false;
43 };
44 actor-typeahead-src = {
45 url = "git+https://tangled.org/@jakelazaroff.com/actor-typeahead";
46 flake = false;
47 };
48 mermaid-src = {
49 url = "https://cdn.jsdelivr.net/npm/mermaid@11.12.3/dist/mermaid.min.js";
50 flake = false;
51 };
52 hls-src = {
53 url = "https://cdn.jsdelivr.net/npm/hls.js@1.5.13/dist/hls.min.js";
54 flake = false;
55 };
56 ibm-plex-mono-src = {
57 url = "https://github.com/IBM/plex/releases/download/%40ibm%2Fplex-mono%401.1.0/ibm-plex-mono.zip";
58 flake = false;
59 };
60 sqlite-lib-src = {
61 url = "https://sqlite.org/2024/sqlite-amalgamation-3450100.zip";
62 flake = false;
63 };
64 fetch-tangled = {
65 url = "git+https://tangled.org/isabelroses.com/fetch-tangled";
66 inputs.nixpkgs.follows = "nixpkgs";
67 };
68 };
69
70 outputs = {
71 self,
72 nixpkgs,
73 fenix,
74 gomod2nix,
75 indigo,
76 htmx-src,
77 htmx-ws-src,
78 lucide-src,
79 inter-fonts-src,
80 sqlite-lib-src,
81 ibm-plex-mono-src,
82 actor-typeahead-src,
83 mermaid-src,
84 hls-src,
85 microvm,
86 fetch-tangled,
87 ...
88 }: let
89 supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"];
90 forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
91 nixpkgsFor = forAllSystems (system:
92 import nixpkgs {
93 inherit system;
94 overlays = [fetch-tangled.overlays.default];
95 });
96
97 mkPackageSet = pkgs:
98 pkgs.lib.makeScope pkgs.newScope (self: {
99 src = let
100 fs = pkgs.lib.fileset;
101 in
102 fs.toSource {
103 root = ./.;
104 fileset = fs.difference (fs.intersection (fs.gitTracked ./.) (fs.fileFilter (file: !(file.hasExt "nix")) ./.)) (fs.maybeMissing ./.jj);
105 };
106 rustSrc = let
107 fs = pkgs.lib.fileset;
108 in
109 fs.toSource {
110 root = ./.;
111 fileset =
112 fs.intersection
113 (fs.fromSource self.src)
114 (fs.unions [
115 ./Cargo.toml
116 ./Cargo.lock
117 ./shuttle
118 ./bobbin
119 ]);
120 };
121 buildGoApplication =
122 (self.callPackage "${gomod2nix}/builder" {
123 gomod2nix = gomod2nix.legacyPackages.${pkgs.stdenv.hostPlatform.system}.gomod2nix;
124 }).buildGoApplication;
125 rustPlatform = pkgs.makeRustPlatform {
126 inherit (fenix.packages.${pkgs.stdenv.hostPlatform.system}.stable) rustc cargo;
127 };
128 rustPlatformStatic = let
129 system = pkgs.stdenv.hostPlatform.system;
130 muslTarget = pkgs.pkgsStatic.stdenv.hostPlatform.rust.rustcTarget;
131 toolchain = fenix.packages.${system}.combine [
132 fenix.packages.${system}.stable.cargo
133 fenix.packages.${system}.stable.rustc
134 fenix.packages.${system}.targets.${muslTarget}.stable.rust-std
135 ];
136 in
137 pkgs.pkgsStatic.makeRustPlatform {
138 cargo = toolchain;
139 rustc = toolchain;
140 };
141 modules = ./nix/gomod2nix.toml;
142 sqlite-lib = self.callPackage ./nix/pkgs/sqlite-lib.nix {
143 inherit sqlite-lib-src;
144 };
145 lexgen = self.callPackage ./nix/pkgs/lexgen.nix {inherit indigo;};
146 goat = self.callPackage ./nix/pkgs/goat.nix {inherit indigo;};
147 appview-static-files = self.callPackage ./nix/pkgs/appview-static-files.nix {
148 inherit htmx-src htmx-ws-src lucide-src inter-fonts-src ibm-plex-mono-src actor-typeahead-src mermaid-src hls-src;
149 };
150 appview = self.callPackage ./nix/pkgs/appview.nix {};
151 blog = self.callPackage ./nix/pkgs/blog.nix {};
152 docs = self.callPackage ./nix/pkgs/docs.nix {
153 inherit inter-fonts-src ibm-plex-mono-src lucide-src;
154 inherit (pkgs) pagefind;
155 };
156 spindle = self.callPackage ./nix/pkgs/spindle.nix {};
157 shuttle = self.callPackage ./nix/pkgs/shuttle.nix {
158 src = self.rustSrc;
159 };
160 shuttle-static = self.callPackage ./nix/pkgs/shuttle.nix {
161 src = self.rustSrc;
162 rustPlatform = self.rustPlatformStatic;
163 };
164 knot-unwrapped = self.callPackage ./nix/pkgs/knot-unwrapped.nix {};
165 knot = self.callPackage ./nix/pkgs/knot.nix {};
166 dolly = self.callPackage ./nix/pkgs/dolly.nix {};
167 tap = self.callPackage ./nix/pkgs/tap.nix {};
168 knotmirror = self.callPackage ./nix/pkgs/knotmirror.nix {};
169 bobbin = self.callPackage ./nix/pkgs/bobbin.nix {};
170 zoekt-webserver = self.callPackage ./nix/pkgs/zoekt-webserver.nix {};
171 zoekt-tngl-indexserver = self.callPackage ./nix/pkgs/zoekt-tngl-indexserver.nix {};
172 });
173 in {
174 overlays.default = final: prev: {
175 inherit
176 (mkPackageSet final)
177 lexgen
178 goat
179 sqlite-lib
180 spindle
181 shuttle
182 knot-unwrapped
183 knot
184 appview
185 docs
186 dolly
187 tap
188 knotmirror
189 bobbin
190 zoekt-webserver
191 zoekt-tngl-indexserver
192 ;
193 };
194
195 packages = forAllSystems (system: let
196 pkgs = nixpkgsFor.${system};
197 linuxPkgs = nixpkgsFor."x86_64-linux";
198 packages = mkPackageSet pkgs;
199 staticPackages = mkPackageSet pkgs.pkgsStatic;
200 crossPackages = mkPackageSet pkgs.pkgsCross.gnu64.pkgsStatic;
201 in {
202 inherit
203 (packages)
204 appview
205 appview-static-files
206 blog
207 lexgen
208 goat
209 spindle
210 knot
211 knot-unwrapped
212 sqlite-lib
213 docs
214 shuttle
215 shuttle-static
216 dolly
217 tap
218 knotmirror
219 bobbin
220 zoekt-webserver
221 zoekt-tngl-indexserver
222 ;
223
224 pkgsStatic-appview = staticPackages.appview;
225 pkgsStatic-knot = staticPackages.knot;
226 pkgsStatic-knot-unwrapped = staticPackages.knot-unwrapped;
227 pkgsStatic-spindle = staticPackages.spindle;
228 pkgsStatic-sqlite-lib = staticPackages.sqlite-lib;
229 pkgsStatic-dolly = staticPackages.dolly;
230
231 pkgsCross-gnu64-pkgsStatic-appview = crossPackages.appview;
232 pkgsCross-gnu64-pkgsStatic-knot = crossPackages.knot;
233 pkgsCross-gnu64-pkgsStatic-knot-unwrapped = crossPackages.knot-unwrapped;
234 pkgsCross-gnu64-pkgsStatic-spindle = crossPackages.spindle;
235 pkgsCross-gnu64-pkgsStatic-dolly = crossPackages.dolly;
236
237 treefmt-wrapper = pkgs.treefmt.withConfig {
238 settings.formatter = {
239 alejandra = {
240 command = pkgs.lib.getExe pkgs.alejandra;
241 includes = ["*.nix"];
242 };
243
244 gofmt = {
245 command = pkgs.lib.getExe' pkgs.go "gofmt";
246 options = ["-w"];
247 includes = ["*.go"];
248 };
249
250 rustfmt = {
251 command = pkgs.lib.getExe' fenix.packages.${system}.stable.rustfmt "rustfmt";
252 options = ["--edition" "2024"];
253 includes = ["*.rs"];
254 excludes = ["**/src/_lex/**"];
255 };
256
257 # prettier = let
258 # wrapper = pkgs.runCommandLocal "prettier-wrapper" {nativeBuildInputs = [pkgs.makeWrapper];} ''
259 # makeWrapper ${pkgs.prettier}/bin/prettier "$out" --add-flags "--plugin=${pkgs.prettier-plugin-go-template}/lib/node_modules/prettier-plugin-go-template/lib/index.js"
260 # '';
261 # in {
262 # command = wrapper;
263 # options = ["-w"];
264 # includes = ["*.html"];
265 # # causes Go template plugin errors: https://github.com/NiklasPor/prettier-plugin-go-template/issues/120
266 # excludes = ["appview/pages/templates/layouts/repobase.html" "appview/pages/templates/repo/tags.html"];
267 # };
268 };
269 };
270
271 spindle-nixos-image = linuxPkgs.callPackage ./nix/pkgs/spindle-nixos-image.nix {
272 nixosSystem = self.nixosConfigurations.spindle-nixos;
273 };
274 spindle-nixos-image-tarball = linuxPkgs.runCommand "spindle-nixos-image-tarball.tar.gz" {} ''
275 tar -S -C ${self.packages.${system}.spindle-nixos-image} -h -czf $out .
276 '';
277
278 spindle-alpine-image = let
279 branch = "3.24";
280 version = "${branch}.0";
281 arch = "x86_64";
282 cdn = "https://dl-cdn.alpinelinux.org/alpine/v${branch}/releases/${arch}";
283
284 shuttle = (mkPackageSet linuxPkgs).shuttle-static;
285 in
286 linuxPkgs.callPackage ./nix/pkgs/spindle-alpine-image.nix {
287 inherit arch shuttle;
288 repositories = [
289 "https://dl-cdn.alpinelinux.org/alpine/v${branch}/main"
290 "https://dl-cdn.alpinelinux.org/alpine/v${branch}/community"
291 ];
292 rootfs = linuxPkgs.fetchurl {
293 url = "${cdn}/alpine-minirootfs-${version}-${arch}.tar.gz";
294 hash = "sha256-3poRwODn6clNs+2K97RQ6vwLE2h71+kZnVUFDyCqCok=";
295 };
296 kernel = linuxPkgs.fetchurl {
297 url = "${cdn}/netboot-${version}/vmlinuz-virt";
298 hash = "sha256-Hmv5Ancgx1w+0NeRcfIbV5HuQMqXldB8fG4E3F6irpA=";
299 };
300 initramfs = linuxPkgs.fetchurl {
301 url = "${cdn}/netboot-${version}/initramfs-virt";
302 hash = "sha256-ZCWGSaVMOYOmLz1Gwsf2RhYarMqk+tFVA6MMDWiHVJQ=";
303 };
304 modloop = linuxPkgs.fetchurl {
305 url = "${cdn}/netboot-${version}/modloop-virt";
306 hash = "sha256-p3yO7yU28k04iT01sOzhDmEYi+Yl7VZs5r3RYsWCBX0=";
307 };
308 };
309 spindle-alpine-image-tarball = linuxPkgs.runCommand "spindle-alpine-image-tarball.tar.gz" {} ''
310 tar -S -C ${self.packages.${system}.spindle-alpine-image} -h -czf $out .
311 '';
312 });
313 defaultPackage = forAllSystems (system: self.packages.${system}.appview);
314 devShells = forAllSystems (system: let
315 pkgs = nixpkgsFor.${system};
316 packages' = self.packages.${system};
317 staticShell = args:
318 (pkgs.mkShell.override {
319 stdenv = pkgs.pkgsStatic.stdenv;
320 }) (args
321 // {
322 nativeBuildInputs =
323 args.nativeBuildInputs
324 ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [
325 pkgs.darwin.cctools
326 ];
327 });
328 in {
329 default = staticShell {
330 nativeBuildInputs =
331 [
332 pkgs.go
333 pkgs.air
334 pkgs.gopls
335 pkgs.httpie
336 pkgs.litecli
337 pkgs.websocat
338 pkgs.tailwindcss
339 pkgs.nixos-shell
340 pkgs.redis
341 pkgs.worker-build
342 pkgs.cargo-generate
343 pkgs.qemu
344 pkgs.cdrkit
345 pkgs.buf
346 pkgs.protobuf
347 pkgs.protoc-gen-prost
348 pkgs.protoc-gen-prost-crate
349 pkgs.protoc-gen-prost-serde
350 pkgs.protoc-gen-go
351 (fenix.packages.${system}.combine [
352 fenix.packages.${system}.stable.cargo
353 fenix.packages.${system}.stable.rustc
354 fenix.packages.${system}.stable.rust-src
355 fenix.packages.${system}.stable.clippy
356 fenix.packages.${system}.stable.rustfmt
357 fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.rust-std
358 ])
359 pkgs.coreutils # for those of us who are on systems that use busybox (alpine)
360 packages'.lexgen
361 packages'.treefmt-wrapper
362 packages'.tap
363 pkgs.e2fsprogs
364 pkgs.util-linux
365 ]
366 ++ pkgs.lib.optionals pkgs.stdenv.isLinux [
367 pkgs.parted
368 pkgs.slirp4netns
369 pkgs.iproute2
370 ];
371 shellHook = ''
372 mkdir -p appview/pages/static
373 # temporary self-heal for workspaces that copied static assets as read-only
374 [ -d appview/pages/static/icons ] && [ ! -w appview/pages/static/icons ] && chmod -R u+rwX appview/pages/static
375 # no preserve is needed because watch-tailwind will want to be able to overwrite
376 cp -fr --no-preserve=ownership,mode ${packages'.appview-static-files}/* appview/pages/static
377 export TANGLED_OAUTH_CLIENT_KID="$(date +%s)"
378 export TANGLED_OAUTH_CLIENT_SECRET="$(${packages'.goat}/bin/goat key generate -t P-256 | grep -A1 "Secret Key" | tail -n1 | awk '{print $1}')"
379 # Make xcrun (Nix stub) able to find ld from the system Command Line Tools.
380 # Without this, worker-build/cargo fails with "error: tool 'ld' not found".
381 if [ -d /Library/Developer/CommandLineTools/usr/bin ]; then
382 export PATH=/Library/Developer/CommandLineTools/usr/bin:$PATH
383 fi
384 '';
385 env.CGO_ENABLED = 1;
386 };
387 });
388 apps = forAllSystems (system: let
389 pkgs = nixpkgsFor."${system}";
390 packages' = self.packages.${system};
391 air-watcher = name: arg:
392 pkgs.writeShellScriptBin "run"
393 ''
394 export PATH=${pkgs.go}/bin:$PATH
395 ${pkgs.air}/bin/air -c ./.air/${name}.toml \
396 -build.args_bin "${arg}"
397 '';
398 tailwind-watcher =
399 pkgs.writeShellScriptBin "run"
400 ''
401 export BROWSERSLIST_IGNORE_OLD_DATA=true
402 ${pkgs.tailwindcss}/bin/tailwindcss --watch=always -i input.css -o ./appview/pages/static/tw.css
403 '';
404 in {
405 watch-appview = {
406 type = "app";
407 program = toString (pkgs.writeShellScript "watch-appview" ''
408 echo "copying static files to appview/pages/static..."
409 mkdir -p appview/pages/static
410 ${pkgs.coreutils}/bin/cp -fr --no-preserve=ownership,mode ${packages'.appview-static-files}/* appview/pages/static
411 ${air-watcher "appview" ""}/bin/run
412 '');
413 };
414 watch-knot = {
415 type = "app";
416 program = ''${air-watcher "knot" "server"}/bin/run'';
417 };
418 watch-spindle = {
419 type = "app";
420 program = ''${air-watcher "spindle" ""}/bin/run'';
421 };
422 watch-blog = {
423 type = "app";
424 program = toString (pkgs.writeShellScript "watch-blog" ''
425 echo "copying static files to appview/pages/static..."
426 mkdir -p appview/pages/static
427 ${pkgs.coreutils}/bin/cp -fr --no-preserve=ownership,mode ${packages'.appview-static-files}/* appview/pages/static
428 ${air-watcher "blog" "serve"}/bin/run
429 '');
430 };
431 watch-tailwind = {
432 type = "app";
433 program = ''${tailwind-watcher}/bin/run'';
434 };
435 serve-docs = {
436 type = "app";
437 program = toString (pkgs.writeShellScript "serve-docs" ''
438 echo "building docs..."
439 docsOut=$(nix build --no-link --print-out-paths .#docs)
440 echo "serving docs at http://localhost:1414"
441 cd "$docsOut"
442 exec ${pkgs.python3}/bin/python3 -m http.server 1414
443 '');
444 };
445 regenerate-proto = {
446 type = "app";
447 program =
448 (pkgs.writeShellApplication {
449 name = "regenerate-proto";
450 runtimeInputs = with pkgs; [git buf coreutils];
451 text = ''
452 rootDir=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
453 cd "$rootDir"
454 echo ">>> regenerating protobuf files.."
455 buf generate
456 echo ">>> generating file descriptor set for shuttle..."
457 buf build -o shuttle/src/gen/file_descriptor_set.bin
458 echo ">>> done"
459 '';
460 })
461 + "/bin/regenerate-proto";
462 };
463 vm = let
464 guestSystem =
465 if pkgs.stdenv.hostPlatform.isAarch64
466 then "aarch64-linux"
467 else "x86_64-linux";
468 in {
469 type = "app";
470 program =
471 (pkgs.writeShellApplication {
472 name = "launch-vm";
473 text = ''
474 rootDir=$(jj --ignore-working-copy root || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1)
475 cd "$rootDir"
476
477 # Reset TMPDIR in case it points to a stale nix-shell temp dir
478 export TMPDIR=/tmp
479
480 mkdir -p nix/vm-data/{knot,repos,spindle,spindle-logs}
481
482 export TANGLED_VM_DATA_DIR="$rootDir/nix/vm-data"
483 exec ${pkgs.lib.getExe
484 (import ./nix/vm.nix {
485 inherit nixpkgs self;
486 system = guestSystem;
487 hostSystem = system;
488 }).config.system.build.vm}
489 '';
490 })
491 + /bin/launch-vm;
492 };
493 gomod2nix = {
494 type = "app";
495 program = toString (pkgs.writeShellScript "gomod2nix" ''
496 ${gomod2nix.legacyPackages.${system}.gomod2nix}/bin/gomod2nix generate --outdir ./nix
497 '');
498 };
499 lexgen = {
500 type = "app";
501 program =
502 (pkgs.writeShellApplication {
503 name = "lexgen";
504 text = ''
505 if ! command -v lexgen > /dev/null; then
506 echo "error: must be executed from devshell"
507 exit 1
508 fi
509
510 rootDir=$(jj --ignore-working-copy root || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1)
511 cd "$rootDir"
512
513 rm -f api/tangled/*
514 lexgen --build-file lexicon-build-config.json lexicons
515 sed -i.bak 's/\tutil/\/\/\tutil/' api/tangled/*
516 # lexgen generates incomplete Marshaler/Unmarshaler for union types
517 find api/tangled/*.go -not -name "cbor_gen.go" -exec \
518 sed -i '/^func.*\(MarshalCBOR\|UnmarshalCBOR\)/,/^}/ s/^/\/\/ /' {} +
519 ${pkgs.gotools}/bin/goimports -w api/tangled/*
520 CGO_ENABLED=0 go run ./cmd/cborgen/
521 lexgen --build-file lexicon-build-config.json lexicons
522 rm api/tangled/*.bak
523 '';
524 })
525 + /bin/lexgen;
526 };
527 });
528
529 nixosModules.appview = {
530 lib,
531 pkgs,
532 ...
533 }: {
534 imports = [./nix/modules/appview.nix];
535
536 services.tangled.appview.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.appview;
537 };
538 nixosModules.knotmirror = {
539 lib,
540 pkgs,
541 ...
542 }: {
543 imports = [./nix/modules/knotmirror.nix];
544
545 services.tangled.knotmirror.tap-package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.tap;
546 services.tangled.knotmirror.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.knotmirror;
547 };
548 nixosModules.zoekt-tnglserver = {
549 lib,
550 pkgs,
551 ...
552 }: {
553 imports = [./nix/modules/zoekt-tnglserver.nix];
554
555 services.tangled.zoekt.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.zoekt-tngl-indexserver;
556 services.tangled.zoekt.zoekt-webserver-package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.zoekt-webserver;
557 };
558 nixosModules.knot = {
559 lib,
560 pkgs,
561 ...
562 }: {
563 imports = [./nix/modules/knot.nix];
564
565 services.tangled.knot.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.knot;
566 };
567 nixosModules.spindle = {
568 lib,
569 pkgs,
570 ...
571 }: {
572 imports = [./nix/modules/spindle.nix];
573
574 services.tangled.spindle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.spindle;
575 };
576 nixosModules.shuttle = {
577 lib,
578 pkgs,
579 ...
580 }: {
581 imports = [./nix/modules/shuttle.nix];
582
583 services.tangled.shuttle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.shuttle;
584 };
585
586 nixosModules.spindle-nixos = import ./nix/microvm/spindle-vm.nix {inherit self microvm;} ./nix/microvm/qemu.nix;
587
588 formatter = forAllSystems (system: self.packages.${system}.treefmt-wrapper);
589
590 nixosConfigurations = let
591 spindleNixosBase = nixpkgs.lib.nixosSystem {
592 system = "x86_64-linux";
593 modules = [self.nixosModules.spindle-nixos];
594 };
595 in {
596 spindle-nixos = spindleNixosBase;
597 };
598 };
599}