zzstoatzz.io
search for standard sites
pub-search.waow.tech
search
zig
blog
atproto
refactor(bans): single source of truth for banned DIDs (banned-dids.txt)
The ban list lived in four places across two languages (policy.zig, ingester,
both purge scripts) — adding crownnote meant editing all four, exactly the
smell the exclusions doc flagged as debt. Consolidate to /banned-dids.txt at
the repo root:
- Zig (backend + ingester): wired in via build.zig as an anonymous import,
@embedFile'd and parsed at comptime into BANNED_DIDS. The existing
policy.zig tests (exact DIDs + exact banned_dids_sql string) guard the parse.
- Python (both purge scripts): read the file at runtime.
DIDs now live in exactly one place; each consumer references it. Registry of
who/why/evidence stays in docs/exclusions.md.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
author
zzstoatzz
co-author
Claude Opus 4.8
date
(Jun 29, 2026, 11:00 AM -0500)
commit
216f6d2d
216f6d2dfb5403bb18aa376b80ce80df190f5bd8
parent
db275093
db27509337d581ae826a51f2d5ccbcc506d2f553
+94
-30
+7
backend/build.zig
+7
backend/build.zig
···
61
61
}),
62
62
});
63
63
64
+
// single source of truth for banned DIDs (repo root), embedded at comptime
65
+
// by policy.zig. mirrored at runtime by scripts/purge-*. see banned-dids.txt.
66
+
const banned_dids_mod = b.createModule(.{ .root_source_file = b.path("../banned-dids.txt") });
67
+
exe.root_module.addImport("banned_dids", banned_dids_mod);
68
+
64
69
b.installArtifact(exe);
65
70
66
71
const run_cmd = b.addRunArtifact(exe);
···
82
87
.imports = imports,
83
88
}),
84
89
});
90
+
91
+
unit_tests.root_module.addImport("banned_dids", banned_dids_mod);
85
92
86
93
const run_tests = b.addRunArtifact(unit_tests);
87
94
const test_step = b.step("test", "Run unit tests");
+21
-9
backend/src/policy.zig
+21
-9
backend/src/policy.zig
···
6
6
7
7
const std = @import("std");
8
8
9
-
/// Bulk-archive repos that never enter the corpus (purged 2026-06-10).
10
-
pub const BANNED_DIDS = [_][]const u8{
11
-
// registry of who/why/evidence: docs/exclusions.md — every entry here
12
-
// must have one there. mirror changes into ingester/src/main.zig and
13
-
// scripts/purge-banned-turso + purge-bridgyfed-vectors.
14
-
"did:plc:oql6ds5vnff4ugar6rruliwd", // drivepatents.com patent bot
15
-
"did:plc:2s32mlusc66sjb256aenynfc", // destinationcharged.com NHTSA recall mirror
16
-
"did:plc:llnmp5t7s3u4dzjqyhp76h62", // crownnote.com music-charts mirror
17
-
};
9
+
/// Bulk-archive repos that never enter the corpus. The list is the single
10
+
/// source of truth in `/banned-dids.txt` (repo root), wired in via build.zig
11
+
/// and parsed here at comptime. Registry of who/why/evidence: docs/exclusions.md.
12
+
pub const BANNED_DIDS = parseBannedDids(@embedFile("banned_dids"));
13
+
14
+
/// Parse the shared banned-dids.txt: one DID per line, '#' starts a comment
15
+
/// (whole-line or inline), blank lines ignored.
16
+
fn parseBannedDids(comptime data: []const u8) []const []const u8 {
17
+
comptime {
18
+
@setEvalBranchQuota(100_000);
19
+
var list: []const []const u8 = &.{};
20
+
var lines = std.mem.splitScalar(u8, data, '\n');
21
+
while (lines.next()) |raw| {
22
+
const code = if (std.mem.indexOfScalar(u8, raw, '#')) |h| raw[0..h] else raw;
23
+
const did = std.mem.trim(u8, code, " \t\r");
24
+
if (did.len == 0) continue;
25
+
list = list ++ &[_][]const u8{did};
26
+
}
27
+
return list;
28
+
}
29
+
}
18
30
19
31
pub fn isBanned(did: []const u8) bool {
20
32
for (BANNED_DIDS) |banned| {
+17
banned-dids.txt
+17
banned-dids.txt
···
1
+
# Banned DIDs — the single source of truth for corpus exclusions.
2
+
#
3
+
# Registry of who/when/why/evidence: docs/exclusions.md (every DID here MUST
4
+
# have an entry there).
5
+
#
6
+
# Consumed by:
7
+
# - backend/src/policy.zig (comptime @embedFile -> isBanned / banned_dids_sql)
8
+
# - ingester/src/main.zig (comptime @embedFile -> firehose drop)
9
+
# - scripts/purge-banned-turso (runtime read)
10
+
# - scripts/purge-bridgyfed-vectors (runtime read)
11
+
#
12
+
# Format: one DID per line. Blank lines and lines starting with '#' are
13
+
# ignored; an inline '# comment' after the DID is allowed.
14
+
15
+
did:plc:oql6ds5vnff4ugar6rruliwd # drivepatents.com patent bot
16
+
did:plc:2s32mlusc66sjb256aenynfc # destinationcharged.com NHTSA recall mirror
17
+
did:plc:llnmp5t7s3u4dzjqyhp76h62 # crownnote.com music-charts mirror
+6
ingester/build.zig
+6
ingester/build.zig
···
25
25
}),
26
26
});
27
27
28
+
// shared single source of truth for banned DIDs (repo root); embedded at
29
+
// comptime by main.zig. see banned-dids.txt + backend/src/policy.zig.
30
+
const banned_dids_mod = b.createModule(.{ .root_source_file = b.path("../banned-dids.txt") });
31
+
exe.root_module.addImport("banned_dids", banned_dids_mod);
32
+
28
33
b.installArtifact(exe);
29
34
30
35
const run_cmd = b.addRunArtifact(exe);
···
43
48
.imports = imports,
44
49
}),
45
50
});
51
+
unit_tests.root_module.addImport("banned_dids", banned_dids_mod);
46
52
const run_tests = b.addRunArtifact(unit_tests);
47
53
const test_step = b.step("test", "Run unit tests");
48
54
test_step.dependOn(&run_tests.step);
+20
-8
ingester/src/main.zig
+20
-8
ingester/src/main.zig
···
33
33
"com.whtwnd.blog.entry",
34
34
};
35
35
36
-
// banned repos: bulk-archive bots that flooded the corpus (96k+ automotive
37
-
// patents, ~44%+ of all docs). decision 2026-06-10: purge and ban entirely.
38
-
const BANNED_DIDS = [_][]const u8{
39
-
// registry of who/why: docs/exclusions.md (mirror of backend policy.zig)
40
-
"did:plc:oql6ds5vnff4ugar6rruliwd", // drivepatents.com
41
-
"did:plc:2s32mlusc66sjb256aenynfc", // destinationcharged.com NHTSA recall mirror
42
-
"did:plc:llnmp5t7s3u4dzjqyhp76h62", // crownnote.com music-charts mirror
43
-
};
36
+
// banned repos: bulk-archive bots that flood the corpus. Single source of
37
+
// truth is /banned-dids.txt (repo root), shared with backend/src/policy.zig
38
+
// and scripts/purge-*; wired in via build.zig, parsed here at comptime.
39
+
// registry of who/why/evidence: docs/exclusions.md.
40
+
const BANNED_DIDS = parseBannedDids(@embedFile("banned_dids"));
41
+
42
+
fn parseBannedDids(comptime data: []const u8) []const []const u8 {
43
+
comptime {
44
+
@setEvalBranchQuota(100_000);
45
+
var list: []const []const u8 = &.{};
46
+
var lines = std.mem.splitScalar(u8, data, '\n');
47
+
while (lines.next()) |raw| {
48
+
const code = if (std.mem.indexOfScalar(u8, raw, '#')) |h| raw[0..h] else raw;
49
+
const did = std.mem.trim(u8, code, " \t\r");
50
+
if (did.len == 0) continue;
51
+
list = list ++ &[_][]const u8{did};
52
+
}
53
+
return list;
54
+
}
55
+
}
44
56
45
57
fn isBanned(did: []const u8) bool {
46
58
for (BANNED_DIDS) |b| {
+15
-7
scripts/purge-banned-turso
+15
-7
scripts/purge-banned-turso
···
34
34
import httpx
35
35
from pydantic_settings import BaseSettings, SettingsConfigDict
36
36
37
-
# mirrors backend/src/policy.zig BANNED_DIDS
38
-
BANNED_DIDS = [
39
-
# registry of who/why: docs/exclusions.md (mirror of backend policy.zig)
40
-
"did:plc:oql6ds5vnff4ugar6rruliwd", # drivepatents.com patent bot
41
-
"did:plc:2s32mlusc66sjb256aenynfc", # destinationcharged.com NHTSA recall mirror
42
-
"did:plc:llnmp5t7s3u4dzjqyhp76h62", # crownnote.com music-charts mirror
43
-
]
37
+
def load_banned_dids() -> list[str]:
38
+
"""Single source of truth: /banned-dids.txt at the repo root (shared with
39
+
backend/src/policy.zig + ingester). One DID per line; '#' starts a comment."""
40
+
import pathlib
41
+
42
+
path = pathlib.Path(__file__).resolve().parent.parent / "banned-dids.txt"
43
+
out = []
44
+
for line in path.read_text().splitlines():
45
+
did = line.split("#", 1)[0].strip()
46
+
if did:
47
+
out.append(did)
48
+
return out
49
+
50
+
51
+
BANNED_DIDS = load_banned_dids()
44
52
45
53
BATCH_SIZE = 300
46
54
SLEEP_BETWEEN_BATCHES = 12.0
+8
-6
scripts/purge-bridgyfed-vectors
+8
-6
scripts/purge-bridgyfed-vectors
···
21
21
import argparse
22
22
import asyncio
23
23
import os
24
+
import pathlib
24
25
import sys
25
26
import time
26
27
···
175
176
if not args.apply:
176
177
log("=== DRY RUN (pass --apply to delete) ===\n")
177
178
178
-
# banned bulk-archive bots — purged unconditionally (mirrors the ingester's
179
-
# BANNED_DIDS list)
180
-
# registry of who/why: docs/exclusions.md (mirror of backend policy.zig)
179
+
# banned bulk-archive bots — purged unconditionally. Single source of truth:
180
+
# /banned-dids.txt at the repo root (shared with policy.zig + ingester).
181
+
# registry of who/why/evidence: docs/exclusions.md
182
+
banned_path = pathlib.Path(__file__).resolve().parent.parent / "banned-dids.txt"
181
183
banned = {
182
-
"did:plc:oql6ds5vnff4ugar6rruliwd", # drivepatents.com
183
-
"did:plc:2s32mlusc66sjb256aenynfc", # destinationcharged.com NHTSA recall mirror
184
-
"did:plc:llnmp5t7s3u4dzjqyhp76h62", # crownnote.com music-charts mirror
184
+
did
185
+
for line in banned_path.read_text().splitlines()
186
+
if (did := line.split("#", 1)[0].strip())
185
187
}
186
188
187
189
sync_client = httpx.Client()