zzstoatzz.io
This repository has no description
1//! record_to_text heuristic — port of embedder/build_pack.py.
2//!
3//! walks arbitrary atproto record JSON and produces embedding-ready text.
4//! rules:
5//! - prepend `collection: <nsid>` so the lex name is in the vector
6//! - drop atproto plumbing keys ($type, cid, rev, sig, prev, version, $link)
7//! - drop identifier-shaped strings (DIDs, at-uris, CIDs, TID rkeys, hex hashes)
8//! - convert iso timestamps to year only (keeps temporal signal, not noise)
9//! - render as flat `key.path: value` lines
10//! - strongRefs and DIDs are currently dropped (v2: deref via slingshot)
11//! - truncate final text at 4000 chars
12
13const std = @import("std");
14const json = std.json;
15const Allocator = std.mem.Allocator;
16
17const MAX_TEXT_LEN = 4000;
18
19const NOISE_KEYS = [_][]const u8{
20 "$type", "cid", "rev", "sig", "prev", "version", "$link",
21};
22
23fn isNoiseKey(k: []const u8) bool {
24 if (k.len == 0) return false;
25 if (k[0] == '$') return true;
26 for (NOISE_KEYS) |n| if (std.mem.eql(u8, k, n)) return true;
27 return false;
28}
29
30fn isIdentifier(s: []const u8) bool {
31 if (s.len == 0) return false;
32 if (std.mem.startsWith(u8, s, "did:plc:")) return true;
33 if (std.mem.startsWith(u8, s, "did:web:")) return true;
34 if (std.mem.startsWith(u8, s, "at://")) return true;
35 // CID v1 raw/dag-cbor
36 if (std.mem.startsWith(u8, s, "bafy")) return true;
37 if (std.mem.startsWith(u8, s, "bafk")) return true;
38 // TID format: exactly 13 chars, lowercase base32 subset [2-7a-z]
39 if (s.len == 13) {
40 var all_tid = true;
41 for (s) |ch| {
42 const ok = (ch >= 'a' and ch <= 'z') or (ch >= '2' and ch <= '7');
43 if (!ok) {
44 all_tid = false;
45 break;
46 }
47 }
48 if (all_tid) return true;
49 }
50 // hex hash of 32+ chars
51 if (s.len >= 32) {
52 var all_hex = true;
53 for (s) |ch| {
54 const ok = (ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'f');
55 if (!ok) {
56 all_hex = false;
57 break;
58 }
59 }
60 if (all_hex) return true;
61 }
62 return false;
63}
64
65fn looksLikeIsoTimestamp(s: []const u8) bool {
66 // YYYY-MM-DDT... pattern (loose)
67 if (s.len < 11) return false;
68 for (0..4) |i| if (!std.ascii.isDigit(s[i])) return false;
69 if (s[4] != '-') return false;
70 for (5..7) |i| if (!std.ascii.isDigit(s[i])) return false;
71 if (s[7] != '-') return false;
72 for (8..10) |i| if (!std.ascii.isDigit(s[i])) return false;
73 return s[10] == 'T' or s[10] == ' ';
74}
75
76/// shape check: is this a strongRef {uri: "at://...", cid: "bafy..."}?
77fn isStrongRef(v: json.Value) bool {
78 if (v != .object) return false;
79 const uri_v = v.object.get("uri") orelse return false;
80 const cid_v = v.object.get("cid") orelse return false;
81 if (uri_v != .string or cid_v != .string) return false;
82 return std.mem.startsWith(u8, uri_v.string, "at://");
83}
84
85pub const Builder = struct {
86 buf: std.ArrayList(u8),
87 allocator: Allocator,
88
89 pub fn init(allocator: Allocator) Builder {
90 return .{ .buf = .empty, .allocator = allocator };
91 }
92
93 fn appendLine(self: *Builder, path: []const u8, value: []const u8) !void {
94 if (self.buf.items.len >= MAX_TEXT_LEN) return;
95 if (self.buf.items.len > 0) try self.buf.append(self.allocator, '\n');
96 if (path.len > 0) {
97 try self.buf.appendSlice(self.allocator, path);
98 try self.buf.appendSlice(self.allocator, ": ");
99 }
100 try self.buf.appendSlice(self.allocator, value);
101 }
102};
103
104/// walk a parsed record value, appending flattened text to `builder`.
105fn walk(builder: *Builder, node: json.Value, path: []const u8) anyerror!void {
106 if (builder.buf.items.len >= MAX_TEXT_LEN) return;
107
108 // strongRef short-circuit: drop (v2: inline dereffed content)
109 if (isStrongRef(node)) return;
110
111 switch (node) {
112 .object => |obj| {
113 var it = obj.iterator();
114 while (it.next()) |entry| {
115 const k = entry.key_ptr.*;
116 if (isNoiseKey(k)) continue;
117
118 const next_path = if (path.len == 0)
119 try builder.allocator.dupe(u8, k)
120 else
121 try std.fmt.allocPrint(builder.allocator, "{s}.{s}", .{ path, k });
122
123 try walk(builder, entry.value_ptr.*, next_path);
124 }
125 },
126 .array => |arr| {
127 // pure string arrays → comma-joined
128 var all_strings = true;
129 for (arr.items) |item| {
130 if (item != .string) {
131 all_strings = false;
132 break;
133 }
134 }
135 if (all_strings) {
136 var kept: std.ArrayList([]const u8) = .empty;
137 for (arr.items) |item| {
138 if (!isIdentifier(item.string)) {
139 try kept.append(builder.allocator, item.string);
140 }
141 }
142 if (kept.items.len > 0) {
143 const joined = try std.mem.join(builder.allocator, ", ", kept.items);
144 try builder.appendLine(path, joined);
145 }
146 } else {
147 for (arr.items) |item| try walk(builder, item, path);
148 }
149 },
150 .string => |s| {
151 // DID reference → drop (v2: inline profile)
152 if (std.mem.startsWith(u8, s, "did:plc:") or std.mem.startsWith(u8, s, "did:web:")) return;
153 if (isIdentifier(s)) return;
154 if (looksLikeIsoTimestamp(s)) {
155 try builder.appendLine(path, s[0..4]); // keep year only
156 return;
157 }
158 try builder.appendLine(path, s);
159 },
160 .integer => |i| {
161 if (i == 0) return;
162 const s = try std.fmt.allocPrint(builder.allocator, "{d}", .{i});
163 try builder.appendLine(path, s);
164 },
165 .float => |f| {
166 if (f == 0.0) return;
167 const s = try std.fmt.allocPrint(builder.allocator, "{d}", .{f});
168 try builder.appendLine(path, s);
169 },
170 .bool => return, // rarely meaningful for retrieval
171 .null => return,
172 .number_string => |s| try builder.appendLine(path, s),
173 }
174}
175
176/// produce embedding-ready text for a record. result is allocated in `arena`
177/// and truncated at MAX_TEXT_LEN.
178pub fn recordToText(
179 arena: Allocator,
180 collection: []const u8,
181 value: json.Value,
182) ![]const u8 {
183 var b = Builder.init(arena);
184 // prepend the NSID — strongest structural signal
185 try b.appendLine("collection", collection);
186 try walk(&b, value, "");
187 const len = @min(b.buf.items.len, MAX_TEXT_LEN);
188 return b.buf.items[0..len];
189}