zzstoatzz.io
zig langref cli
nate.tngl.io/zigman
1//! zigman library: parse the zig language reference (a single static HTML page
2//! with stable per-section `id` anchors) into sections and clean markdown.
3//!
4//! The CLI in `main.zig` is a thin shell over this; everything here is pure
5//! (no I/O), so it is all unit-testable.
6
7const std = @import("std");
8
9pub const render = @import("render.zig").render;
10
11/// A heading in the reference's table of contents.
12pub const Section = struct {
13 level: u8, // 1..6
14 id: []const u8, // anchor, e.g. "Sentinel-Terminated-Slices"
15 title: []const u8, // visible text, e.g. "Sentinel-Terminated Slices"
16};
17
18/// Iterates `<hN id="...">title</hN>` headings in document order.
19pub const Sections = struct {
20 html: []const u8,
21 pos: usize = 0,
22 buf: [256]u8 = undefined, // backing store for the cleaned title
23
24 pub fn init(html: []const u8) Sections {
25 return .{ .html = html };
26 }
27
28 pub fn next(it: *Sections) ?Section {
29 while (std.mem.indexOfPos(u8, it.html, it.pos, "<h")) |h| {
30 it.pos = h + 2;
31 const level = headingLevel(it.html[h..]) orelse continue;
32 const gt = std.mem.indexOfScalarPos(u8, it.html, h, '>') orelse return null;
33 const id = attr(it.html[h..gt], "id") orelse continue;
34 const close = std.mem.indexOfPos(u8, it.html, gt, "</h") orelse return null;
35 const title = cleanText(&it.buf, it.html[gt + 1 .. close]);
36 it.pos = close;
37 return .{ .level = level, .id = id, .title = title };
38 }
39 return null;
40 }
41};
42
43/// Returns the HTML fragment for the section with the given anchor: from its
44/// heading up to the next heading of equal-or-higher rank. `null` if absent.
45pub fn sliceSection(html: []const u8, anchor: []const u8) ?[]const u8 {
46 var buf: [128]u8 = undefined;
47 const needle = std.fmt.bufPrint(&buf, "id=\"{s}\"", .{anchor}) catch return null;
48 const at = std.mem.indexOf(u8, html, needle) orelse return null;
49 const tag_open = std.mem.lastIndexOfScalar(u8, html[0..at], '<') orelse return null;
50 const level = headingLevel(html[tag_open..]) orelse return null;
51 var i = at;
52 while (std.mem.indexOfPos(u8, html, i, "<h")) |h| {
53 if (h > tag_open) {
54 if (headingLevel(html[h..])) |l| {
55 if (l <= level) return html[tag_open..h];
56 }
57 }
58 i = h + 2;
59 }
60 return html[tag_open..];
61}
62
63/// Normalize a string to lowercase alphanumerics only (drops spaces, hyphens,
64/// punctuation) so "Error Union Type", "error-union-type", and "error union"
65/// compare on equal footing. Writes into `buf`, returns the used slice.
66fn normalize(buf: []u8, s: []const u8) []const u8 {
67 var n: usize = 0;
68 for (s) |c| {
69 if (std.ascii.isAlphanumeric(c) and n < buf.len) {
70 buf[n] = std.ascii.toLower(c);
71 n += 1;
72 }
73 }
74 return buf[0..n];
75}
76
77/// equal, or equal after dropping a trailing plural 's' on either side
78/// ("optional" ~ "optionals").
79fn eqlOrPlural(a: []const u8, b: []const u8) bool {
80 if (std.mem.eql(u8, a, b)) return true;
81 if (a.len > 1 and a[a.len - 1] == 's' and std.mem.eql(u8, a[0 .. a.len - 1], b)) return true;
82 if (b.len > 1 and b[b.len - 1] == 's' and std.mem.eql(u8, b[0 .. b.len - 1], a)) return true;
83 return false;
84}
85
86/// The "obvious primary" section for a query, when there is one — so a fuzzy
87/// query opens the right section instead of dumping a candidate list. Tiers:
88/// 1. a section whose normalized title/id equals the query (modulo plural),
89/// preferring the shallowest such section;
90/// 2. else, if exactly one section's normalized title *starts with* the query.
91/// Returns that section's id, or null (let the caller fall back to listing).
92pub fn primaryMatch(html: []const u8, query: []const u8) ?[]const u8 {
93 var qbuf: [128]u8 = undefined;
94 const nq = normalize(&qbuf, query);
95 if (nq.len == 0) return null;
96
97 // tier 1: exact (or plural) title/id equality, shallowest wins
98 var best_id: ?[]const u8 = null;
99 var best_level: u8 = 255;
100 // tier 2: unique title prefix
101 var prefix_id: ?[]const u8 = null;
102 var prefix_count: usize = 0;
103
104 var it = Sections.init(html);
105 var nbuf: [256]u8 = undefined;
106 while (it.next()) |s| {
107 const nt = normalize(&nbuf, s.title);
108 if (eqlOrPlural(nt, nq)) {
109 if (s.level < best_level) {
110 best_level = s.level;
111 best_id = s.id;
112 }
113 continue;
114 }
115 // id equality is a strong signal too
116 var ibuf: [256]u8 = undefined;
117 if (eqlOrPlural(normalize(&ibuf, s.id), nq)) {
118 if (s.level < best_level) {
119 best_level = s.level;
120 best_id = s.id;
121 }
122 continue;
123 }
124 if (std.mem.startsWith(u8, nt, nq)) {
125 prefix_count += 1;
126 prefix_id = s.id;
127 }
128 }
129 if (best_id) |id| return id;
130 if (prefix_count == 1) return prefix_id;
131 return null;
132}
133
134/// A langref version is used in both a URL and a cache file path, so constrain
135/// it to characters that can't escape either: "master" or `[0-9A-Za-z.-]+`.
136pub fn validVersion(v: []const u8) bool {
137 if (v.len == 0 or v.len > 32) return false;
138 for (v) |c| {
139 const ok = (c >= '0' and c <= '9') or (c >= 'a' and c <= 'z') or
140 (c >= 'A' and c <= 'Z') or c == '.' or c == '-';
141 if (!ok) return false;
142 }
143 return true;
144}
145
146/// The langref URL for a version, optionally deep-linked to a section anchor.
147/// Caller owns the returned string.
148pub fn url(a: std.mem.Allocator, ver: []const u8, anchor: ?[]const u8) ![]u8 {
149 if (anchor) |id| return std.fmt.allocPrint(a, "https://ziglang.org/documentation/{s}/#{s}", .{ ver, id });
150 return std.fmt.allocPrint(a, "https://ziglang.org/documentation/{s}/", .{ver});
151}
152
153/// A query matches a section if it is a case-insensitive substring of either
154/// the anchor or the visible title — so `slice`, `Slice`, and `terminated` all
155/// find `Sentinel-Terminated-Slices`.
156pub fn matchesQuery(s: Section, query: []const u8) bool {
157 return containsIgnoreCase(s.id, query) or containsIgnoreCase(s.title, query);
158}
159
160pub fn containsIgnoreCase(haystack: []const u8, needle: []const u8) bool {
161 if (needle.len == 0) return true;
162 if (needle.len > haystack.len) return false;
163 var i: usize = 0;
164 while (i + needle.len <= haystack.len) : (i += 1) {
165 if (std.ascii.eqlIgnoreCase(haystack[i .. i + needle.len], needle)) return true;
166 }
167 return false;
168}
169
170pub fn headingLevel(s: []const u8) ?u8 {
171 if (s.len < 3 or s[0] != '<' or s[1] != 'h') return null;
172 if (s[2] < '1' or s[2] > '6') return null;
173 return s[2] - '0';
174}
175
176pub fn attr(tag: []const u8, name: []const u8) ?[]const u8 {
177 var key_buf: [32]u8 = undefined;
178 const key = std.fmt.bufPrint(&key_buf, "{s}=\"", .{name}) catch return null;
179 const start = std.mem.indexOf(u8, tag, key) orelse return null;
180 const vstart = start + key.len;
181 const vend = std.mem.indexOfScalarPos(u8, tag, vstart, '"') orelse return null;
182 return tag[vstart..vend];
183}
184
185/// Visible text of a heading: drop tags and the § pilcrow. Truncates to `buf`.
186fn cleanText(buf: []u8, inner: []const u8) []const u8 {
187 var n: usize = 0;
188 var k: usize = 0;
189 var in_tag = false;
190 while (k < inner.len and n < buf.len) {
191 const c = inner[k];
192 if (c == '<') {
193 in_tag = true;
194 } else if (c == '>') {
195 in_tag = false;
196 } else if (!in_tag) {
197 if (c == 0xC2 and k + 1 < inner.len and inner[k + 1] == 0xA7) {
198 k += 2;
199 continue;
200 }
201 buf[n] = c;
202 n += 1;
203 }
204 k += 1;
205 }
206 return std.mem.trim(u8, buf[0..n], " ");
207}
208
209// ----------------------------------------------------------------------------
210// tests
211// ----------------------------------------------------------------------------
212
213const testing = std.testing;
214
215test "render: prose with inline code and entities" {
216 const md = try render(testing.allocator, "<p>a < b and <code>len</code></p>");
217 defer testing.allocator.free(md);
218 try testing.expectEqualStrings("a < b and `len`", md);
219}
220
221test "render: fenced code block, spans and entities decoded" {
222 const html =
223 \\<pre><code><span class="tok-kw">const</span> x = <span class="tok-str">"hi"</span>;</code></pre>
224 ;
225 const md = try render(testing.allocator, html);
226 defer testing.allocator.free(md);
227 try testing.expectEqualStrings("```zig\nconst x = \"hi\";\n```", md);
228}
229
230test "render: table becomes markdown rows" {
231 const html = "<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>";
232 const md = try render(testing.allocator, html);
233 defer testing.allocator.free(md);
234 try testing.expectEqualStrings("| A | B |\n| --- | --- |\n| 1 | 2 |", md);
235}
236
237test "sliceSection: bounded by next equal-rank heading" {
238 const html = "<h2 id=\"a\">A</h2><p>x</p><h2 id=\"b\">B</h2>";
239 const sec = sliceSection(html, "a").?;
240 try testing.expectEqualStrings("<h2 id=\"a\">A</h2><p>x</p>", sec);
241 try testing.expect(sliceSection(html, "missing") == null);
242}
243
244test "sliceSection: subsections stay within parent" {
245 const html = "<h2 id=\"p\">P</h2><h3 id=\"c\">C</h3><p>y</p><h2 id=\"q\">Q</h2>";
246 const sec = sliceSection(html, "p").?;
247 try testing.expectEqualStrings("<h2 id=\"p\">P</h2><h3 id=\"c\">C</h3><p>y</p>", sec);
248}
249
250test "Sections: yields level, id, cleaned title" {
251 const html = "<h2 id=\"Slices\"><a href=\"#toc\">Slices</a> <a class=\"hdr\">§</a></h2>" ++
252 "<h3 id=\"Sub\">Sub Thing</h3>";
253 var it = Sections.init(html);
254 const a = it.next().?;
255 try testing.expectEqual(@as(u8, 2), a.level);
256 try testing.expectEqualStrings("Slices", a.id);
257 try testing.expectEqualStrings("Slices", a.title);
258 const b = it.next().?;
259 try testing.expectEqual(@as(u8, 3), b.level);
260 try testing.expectEqualStrings("Sub Thing", b.title);
261 try testing.expect(it.next() == null);
262}
263
264test "validVersion: accepts releases and master, rejects path escapes" {
265 try testing.expect(validVersion("master"));
266 try testing.expect(validVersion("0.15.1"));
267 try testing.expect(!validVersion("../../etc/passwd"));
268 try testing.expect(!validVersion("a/b"));
269 try testing.expect(!validVersion(""));
270}
271
272test "url: with and without an anchor" {
273 const a = testing.allocator;
274 const root = try url(a, "master", null);
275 defer a.free(root);
276 try testing.expectEqualStrings("https://ziglang.org/documentation/master/", root);
277 const deep = try url(a, "0.15.1", "comptime");
278 defer a.free(deep);
279 try testing.expectEqualStrings("https://ziglang.org/documentation/0.15.1/#comptime", deep);
280}
281
282test "primaryMatch: fuzzy query resolves to the obvious section" {
283 const html =
284 "<h3 id=\"while-with-Optionals\">while with Optionals</h3>" ++
285 "<h3 id=\"Optional-Type\">Optional Type</h3>" ++
286 "<h2 id=\"Optionals\">Optionals</h2>" ++
287 "<h3 id=\"Error-Union-Type\">Error Union Type</h3>" ++
288 "<h3 id=\"while-with-Error-Unions\">while with Error Unions</h3>" ++
289 "<h2 id=\"Blocks\">Blocks</h2>";
290 // tier 1: plural-normalized title equality, even though substrings match more
291 try testing.expectEqualStrings("Optionals", primaryMatch(html, "optional").?);
292 try testing.expectEqualStrings("Optionals", primaryMatch(html, "Optionals").?);
293 try testing.expectEqualStrings("Blocks", primaryMatch(html, "blocks").?);
294 // tier 2: unique title prefix ("error union" prefixes only "Error Union Type")
295 try testing.expectEqualStrings("Error-Union-Type", primaryMatch(html, "error union").?);
296 // genuinely ambiguous prefix -> null (let caller list)
297 try testing.expect(primaryMatch(html, "while") == null);
298 try testing.expect(primaryMatch(html, "xyzzy") == null);
299}
300
301test "matchesQuery: anchor or title, case-insensitive" {
302 const s: Section = .{ .level = 3, .id = "Sentinel-Terminated-Slices", .title = "Sentinel-Terminated Slices" };
303 try testing.expect(matchesQuery(s, "slice")); // anchor + title
304 try testing.expect(matchesQuery(s, "terminated")); // title word
305 try testing.expect(matchesQuery(s, "SENTINEL")); // case-insensitive
306 try testing.expect(!matchesQuery(s, "comptime"));
307}
308
309test "containsIgnoreCase" {
310 try testing.expect(containsIgnoreCase("Slices", "slice"));
311 try testing.expect(containsIgnoreCase("Optionals", "OPT"));
312 try testing.expect(!containsIgnoreCase("Errors", "slice"));
313 try testing.expect(containsIgnoreCase("anything", ""));
314}