solpbc.org / solstone
personal memory agent
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

fix(indexer): match Python markdown chunk grouping

Native markdown chunking projected about 170,580 excess chunks on the 71,208-file real corpus: 1,377,411 native chunks versus 1,206,831 Python chunks. Add the generated markdown oracle fixture, the isolated differential corpus, and the native implementation changes in one commit because the green-expecting integration test would fail if committed before the behavior fix.

Port the Python structural rules into the native chunker: intro-paragraph consumption before lists and tables; definition-list grouping when at least two items match and matches are at least 50%; one chunk per top-level list item with nested and multi-block item content concatenated in document order; table row splitting with headers carried into every body-row chunk and no header-only chunk; single-chunk blockquotes; heading-only and thematic-break-only inputs producing no chunks; removal of the whole-input fallback; code-fence info strings retained; and pulldown-cmark Options::ENABLE_TABLES.

Replace raw production chunk_markdown access with one bounded format_markdown entry point at both markdown producers. That entry point performs the 2048-char markdown line sanitization, carries warnings through ProducedChunks and ScanReport, and applies the 4096-char header-stub cap.

Capture red-then-green evidence with the unchanged Jaccard >= 0.90 and mutual-top-3 gate. Before the fix, markdown_parity_single_term, markdown_parity_and, and markdown_parity_phrase were each 0.667 Jaccard and markdown_parity_prefix_code_info was 0.0; the run classified unexpected-differs with failed_components ["fulltext"], reproduced on fe07e0f58. After the fix, all five parity cases are 1.0, classification is functionally-equal, and failed_components is empty.

Delete the tautological markdown_producer_wraps_chunker_without_content_changes test and replace it with oracle-driven producer tests plus Rust oracle token assertions against the generated fixture.

Document the single accepted divergence: oversized-chunk stub size tokens are normalized in the fixture because exact parity would require byte-identical rendering, which is explicitly out of scope.

author
Jer Miller date (Jul 21, 2026, 2:44 PM -0600) commit 971f4f84 parent 24d7a1b4
+2526 -109
+24 -1
core/crates/solstone-core-indexer-store/src/scan.rs
··· 702 702 let facet = metadata.facet.to_lowercase(); 703 703 let agent = produced 704 704 .agent_override 705 + .clone() 705 706 .unwrap_or_else(|| metadata.agent.clone()) 706 707 .to_lowercase(); 707 708 let stream_lookup = extract_stream(journal, rel); 708 709 let stream = stream_lookup.stream; 709 710 let bucket = time_bucket(rel); 710 - let warnings: Vec<String> = stream_lookup.warning.into_iter().collect(); 711 + let mut warnings = produced.warnings; 712 + warnings.extend(stream_lookup.warning); 711 713 712 714 for (idx, chunk) in produced.chunks.iter().enumerate() { 713 715 let content = chunk.content.trim(); ··· 1886 1888 0 1887 1889 ); 1888 1890 fs::remove_dir_all(root).expect("cleanup mtime root"); 1891 + } 1892 + 1893 + #[test] 1894 + fn scan_propagates_markdown_sanitize_warnings() { 1895 + let root = temp_root("markdown-sanitize-warning"); 1896 + let rel = "20260717/talents/flow.md"; 1897 + write( 1898 + &root, 1899 + &format!("chronicle/{rel}"), 1900 + &format!("# Flow\n\n{}\n\nkept alpha", "z".repeat(2049)), 1901 + ); 1902 + 1903 + let report = scan_journal(&root, true, "20260717").expect("scan markdown warning"); 1904 + assert_eq!(report.indexed, 1); 1905 + assert_eq!( 1906 + report.warnings, 1907 + vec!["Dropped 1 line(s) exceeding 2048 chars during markdown sanitization"] 1908 + ); 1909 + let conn = Connection::open(db_path(&root)).expect("open db"); 1910 + assert_eq!(chunk_content(&conn, rel), "# Flow\n\nkept alpha"); 1911 + fs::remove_dir_all(root).expect("cleanup markdown warning root"); 1889 1912 } 1890 1913 1891 1914 #[test]
+929 -73
core/crates/solstone-core-indexer/src/chunker.rs
··· 1 1 // SPDX-License-Identifier: AGPL-3.0-only 2 2 // Copyright (c) 2026 sol pbc 3 3 4 - use pulldown_cmark::{Event, HeadingLevel, Parser, Tag, TagEnd}; 4 + use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, LinkType, Options, Parser, Tag, TagEnd}; 5 + 6 + const MAX_LINE_CHARS: usize = 2048; 7 + const MAX_CHUNK_CHARS: usize = 4096; 8 + const OVERLONG_LINE_WARNING: &str = 9 + "Dropped {count} line(s) exceeding 2048 chars during markdown sanitization"; 5 10 6 11 #[derive(Debug, Clone, PartialEq, Eq)] 7 12 pub struct MarkdownChunk { 8 13 pub markdown: String, 9 14 } 10 15 11 - pub fn chunk_markdown(input: &str) -> Vec<MarkdownChunk> { 12 - let mut chunks = Vec::new(); 13 - let mut headers: Vec<(HeadingLevel, String)> = Vec::new(); 14 - let mut heading_level = None; 15 - let mut heading_text = String::new(); 16 - let mut block_text = String::new(); 17 - let mut in_heading = false; 16 + #[derive(Debug, Clone, PartialEq, Eq)] 17 + pub struct MarkdownFormat { 18 + pub chunks: Vec<MarkdownChunk>, 19 + pub warnings: Vec<String>, 20 + } 21 + 22 + #[derive(Debug, Clone, PartialEq, Eq)] 23 + struct Header { 24 + level: HeadingLevel, 25 + text: String, 26 + } 27 + 28 + #[derive(Debug, Clone, PartialEq, Eq)] 29 + enum MarkdownBlock { 30 + Heading(Header), 31 + Paragraph(String), 32 + List(ListBlock), 33 + Table(TableBlock), 34 + Code(CodeBlock), 35 + BlockQuote(String), 36 + ThematicBreak, 37 + HtmlBlock, 38 + } 39 + 40 + #[derive(Debug, Clone, PartialEq, Eq)] 41 + struct ListBlock { 42 + items: Vec<ListItem>, 43 + } 44 + 45 + #[derive(Debug, Clone, PartialEq, Eq)] 46 + struct ListItem { 47 + text: String, 48 + is_definition_item: bool, 49 + } 50 + 51 + #[derive(Debug, Clone, PartialEq, Eq)] 52 + struct TableBlock { 53 + headers: Vec<String>, 54 + rows: Vec<Vec<String>>, 55 + } 56 + 57 + #[derive(Debug, Clone, PartialEq, Eq)] 58 + struct CodeBlock { 59 + info: String, 60 + body: String, 61 + } 18 62 19 - for event in Parser::new(input) { 20 - match event { 21 - Event::Start(Tag::Heading { level, .. }) => { 22 - flush_block(&headers, &mut block_text, &mut chunks); 23 - in_heading = true; 24 - heading_level = Some(level); 25 - heading_text.clear(); 63 + #[derive(Debug, Clone, PartialEq, Eq)] 64 + struct RawChunk { 65 + headers: Vec<Header>, 66 + body: String, 67 + } 68 + 69 + pub fn format_markdown(input: &str) -> MarkdownFormat { 70 + let (sanitized, warnings) = sanitize_markdown(input); 71 + let raw_chunks = chunk_markdown(&sanitized); 72 + let chunks = raw_chunks 73 + .into_iter() 74 + .map(|chunk| { 75 + let rendered = render_chunk(&chunk); 76 + let markdown = if rendered.len() > MAX_CHUNK_CHARS { 77 + render_header_stub(&chunk.headers, rendered.len()) 78 + } else { 79 + rendered 80 + }; 81 + MarkdownChunk { markdown } 82 + }) 83 + .collect(); 84 + MarkdownFormat { chunks, warnings } 85 + } 86 + 87 + fn sanitize_markdown(input: &str) -> (String, Vec<String>) { 88 + let mut clean = Vec::new(); 89 + let mut dropped = 0usize; 90 + for line in input.split('\n') { 91 + if line.len() > MAX_LINE_CHARS { 92 + dropped += 1; 93 + } else { 94 + clean.push(line); 95 + } 96 + } 97 + let warnings = if dropped == 0 { 98 + Vec::new() 99 + } else { 100 + vec![OVERLONG_LINE_WARNING.replace("{count}", &dropped.to_string())] 101 + }; 102 + (clean.join("\n"), warnings) 103 + } 104 + 105 + fn chunk_markdown(input: &str) -> Vec<RawChunk> { 106 + let blocks = parse_blocks(input); 107 + chunk_blocks(&blocks) 108 + } 109 + 110 + fn parse_blocks(input: &str) -> Vec<MarkdownBlock> { 111 + let mut blocks = Vec::new(); 112 + let mut active = ActiveBlock::None; 113 + for event in Parser::new_ext(input, Options::ENABLE_TABLES) { 114 + active = match active { 115 + ActiveBlock::None => start_top_level(event, &mut blocks), 116 + ActiveBlock::Heading(mut heading) => { 117 + if heading.handle(event, &mut blocks) { 118 + ActiveBlock::None 119 + } else { 120 + ActiveBlock::Heading(heading) 121 + } 122 + } 123 + ActiveBlock::Paragraph(mut paragraph) => { 124 + if paragraph.handle(event, &mut blocks) { 125 + ActiveBlock::None 126 + } else { 127 + ActiveBlock::Paragraph(paragraph) 128 + } 26 129 } 27 - Event::End(TagEnd::Heading(_)) => { 28 - if let Some(level) = heading_level.take() { 29 - while headers.last().is_some_and(|(existing, _text)| { 30 - heading_rank(*existing) >= heading_rank(level) 31 - }) { 32 - headers.pop(); 33 - } 34 - let trimmed = heading_text.trim(); 35 - if !trimmed.is_empty() { 36 - headers.push((level, trimmed.to_string())); 37 - } 130 + ActiveBlock::List(mut list) => { 131 + if list.handle(event, &mut blocks) { 132 + ActiveBlock::None 133 + } else { 134 + ActiveBlock::List(list) 38 135 } 39 - in_heading = false; 40 - heading_text.clear(); 136 + } 137 + ActiveBlock::Table(mut table) => { 138 + if table.handle(event, &mut blocks) { 139 + ActiveBlock::None 140 + } else { 141 + ActiveBlock::Table(table) 142 + } 41 143 } 42 - Event::Text(text) | Event::Code(text) => { 43 - if in_heading { 44 - heading_text.push_str(&text); 144 + ActiveBlock::Code(mut code) => { 145 + if code.handle(event, &mut blocks) { 146 + ActiveBlock::None 45 147 } else { 46 - if !block_text.is_empty() && !block_text.ends_with([' ', '\n']) { 47 - block_text.push(' '); 48 - } 49 - block_text.push_str(&text); 148 + ActiveBlock::Code(code) 50 149 } 51 150 } 52 - Event::SoftBreak | Event::HardBreak => { 53 - if in_heading { 54 - heading_text.push(' '); 151 + ActiveBlock::BlockQuote(mut quote) => { 152 + if quote.handle(event, &mut blocks) { 153 + ActiveBlock::None 55 154 } else { 56 - block_text.push('\n'); 155 + ActiveBlock::BlockQuote(quote) 57 156 } 58 157 } 59 - Event::End( 60 - TagEnd::Paragraph 61 - | TagEnd::Item 62 - | TagEnd::CodeBlock 63 - | TagEnd::TableRow 64 - | TagEnd::BlockQuote(_), 65 - ) => flush_block(&headers, &mut block_text, &mut chunks), 66 - _ => {} 158 + ActiveBlock::HtmlBlock => match event { 159 + Event::End(TagEnd::HtmlBlock) => { 160 + blocks.push(MarkdownBlock::HtmlBlock); 161 + ActiveBlock::None 162 + } 163 + _ => ActiveBlock::HtmlBlock, 164 + }, 165 + }; 166 + } 167 + blocks 168 + } 169 + 170 + fn start_top_level(event: Event<'_>, blocks: &mut Vec<MarkdownBlock>) -> ActiveBlock { 171 + match event { 172 + Event::Start(Tag::Heading { level, .. }) => { 173 + ActiveBlock::Heading(HeadingBuilder::new(level)) 174 + } 175 + Event::Start(Tag::Paragraph) => ActiveBlock::Paragraph(TextBlockBuilder::default()), 176 + Event::Start(Tag::List(_)) => ActiveBlock::List(ListBuilder::new()), 177 + Event::Start(Tag::Table(_)) => ActiveBlock::Table(TableBuilder::default()), 178 + Event::Start(Tag::CodeBlock(kind)) => ActiveBlock::Code(CodeBlockBuilder::new(kind)), 179 + Event::Start(Tag::BlockQuote(_)) => ActiveBlock::BlockQuote(QuoteBuilder::default()), 180 + Event::Start(Tag::HtmlBlock) => ActiveBlock::HtmlBlock, 181 + Event::Rule => { 182 + blocks.push(MarkdownBlock::ThematicBreak); 183 + ActiveBlock::None 67 184 } 185 + _ => ActiveBlock::None, 68 186 } 69 - flush_block(&headers, &mut block_text, &mut chunks); 187 + } 188 + 189 + fn chunk_blocks(blocks: &[MarkdownBlock]) -> Vec<RawChunk> { 190 + let mut chunks = Vec::new(); 191 + let mut headers: Vec<Header> = Vec::new(); 192 + let mut intro: Option<String> = None; 70 193 71 - if chunks.is_empty() { 72 - let trimmed = input.trim(); 73 - if !trimmed.is_empty() { 74 - chunks.push(MarkdownChunk { 75 - markdown: trimmed.to_string(), 76 - }); 194 + for (idx, block) in blocks.iter().enumerate() { 195 + match block { 196 + MarkdownBlock::Heading(header) => { 197 + while headers.last().is_some_and(|existing| { 198 + heading_rank(existing.level) >= heading_rank(header.level) 199 + }) { 200 + headers.pop(); 201 + } 202 + if !header.text.trim().is_empty() { 203 + headers.push(header.clone()); 204 + } 205 + intro = None; 206 + } 207 + MarkdownBlock::Paragraph(text) => { 208 + if matches!( 209 + blocks.get(idx + 1), 210 + Some(MarkdownBlock::List(_)) | Some(MarkdownBlock::Table(_)) 211 + ) { 212 + intro = Some(text.clone()); 213 + } else { 214 + push_chunk(&mut chunks, &headers, text.clone()); 215 + intro = None; 216 + } 217 + } 218 + MarkdownBlock::List(list) => { 219 + if is_definition_list(list) { 220 + let mut body = String::new(); 221 + append_intro(&mut body, intro.as_deref()); 222 + for item in &list.items { 223 + append_piece(&mut body, &item.text); 224 + } 225 + push_chunk(&mut chunks, &headers, body); 226 + } else { 227 + for item in &list.items { 228 + let mut body = String::new(); 229 + append_intro(&mut body, intro.as_deref()); 230 + append_piece(&mut body, &item.text); 231 + push_chunk(&mut chunks, &headers, body); 232 + } 233 + } 234 + intro = None; 235 + } 236 + MarkdownBlock::Table(table) => { 237 + for row in &table.rows { 238 + let mut body = String::new(); 239 + append_intro(&mut body, intro.as_deref()); 240 + append_table_row(&mut body, &table.headers); 241 + append_table_row(&mut body, row); 242 + push_chunk(&mut chunks, &headers, body); 243 + } 244 + intro = None; 245 + } 246 + MarkdownBlock::Code(code) => { 247 + let mut body = String::new(); 248 + append_piece(&mut body, &code.info); 249 + append_piece(&mut body, &code.body); 250 + push_chunk(&mut chunks, &headers, body); 251 + intro = None; 252 + } 253 + MarkdownBlock::BlockQuote(text) => { 254 + push_chunk(&mut chunks, &headers, text.clone()); 255 + intro = None; 256 + } 257 + MarkdownBlock::ThematicBreak | MarkdownBlock::HtmlBlock => { 258 + intro = None; 259 + } 77 260 } 78 261 } 79 262 80 263 chunks 81 264 } 82 265 83 - fn flush_block( 84 - headers: &[(HeadingLevel, String)], 85 - block_text: &mut String, 86 - chunks: &mut Vec<MarkdownChunk>, 87 - ) { 88 - let trimmed = block_text.trim(); 266 + fn push_chunk(chunks: &mut Vec<RawChunk>, headers: &[Header], body: String) { 267 + if body.trim().is_empty() { 268 + return; 269 + } 270 + chunks.push(RawChunk { 271 + headers: headers.to_vec(), 272 + body, 273 + }); 274 + } 275 + 276 + fn append_intro(body: &mut String, intro: Option<&str>) { 277 + if let Some(intro) = intro { 278 + append_piece(body, intro); 279 + } 280 + } 281 + 282 + fn append_piece(body: &mut String, piece: &str) { 283 + let trimmed = piece.trim(); 89 284 if trimmed.is_empty() { 90 - block_text.clear(); 91 285 return; 92 286 } 287 + if !body.is_empty() { 288 + body.push_str("\n\n"); 289 + } 290 + body.push_str(trimmed); 291 + } 93 292 293 + fn append_table_row(body: &mut String, cells: &[String]) { 294 + if cells.is_empty() { 295 + return; 296 + } 297 + append_piece(body, &cells.join(" ")); 298 + } 299 + 300 + fn is_definition_list(list: &ListBlock) -> bool { 301 + if list.items.len() < 2 { 302 + return false; 303 + } 304 + let matches = list 305 + .items 306 + .iter() 307 + .filter(|item| item.is_definition_item) 308 + .count(); 309 + matches >= 2 && matches * 2 >= list.items.len() 310 + } 311 + 312 + fn render_chunk(chunk: &RawChunk) -> String { 94 313 let mut markdown = String::new(); 95 - for (level, text) in headers { 96 - markdown.push_str(&"#".repeat(heading_rank(*level))); 314 + for header in &chunk.headers { 315 + markdown.push_str(&"#".repeat(heading_rank(header.level))); 97 316 markdown.push(' '); 98 - markdown.push_str(text); 317 + markdown.push_str(header.text.trim()); 99 318 markdown.push_str("\n\n"); 100 319 } 101 - markdown.push_str(trimmed); 102 - chunks.push(MarkdownChunk { markdown }); 103 - block_text.clear(); 320 + markdown.push_str(chunk.body.trim()); 321 + markdown 322 + } 323 + 324 + fn render_header_stub(headers: &[Header], original_size: usize) -> String { 325 + let mut parts: Vec<String> = headers 326 + .iter() 327 + .map(|header| { 328 + format!( 329 + "{} {}", 330 + "#".repeat(heading_rank(header.level)), 331 + header.text.trim() 332 + ) 333 + }) 334 + .collect(); 335 + parts.push(format!( 336 + "\n[Content too large to index: {} chars]", 337 + format_usize_with_commas(original_size) 338 + )); 339 + parts.join("\n\n") 340 + } 341 + 342 + fn format_usize_with_commas(value: usize) -> String { 343 + let digits = value.to_string(); 344 + let mut out = String::new(); 345 + for (idx, ch) in digits.chars().rev().enumerate() { 346 + if idx > 0 && idx % 3 == 0 { 347 + out.push(','); 348 + } 349 + out.push(ch); 350 + } 351 + out.chars().rev().collect() 104 352 } 105 353 106 354 fn heading_rank(level: HeadingLevel) -> usize { ··· 114 362 } 115 363 } 116 364 365 + enum ActiveBlock { 366 + None, 367 + Heading(HeadingBuilder), 368 + Paragraph(TextBlockBuilder), 369 + List(ListBuilder), 370 + Table(TableBuilder), 371 + Code(CodeBlockBuilder), 372 + BlockQuote(QuoteBuilder), 373 + HtmlBlock, 374 + } 375 + 376 + struct HeadingBuilder { 377 + level: HeadingLevel, 378 + text: TextCollector, 379 + } 380 + 381 + impl HeadingBuilder { 382 + fn new(level: HeadingLevel) -> Self { 383 + Self { 384 + level, 385 + text: TextCollector::default(), 386 + } 387 + } 388 + 389 + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec<MarkdownBlock>) -> bool { 390 + match event { 391 + Event::End(TagEnd::Heading(_)) => { 392 + blocks.push(MarkdownBlock::Heading(Header { 393 + level: self.level, 394 + text: std::mem::take(&mut self.text).finish(), 395 + })); 396 + true 397 + } 398 + event => { 399 + self.text.handle_event(event); 400 + false 401 + } 402 + } 403 + } 404 + } 405 + 406 + #[derive(Default)] 407 + struct TextBlockBuilder { 408 + text: TextCollector, 409 + } 410 + 411 + impl TextBlockBuilder { 412 + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec<MarkdownBlock>) -> bool { 413 + match event { 414 + Event::End(TagEnd::Paragraph) => { 415 + blocks.push(MarkdownBlock::Paragraph( 416 + std::mem::take(&mut self.text).finish(), 417 + )); 418 + true 419 + } 420 + Event::Start(Tag::CodeBlock(kind)) => { 421 + self.text.push_text(&code_info(&kind)); 422 + false 423 + } 424 + event => { 425 + self.text.handle_event(event); 426 + false 427 + } 428 + } 429 + } 430 + } 431 + 432 + #[derive(Default)] 433 + struct QuoteBuilder { 434 + text: TextCollector, 435 + } 436 + 437 + impl QuoteBuilder { 438 + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec<MarkdownBlock>) -> bool { 439 + match event { 440 + Event::End(TagEnd::BlockQuote(_)) => { 441 + blocks.push(MarkdownBlock::BlockQuote( 442 + std::mem::take(&mut self.text).finish(), 443 + )); 444 + true 445 + } 446 + Event::End(TagEnd::Paragraph) | Event::End(TagEnd::Item) => { 447 + self.text.push_text("\n"); 448 + false 449 + } 450 + Event::Start(Tag::CodeBlock(kind)) => { 451 + self.text.push_text(&code_info(&kind)); 452 + false 453 + } 454 + event => { 455 + self.text.handle_event(event); 456 + false 457 + } 458 + } 459 + } 460 + } 461 + 462 + struct CodeBlockBuilder { 463 + info: String, 464 + body: TextCollector, 465 + } 466 + 467 + impl CodeBlockBuilder { 468 + fn new(kind: CodeBlockKind<'_>) -> Self { 469 + Self { 470 + info: code_info(&kind), 471 + body: TextCollector::default(), 472 + } 473 + } 474 + 475 + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec<MarkdownBlock>) -> bool { 476 + match event { 477 + Event::End(TagEnd::CodeBlock) => { 478 + blocks.push(MarkdownBlock::Code(CodeBlock { 479 + info: self.info.trim().to_string(), 480 + body: std::mem::take(&mut self.body).finish(), 481 + })); 482 + true 483 + } 484 + Event::Text(text) => { 485 + self.body.push_text(&text); 486 + false 487 + } 488 + Event::SoftBreak | Event::HardBreak => { 489 + self.body.push_text("\n"); 490 + false 491 + } 492 + _ => false, 493 + } 494 + } 495 + } 496 + 497 + struct ListBuilder { 498 + depth: usize, 499 + items: Vec<ListItem>, 500 + current_item: Option<ListItemBuilder>, 501 + } 502 + 503 + impl ListBuilder { 504 + fn new() -> Self { 505 + Self { 506 + depth: 1, 507 + items: Vec::new(), 508 + current_item: None, 509 + } 510 + } 511 + 512 + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec<MarkdownBlock>) -> bool { 513 + match event { 514 + Event::Start(Tag::List(_)) => { 515 + self.depth += 1; 516 + if let Some(item) = &mut self.current_item { 517 + item.mark_complex(); 518 + } 519 + false 520 + } 521 + Event::End(TagEnd::List(_)) => { 522 + self.depth -= 1; 523 + if self.depth == 0 { 524 + blocks.push(MarkdownBlock::List(ListBlock { 525 + items: std::mem::take(&mut self.items), 526 + })); 527 + true 528 + } else { 529 + false 530 + } 531 + } 532 + Event::Start(Tag::Item) if self.depth == 1 => { 533 + self.current_item = Some(ListItemBuilder::default()); 534 + false 535 + } 536 + Event::End(TagEnd::Item) if self.depth == 1 => { 537 + if let Some(item) = self.current_item.take() { 538 + self.items.push(item.finish()); 539 + } 540 + false 541 + } 542 + Event::Start(Tag::Item) => { 543 + if let Some(item) = &mut self.current_item { 544 + item.mark_complex(); 545 + item.push_separator(); 546 + } 547 + false 548 + } 549 + event => { 550 + if let Some(item) = &mut self.current_item { 551 + item.handle_event(event); 552 + } 553 + false 554 + } 555 + } 556 + } 557 + } 558 + 559 + #[derive(Default)] 560 + struct ListItemBuilder { 561 + text: TextCollector, 562 + block_count: usize, 563 + in_first_paragraph: bool, 564 + leading_strong: bool, 565 + in_leading_strong: bool, 566 + saw_text_before_strong: bool, 567 + strong_text: String, 568 + following_text: String, 569 + complex: bool, 570 + } 571 + 572 + impl ListItemBuilder { 573 + fn handle_event(&mut self, event: Event<'_>) { 574 + match event { 575 + Event::Start(Tag::Paragraph) => { 576 + self.block_count += 1; 577 + self.in_first_paragraph = self.block_count == 1; 578 + } 579 + Event::End(TagEnd::Paragraph) => { 580 + self.in_first_paragraph = false; 581 + self.push_separator(); 582 + } 583 + Event::Start(Tag::CodeBlock(kind)) => { 584 + self.block_count += 1; 585 + self.mark_complex(); 586 + self.push_text(&code_info(&kind)); 587 + } 588 + Event::End(TagEnd::CodeBlock) => { 589 + self.push_separator(); 590 + } 591 + Event::Start(Tag::Strong) => { 592 + self.ensure_text_block(); 593 + if self.in_first_paragraph && !self.saw_text_before_strong && !self.leading_strong { 594 + self.leading_strong = true; 595 + self.in_leading_strong = true; 596 + } 597 + } 598 + Event::End(TagEnd::Strong) => { 599 + self.in_leading_strong = false; 600 + } 601 + Event::Text(text) | Event::Code(text) => { 602 + self.ensure_text_block(); 603 + self.push_text(&text); 604 + } 605 + Event::InlineHtml(html) => { 606 + self.ensure_text_block(); 607 + self.push_text(&html); 608 + } 609 + Event::SoftBreak | Event::HardBreak => { 610 + self.ensure_text_block(); 611 + self.push_text("\n"); 612 + } 613 + event => { 614 + self.text.handle_event(event); 615 + } 616 + } 617 + } 618 + 619 + fn push_text(&mut self, text: &str) { 620 + if self.in_first_paragraph { 621 + if self.in_leading_strong { 622 + self.strong_text.push_str(text); 623 + } else if self.leading_strong { 624 + self.following_text.push_str(text); 625 + } else if !text.trim().is_empty() { 626 + self.saw_text_before_strong = true; 627 + } 628 + } 629 + self.text.push_text(text); 630 + } 631 + 632 + fn ensure_text_block(&mut self) { 633 + if self.block_count == 0 { 634 + self.block_count = 1; 635 + self.in_first_paragraph = true; 636 + } 637 + } 638 + 639 + fn push_separator(&mut self) { 640 + self.text.push_text("\n"); 641 + } 642 + 643 + fn mark_complex(&mut self) { 644 + self.complex = true; 645 + } 646 + 647 + fn finish(self) -> ListItem { 648 + let text = self.text.finish(); 649 + let strong_has_colon = self.strong_text.trim_end().ends_with(':') 650 + || self.following_text.trim_start().starts_with(':'); 651 + let is_definition_item = !self.complex 652 + && self.block_count == 1 653 + && self.leading_strong 654 + && strong_has_colon 655 + && !text.trim().ends_with('.'); 656 + ListItem { 657 + text, 658 + is_definition_item, 659 + } 660 + } 661 + } 662 + 663 + #[derive(Default)] 664 + struct TableBuilder { 665 + headers: Vec<String>, 666 + rows: Vec<Vec<String>>, 667 + in_head: bool, 668 + current_row: Option<Vec<String>>, 669 + current_cell: Option<TextCollector>, 670 + } 671 + 672 + impl TableBuilder { 673 + fn handle(&mut self, event: Event<'_>, blocks: &mut Vec<MarkdownBlock>) -> bool { 674 + match event { 675 + Event::Start(Tag::TableHead) => { 676 + self.in_head = true; 677 + false 678 + } 679 + Event::End(TagEnd::TableHead) => { 680 + self.in_head = false; 681 + false 682 + } 683 + Event::Start(Tag::TableRow) => { 684 + self.current_row = Some(Vec::new()); 685 + false 686 + } 687 + Event::End(TagEnd::TableRow) => { 688 + if let Some(row) = self.current_row.take() { 689 + if self.in_head { 690 + self.headers = row; 691 + } else { 692 + self.rows.push(row); 693 + } 694 + } 695 + false 696 + } 697 + Event::Start(Tag::TableCell) => { 698 + self.current_cell = Some(TextCollector::default()); 699 + false 700 + } 701 + Event::End(TagEnd::TableCell) => { 702 + if let Some(cell) = self.current_cell.take() { 703 + let text = cell.finish(); 704 + if let Some(row) = &mut self.current_row { 705 + row.push(text); 706 + } else if self.in_head { 707 + self.headers.push(text); 708 + } 709 + } 710 + false 711 + } 712 + Event::End(TagEnd::Table) => { 713 + blocks.push(MarkdownBlock::Table(TableBlock { 714 + headers: std::mem::take(&mut self.headers), 715 + rows: std::mem::take(&mut self.rows), 716 + })); 717 + true 718 + } 719 + event => { 720 + if let Some(cell) = &mut self.current_cell { 721 + cell.handle_event(event); 722 + } 723 + false 724 + } 725 + } 726 + } 727 + } 728 + 729 + #[derive(Default)] 730 + struct TextCollector { 731 + text: String, 732 + links: Vec<LinkContext>, 733 + } 734 + 735 + impl TextCollector { 736 + fn handle_event(&mut self, event: Event<'_>) { 737 + match event { 738 + Event::Text(text) | Event::Code(text) => self.push_text(&text), 739 + Event::SoftBreak | Event::HardBreak => self.push_text("\n"), 740 + Event::InlineHtml(html) => self.push_text(&html), 741 + Event::Start(Tag::Link { 742 + link_type, 743 + dest_url, 744 + title, 745 + id, 746 + }) => self 747 + .links 748 + .push(LinkContext::new(link_type, dest_url, title, id)), 749 + Event::Start(Tag::Image { 750 + link_type, 751 + dest_url, 752 + title, 753 + id, 754 + }) => self 755 + .links 756 + .push(LinkContext::new(link_type, dest_url, title, id)), 757 + Event::End(TagEnd::Link | TagEnd::Image) => { 758 + if let Some(link) = self.links.pop() { 759 + for extra in link.extra_text() { 760 + self.push_text(&extra); 761 + } 762 + } 763 + } 764 + _ => {} 765 + } 766 + } 767 + 768 + fn push_text(&mut self, text: &str) { 769 + if text.is_empty() { 770 + return; 771 + } 772 + if !self.text.is_empty() 773 + && !self.text.ends_with(char::is_whitespace) 774 + && !text.starts_with(char::is_whitespace) 775 + { 776 + self.text.push(' '); 777 + } 778 + self.text.push_str(text); 779 + } 780 + 781 + fn finish(self) -> String { 782 + self.text.trim().to_string() 783 + } 784 + } 785 + 786 + struct LinkContext { 787 + link_type: LinkType, 788 + dest_url: String, 789 + title: String, 790 + id: String, 791 + } 792 + 793 + impl LinkContext { 794 + fn new( 795 + link_type: LinkType, 796 + dest_url: impl ToString, 797 + title: impl ToString, 798 + id: impl ToString, 799 + ) -> Self { 800 + Self { 801 + link_type, 802 + dest_url: dest_url.to_string(), 803 + title: title.to_string(), 804 + id: id.to_string(), 805 + } 806 + } 807 + 808 + fn extra_text(&self) -> Vec<String> { 809 + match self.link_type { 810 + LinkType::Inline => [self.dest_url.trim(), self.title.trim()] 811 + .into_iter() 812 + .filter(|value| !value.is_empty()) 813 + .map(str::to_string) 814 + .collect(), 815 + LinkType::Reference => { 816 + if self.id.trim().is_empty() { 817 + Vec::new() 818 + } else { 819 + vec![self.id.trim().to_string()] 820 + } 821 + } 822 + LinkType::Autolink | LinkType::Email => Vec::new(), 823 + _ => Vec::new(), 824 + } 825 + } 826 + } 827 + 828 + fn code_info(kind: &CodeBlockKind<'_>) -> String { 829 + match kind { 830 + CodeBlockKind::Fenced(info) => info.trim().to_string(), 831 + CodeBlockKind::Indented => String::new(), 832 + } 833 + } 834 + 117 835 #[cfg(test)] 118 836 mod tests { 837 + use serde_json::Value; 838 + 119 839 use super::*; 120 840 841 + const MARKDOWN_FIXTURE: &str = include_str!(concat!( 842 + env!("CARGO_MANIFEST_DIR"), 843 + "/../../fixtures/markdown_chunks.json" 844 + )); 845 + const OVERSIZED_SIZE_NORMALIZATION: &str = "oversized_size"; 846 + const OVERSIZED_SIZE_TOKEN: &str = "normalizedsize"; 847 + 121 848 #[test] 122 - fn chunks_preserve_header_context_and_non_empty_content() { 123 - let chunks = chunk_markdown("# Title\n\nIntro\n\n## Section\n\nBody"); 124 - assert_eq!(chunks.len(), 2); 125 - assert_eq!(chunks[0].markdown, "# Title\n\nIntro"); 126 - assert_eq!(chunks[1].markdown, "# Title\n\n## Section\n\nBody"); 127 - assert!(chunks.iter().all(|chunk| !chunk.markdown.trim().is_empty())); 849 + fn markdown_chunks_match_python_oracle_tokens() { 850 + let fixture: Value = 851 + serde_json::from_str(MARKDOWN_FIXTURE).expect("parse markdown chunks fixture"); 852 + for case in fixture["cases"].as_array().expect("fixture cases") { 853 + let id = case["id"].as_str().expect("case id"); 854 + let input = case["input"].as_str().expect("case input"); 855 + let formatted = format_markdown(input); 856 + let expected_warnings = strings(&case["warnings"]); 857 + assert_eq!(formatted.warnings, expected_warnings, "{id} warnings"); 858 + assert_eq!( 859 + formatted.chunks.len(), 860 + case["chunk_count"].as_u64().expect("chunk count") as usize, 861 + "{id} chunk count" 862 + ); 863 + 864 + for (idx, expected_chunk) in case["chunks"] 865 + .as_array() 866 + .expect("case chunks") 867 + .iter() 868 + .enumerate() 869 + { 870 + let normalizations = strings(&expected_chunk["normalizations"]); 871 + let recorded_tokens = strings(&expected_chunk["tokens"]); 872 + let python_markdown = expected_chunk["markdown"] 873 + .as_str() 874 + .expect("python rendered markdown"); 875 + assert_eq!( 876 + normalize_tokens(rust_tokenize(python_markdown), &normalizations), 877 + recorded_tokens, 878 + "{id}:{idx} fixture tokenizer" 879 + ); 880 + assert_eq!( 881 + normalize_tokens( 882 + rust_tokenize(&formatted.chunks[idx].markdown), 883 + &normalizations 884 + ), 885 + recorded_tokens, 886 + "{id}:{idx} native tokens; native chunk {:?}", 887 + formatted.chunks[idx].markdown 888 + ); 889 + } 890 + } 891 + } 892 + 893 + #[test] 894 + fn empty_context_only_inputs_produce_no_chunks() { 895 + for input in ["", " \n", "# Heading\n", "---\n", "| A |\n| --- |\n"] { 896 + assert!(format_markdown(input).chunks.is_empty(), "{input:?}"); 897 + } 898 + } 899 + 900 + #[test] 901 + fn overlong_lines_are_dropped_with_warning() { 902 + let input = format!("# Long\n\n{}\n\nkept alpha", "z".repeat(MAX_LINE_CHARS + 1)); 903 + let formatted = format_markdown(&input); 904 + assert_eq!( 905 + formatted.warnings, 906 + vec!["Dropped 1 line(s) exceeding 2048 chars during markdown sanitization"] 907 + ); 908 + assert_eq!(formatted.chunks.len(), 1); 909 + assert!(formatted.chunks[0].markdown.contains("kept alpha")); 910 + assert!(!formatted.chunks[0].markdown.contains('z')); 911 + } 912 + 913 + #[test] 914 + fn oversized_chunks_become_header_stub() { 915 + let oversized_line = "alpha ".repeat(300); 916 + let input = format!( 917 + "# Big\n\n{}\n{}\n{}", 918 + oversized_line, oversized_line, oversized_line 919 + ); 920 + let formatted = format_markdown(&input); 921 + assert_eq!(formatted.chunks.len(), 1); 922 + assert!(formatted.chunks[0].markdown.starts_with("# Big\n\n")); 923 + assert!( 924 + formatted.chunks[0] 925 + .markdown 926 + .contains("[Content too large to index:") 927 + ); 928 + assert!(!formatted.chunks[0].markdown.contains("alpha alpha")); 929 + } 930 + 931 + fn strings(value: &Value) -> Vec<String> { 932 + value 933 + .as_array() 934 + .map(|items| { 935 + items 936 + .iter() 937 + .map(|item| item.as_str().expect("string item").to_string()) 938 + .collect() 939 + }) 940 + .unwrap_or_default() 941 + } 942 + 943 + fn rust_tokenize(text: &str) -> Vec<String> { 944 + text.split(|ch: char| !ch.is_ascii_alphanumeric()) 945 + .filter(|token| !token.is_empty()) 946 + .map(|token| token.to_ascii_lowercase()) 947 + .collect() 948 + } 949 + 950 + fn normalize_tokens(tokens: Vec<String>, normalizations: &[String]) -> Vec<String> { 951 + if normalizations 952 + .iter() 953 + .any(|normalization| normalization == OVERSIZED_SIZE_NORMALIZATION) 954 + { 955 + normalize_oversized_size_tokens(tokens) 956 + } else { 957 + tokens 958 + } 959 + } 960 + 961 + fn normalize_oversized_size_tokens(tokens: Vec<String>) -> Vec<String> { 962 + let mut normalized = Vec::new(); 963 + let mut i = 0; 964 + while i < tokens.len() { 965 + if i + 5 < tokens.len() 966 + && tokens[i..i + 5] == ["content", "too", "large", "to", "index"] 967 + { 968 + normalized.extend_from_slice(&tokens[i..i + 5]); 969 + let mut j = i + 5; 970 + while j < tokens.len() && tokens[j] != "chars" { 971 + j += 1; 972 + } 973 + if j < tokens.len() { 974 + normalized.push(OVERSIZED_SIZE_TOKEN.to_string()); 975 + normalized.push("chars".to_string()); 976 + i = j + 1; 977 + continue; 978 + } 979 + } 980 + normalized.push(tokens[i].clone()); 981 + i += 1; 982 + } 983 + normalized 128 984 } 129 985 }
+2
core/crates/solstone-core-indexer/src/content/ai_chat.rs
··· 10 10 return ProducedChunks { 11 11 chunks: Vec::new(), 12 12 agent_override: None, 13 + warnings: Vec::new(), 13 14 }; 14 15 } 15 16 ··· 37 38 ProducedChunks { 38 39 chunks, 39 40 agent_override: Some(format!("import.{source_key}")), 41 + warnings: Vec::new(), 40 42 } 41 43 } 42 44
+1
core/crates/solstone-core-indexer/src/content/browser.rs
··· 23 23 ProducedChunks { 24 24 chunks, 25 25 agent_override: Some("browser".to_string()), 26 + warnings: Vec::new(), 26 27 } 27 28 } 28 29
+1
core/crates/solstone-core-indexer/src/content/chat.rs
··· 55 55 ProducedChunks { 56 56 chunks, 57 57 agent_override: Some("chat".to_string()), 58 + warnings: Vec::new(), 58 59 } 59 60 } 60 61
+1
core/crates/solstone-core-indexer/src/content/day_accumulator.rs
··· 16 16 ProducedChunks { 17 17 chunks, 18 18 agent_override: Some(file_stem(rel).to_lowercase()), 19 + warnings: Vec::new(), 19 20 } 20 21 } 21 22
+1
core/crates/solstone-core-indexer/src/content/documents.rs
··· 19 19 ProducedChunks { 20 20 chunks, 21 21 agent_override: Some("documents".to_string()), 22 + warnings: Vec::new(), 22 23 } 23 24 } 24 25
+1
core/crates/solstone-core-indexer/src/content/facet_entities.rs
··· 23 23 ProducedChunks { 24 24 chunks, 25 25 agent_override: Some(agent_for_rel(rel).to_string()), 26 + warnings: Vec::new(), 26 27 } 27 28 } 28 29
+2
core/crates/solstone-core-indexer/src/content/imports.rs
··· 13 13 return ProducedChunks { 14 14 chunks: Vec::new(), 15 15 agent_override: None, 16 + warnings: Vec::new(), 16 17 }; 17 18 }; 18 19 let source = header ··· 34 35 ProducedChunks { 35 36 chunks, 36 37 agent_override: Some(format!("import.{source}")), 38 + warnings: Vec::new(), 37 39 } 38 40 } 39 41
+153 -24
core/crates/solstone-core-indexer/src/content/mod.rs
··· 21 21 use glob::{MatchOptions, Pattern}; 22 22 use serde_json::{Map, Value}; 23 23 24 - use crate::chunker::chunk_markdown; 24 + use crate::chunker::format_markdown; 25 25 26 26 #[derive(Debug, Clone, Copy, PartialEq, Eq)] 27 27 pub enum Family { ··· 51 51 pub struct ProducedChunks { 52 52 pub chunks: Vec<IndexChunk>, 53 53 pub agent_override: Option<String>, 54 + pub warnings: Vec<String>, 54 55 } 55 56 56 57 #[derive(Debug, Clone, Copy, PartialEq, Eq)] ··· 243 244 244 245 pub fn produce_chunks(family: Family, rel: &str, text: &str) -> ProducedChunks { 245 246 match family { 246 - Family::Markdown => ProducedChunks { 247 - chunks: chunk_markdown(text) 248 - .into_iter() 249 - .map(|chunk| IndexChunk { 250 - content: chunk.markdown, 251 - }) 252 - .collect(), 253 - agent_override: None, 254 - }, 247 + Family::Markdown => { 248 + let formatted = format_markdown(text); 249 + ProducedChunks { 250 + chunks: formatted 251 + .chunks 252 + .into_iter() 253 + .map(|chunk| IndexChunk { 254 + content: chunk.markdown, 255 + }) 256 + .collect(), 257 + agent_override: None, 258 + warnings: formatted.warnings, 259 + } 260 + } 255 261 Family::Event => ProducedChunks { 256 262 chunks: events::render(&parse_jsonl_objects(text)), 257 263 agent_override: Some("event".to_string()), 264 + warnings: Vec::new(), 258 265 }, 259 266 Family::Activity => ProducedChunks { 260 267 chunks: activities::render(&parse_jsonl_objects(text)), 261 268 agent_override: Some("activity".to_string()), 269 + warnings: Vec::new(), 262 270 }, 263 271 Family::ActionLog => ProducedChunks { 264 272 chunks: action_logs::render(&parse_jsonl_objects(text)), 265 273 agent_override: Some("action".to_string()), 274 + warnings: Vec::new(), 266 275 }, 267 276 Family::StructuredImport => imports::render(&parse_jsonl_objects(text)), 268 277 Family::AiChat => ai_chat::render(rel, &parse_jsonl_objects(text)), ··· 273 282 Family::Observation => ProducedChunks { 274 283 chunks: observations::render(&parse_jsonl_objects(text)), 275 284 agent_override: Some("observation".to_string()), 285 + warnings: Vec::new(), 276 286 }, 277 287 Family::Documents => documents::render(&parse_json_object(text)), 278 288 Family::Screen => screen::render(&parse_json_object(text)), ··· 405 415 #[cfg(test)] 406 416 mod tests { 407 417 use super::*; 408 - use crate::chunker::chunk_markdown; 409 418 410 419 #[test] 411 420 fn classifies_indexable_families() { ··· 545 554 } 546 555 547 556 #[test] 548 - fn markdown_producer_wraps_chunker_without_content_changes() { 549 - let text = "# Title\n\nIntro\n\n## Section\n\nBody"; 550 - let expected: Vec<String> = chunk_markdown(text) 551 - .into_iter() 552 - .map(|chunk| chunk.markdown) 553 - .collect(); 554 - let produced = produce_chunks(Family::Markdown, "20240101/talents/flow.md", text); 555 - let got: Vec<String> = produced 556 - .chunks 557 - .into_iter() 558 - .map(|chunk| chunk.content) 559 - .collect(); 560 - assert_eq!(got, expected); 557 + fn markdown_producer_ports_grouping_rules() { 558 + let produced = produce_chunks( 559 + Family::Markdown, 560 + "20240101/talents/flow.md", 561 + "# Tasks\n\nintro alpha\n\n- item one\n- item two\n", 562 + ); 563 + assert_eq!(produced.chunks.len(), 2); 564 + assert!( 565 + produced 566 + .chunks 567 + .iter() 568 + .all(|chunk| tokenizes_to(&chunk.content, &["tasks", "intro", "alpha", "item"])) 569 + ); 570 + assert!( 571 + !produced 572 + .chunks 573 + .iter() 574 + .any(|chunk| chunk.content.trim() == "# Tasks\n\nintro alpha") 575 + ); 576 + 577 + let produced = produce_chunks( 578 + Family::Markdown, 579 + "20240101/talents/flow.md", 580 + "# Tasks\n\n- item one\n- item two\n", 581 + ); 582 + assert_eq!(produced.chunks.len(), 2); 583 + 584 + let produced = produce_chunks( 585 + Family::Markdown, 586 + "20240101/talents/flow.md", 587 + "# Definitions\n\n- **alpha:** value one\n- ordinary note.\n- **beta:** value two\n- ordinary other.\n", 588 + ); 589 + assert_eq!(produced.chunks.len(), 1); 590 + assert_eq!( 591 + tokens(&produced.chunks[0].content), 592 + [ 593 + "definitions", 594 + "alpha", 595 + "value", 596 + "one", 597 + "ordinary", 598 + "note", 599 + "beta", 600 + "value", 601 + "two", 602 + "ordinary", 603 + "other" 604 + ] 605 + ); 561 606 assert_eq!(produced.agent_override, None); 607 + assert!(produced.warnings.is_empty()); 608 + } 609 + 610 + #[test] 611 + fn markdown_producer_ports_table_and_heading_rules() { 612 + let produced = produce_chunks( 613 + Family::Markdown, 614 + "20240101/talents/flow.md", 615 + "# Root\n\n## Matrix\n\nintro alpha\n\n| Name | Value |\n| --- | --- |\n| beta | one |\n| gamma | two |\n", 616 + ); 617 + assert_eq!(produced.chunks.len(), 2); 618 + assert_eq!( 619 + tokens(&produced.chunks[0].content), 620 + [ 621 + "root", "matrix", "intro", "alpha", "name", "value", "beta", "one" 622 + ] 623 + ); 624 + assert_eq!( 625 + tokens(&produced.chunks[1].content), 626 + [ 627 + "root", "matrix", "intro", "alpha", "name", "value", "gamma", "two" 628 + ] 629 + ); 630 + 631 + let produced = produce_chunks( 632 + Family::Markdown, 633 + "20240101/talents/flow.md", 634 + "# Root\n\n## Empty\n\n| Name | Value |\n| --- | --- |\n", 635 + ); 636 + assert!(produced.chunks.is_empty()); 637 + } 638 + 639 + #[test] 640 + fn markdown_producer_drops_overlong_lines_and_stubs_oversized_chunks() { 641 + let produced = produce_chunks( 642 + Family::Markdown, 643 + "20240101/talents/flow.md", 644 + &format!("# Long\n\n{}\n\nkept alpha\n", "z".repeat(2049)), 645 + ); 646 + assert_eq!(produced.chunks.len(), 1); 647 + assert_eq!( 648 + produced.warnings, 649 + vec!["Dropped 1 line(s) exceeding 2048 chars during markdown sanitization"] 650 + ); 651 + assert_eq!( 652 + tokens(&produced.chunks[0].content), 653 + ["long", "kept", "alpha"] 654 + ); 655 + 656 + let oversized_line = "alpha ".repeat(300); 657 + let produced = produce_chunks( 658 + Family::Markdown, 659 + "20240101/talents/flow.md", 660 + &format!( 661 + "# Big\n\n{}\n{}\n{}", 662 + oversized_line, oversized_line, oversized_line 663 + ), 664 + ); 665 + assert_eq!(produced.chunks.len(), 1); 666 + assert!( 667 + produced.chunks[0] 668 + .content 669 + .contains("[Content too large to index:") 670 + ); 671 + assert_eq!( 672 + &tokens(&produced.chunks[0].content)[..6], 673 + ["big", "content", "too", "large", "to", "index"] 674 + ); 675 + } 676 + 677 + fn tokens(text: &str) -> Vec<String> { 678 + text.split(|ch: char| !ch.is_ascii_alphanumeric()) 679 + .filter(|token| !token.is_empty()) 680 + .map(|token| token.to_ascii_lowercase()) 681 + .collect() 682 + } 683 + 684 + fn tokenizes_to(text: &str, expected_prefix: &[&str]) -> bool { 685 + tokens(text).starts_with( 686 + &expected_prefix 687 + .iter() 688 + .map(|token| token.to_string()) 689 + .collect::<Vec<_>>(), 690 + ) 562 691 } 563 692 564 693 #[test]
+1
core/crates/solstone-core-indexer/src/content/morning_briefing.rs
··· 26 26 ProducedChunks { 27 27 chunks, 28 28 agent_override: Some("morning_briefing".to_string()), 29 + warnings: Vec::new(), 29 30 } 30 31 } 31 32
+1
core/crates/solstone-core-indexer/src/content/screen.rs
··· 17 17 ProducedChunks { 18 18 chunks, 19 19 agent_override: Some("screen".to_string()), 20 + warnings: Vec::new(), 20 21 } 21 22 } 22 23
+1
core/crates/solstone-core-indexer/src/content/sense.rs
··· 17 17 ProducedChunks { 18 18 chunks, 19 19 agent_override: Some("sense".to_string()), 20 + warnings: Vec::new(), 20 21 } 21 22 } 22 23
+83 -2
core/crates/solstone-core-indexer/src/segment_aggregate.rs
··· 6 6 7 7 use glob::{Pattern, glob}; 8 8 9 - use crate::chunker::chunk_markdown; 9 + use crate::chunker::format_markdown; 10 10 use crate::paths::resolve_journal_path; 11 11 use crate::segment::time_bucket; 12 12 use crate::stream::extract_stream; ··· 87 87 .unwrap_or_default() 88 88 .to_string(); 89 89 let bucket = time_bucket(rel_segment); 90 + let formatted = format_markdown(&content); 91 + warnings.extend(formatted.warnings); 92 + 90 93 let mut rows = Vec::new(); 91 - for chunk in chunk_markdown(&content) { 94 + for chunk in formatted.chunks { 92 95 let content = chunk.markdown.trim(); 93 96 if content.is_empty() { 94 97 continue; ··· 143 146 } 144 147 } 145 148 } 149 + 150 + #[cfg(test)] 151 + mod tests { 152 + use super::*; 153 + use std::time::{SystemTime, UNIX_EPOCH}; 154 + 155 + fn temp_root(name: &str) -> PathBuf { 156 + let stamp = SystemTime::now() 157 + .duration_since(UNIX_EPOCH) 158 + .expect("time should be available") 159 + .as_nanos(); 160 + std::env::temp_dir().join(format!("solstone-core-indexer-aggregate-{name}-{stamp}")) 161 + } 162 + 163 + fn write(root: &Path, rel: &str, text: &str) { 164 + let path = root.join(rel); 165 + fs::create_dir_all(path.parent().expect("test path should have parent")) 166 + .expect("create parent"); 167 + fs::write(path, text).expect("write test file"); 168 + } 169 + 170 + #[test] 171 + fn aggregate_uses_markdown_formatter_cardinality_and_tokens() { 172 + let root = temp_root("markdown-cardinality"); 173 + write( 174 + &root, 175 + "chronicle/20240102/default/090000_300/talents/audio.md", 176 + "# Audio\n\nintro alpha\n\n- item one\n- item two\n", 177 + ); 178 + 179 + let aggregate = build_segment_aggregate(&root, "20240102/default/090000_300"); 180 + 181 + assert!(aggregate.complete); 182 + assert!(aggregate.warnings.is_empty()); 183 + assert_eq!(aggregate.rows.len(), 2); 184 + assert_eq!( 185 + tokens(&aggregate.rows[0].content), 186 + ["audio", "intro", "alpha", "item", "one"] 187 + ); 188 + assert_eq!( 189 + tokens(&aggregate.rows[1].content), 190 + ["audio", "intro", "alpha", "item", "two"] 191 + ); 192 + 193 + fs::remove_dir_all(root).expect("cleanup aggregate root"); 194 + } 195 + 196 + #[test] 197 + fn aggregate_retains_markdown_formatter_warnings() { 198 + let root = temp_root("markdown-warning"); 199 + write( 200 + &root, 201 + "chronicle/20240102/default/090000_300/talents/audio.md", 202 + &format!("# Audio\n\n{}\n\nkept alpha\n", "z".repeat(2049)), 203 + ); 204 + 205 + let aggregate = build_segment_aggregate(&root, "20240102/default/090000_300"); 206 + 207 + assert_eq!( 208 + aggregate.warnings, 209 + vec!["Dropped 1 line(s) exceeding 2048 chars during markdown sanitization"] 210 + ); 211 + assert_eq!(aggregate.rows.len(), 1); 212 + assert_eq!( 213 + tokens(&aggregate.rows[0].content), 214 + ["audio", "kept", "alpha"] 215 + ); 216 + 217 + fs::remove_dir_all(root).expect("cleanup aggregate root"); 218 + } 219 + 220 + fn tokens(text: &str) -> Vec<String> { 221 + text.split(|ch: char| !ch.is_ascii_alphanumeric()) 222 + .filter(|token| !token.is_empty()) 223 + .map(|token| token.to_ascii_lowercase()) 224 + .collect() 225 + } 226 + }
+556
core/fixtures/markdown_chunks.json
··· 1 + { 2 + "cases": [ 3 + { 4 + "chunk_count": 0, 5 + "chunks": [], 6 + "id": "empty", 7 + "input": "", 8 + "warnings": [] 9 + }, 10 + { 11 + "chunk_count": 0, 12 + "chunks": [], 13 + "id": "whitespace_only", 14 + "input": " \n\t\n", 15 + "warnings": [] 16 + }, 17 + { 18 + "chunk_count": 0, 19 + "chunks": [], 20 + "id": "heading_only", 21 + "input": "# Heading\n", 22 + "warnings": [] 23 + }, 24 + { 25 + "chunk_count": 0, 26 + "chunks": [], 27 + "id": "thematic_break_only", 28 + "input": "---\n", 29 + "warnings": [] 30 + }, 31 + { 32 + "chunk_count": 0, 33 + "chunks": [], 34 + "id": "header_only_table", 35 + "input": "| Name | Value |\n| --- | --- |\n", 36 + "warnings": [] 37 + }, 38 + { 39 + "chunk_count": 2, 40 + "chunks": [ 41 + { 42 + "markdown": "# Root\n\n## Child\n\nalpha paragraph\n", 43 + "tokens": [ 44 + "root", 45 + "child", 46 + "alpha", 47 + "paragraph" 48 + ] 49 + }, 50 + { 51 + "markdown": "# Root\n\n## Child\n\n### Leaf\n\nbeta paragraph\n", 52 + "tokens": [ 53 + "root", 54 + "child", 55 + "leaf", 56 + "beta", 57 + "paragraph" 58 + ] 59 + } 60 + ], 61 + "id": "nested_heading_context", 62 + "input": "# Root\n\n## Child\n\nalpha paragraph\n\n### Leaf\n\nbeta paragraph\n", 63 + "warnings": [] 64 + }, 65 + { 66 + "chunk_count": 2, 67 + "chunks": [ 68 + { 69 + "markdown": "# Notes\n\nalpha paragraph\n", 70 + "tokens": [ 71 + "notes", 72 + "alpha", 73 + "paragraph" 74 + ] 75 + }, 76 + { 77 + "markdown": "# Notes\n\nbeta paragraph\n", 78 + "tokens": [ 79 + "notes", 80 + "beta", 81 + "paragraph" 82 + ] 83 + } 84 + ], 85 + "id": "ordinary_paragraphs", 86 + "input": "# Notes\n\nalpha paragraph\n\nbeta paragraph\n", 87 + "warnings": [] 88 + }, 89 + { 90 + "chunk_count": 2, 91 + "chunks": [ 92 + { 93 + "markdown": "# Tasks\n\n- alpha item\n", 94 + "tokens": [ 95 + "tasks", 96 + "alpha", 97 + "item" 98 + ] 99 + }, 100 + { 101 + "markdown": "# Tasks\n\n- beta item\n", 102 + "tokens": [ 103 + "tasks", 104 + "beta", 105 + "item" 106 + ] 107 + } 108 + ], 109 + "id": "ordinary_list", 110 + "input": "# Tasks\n\n- alpha item\n- beta item\n", 111 + "warnings": [] 112 + }, 113 + { 114 + "chunk_count": 2, 115 + "chunks": [ 116 + { 117 + "markdown": "# Tasks\n\nintro alpha\n\n- alpha item\n", 118 + "tokens": [ 119 + "tasks", 120 + "intro", 121 + "alpha", 122 + "alpha", 123 + "item" 124 + ] 125 + }, 126 + { 127 + "markdown": "# Tasks\n\nintro alpha\n\n- beta item\n", 128 + "tokens": [ 129 + "tasks", 130 + "intro", 131 + "alpha", 132 + "beta", 133 + "item" 134 + ] 135 + } 136 + ], 137 + "id": "intro_list", 138 + "input": "# Tasks\n\nintro alpha\n\n- alpha item\n- beta item\n", 139 + "warnings": [] 140 + }, 141 + { 142 + "chunk_count": 2, 143 + "chunks": [ 144 + { 145 + "markdown": "# Metrics\n\nintro alpha\n\n| Name | Value |\n| --- | --- |\n| alpha | one |\n", 146 + "tokens": [ 147 + "metrics", 148 + "intro", 149 + "alpha", 150 + "name", 151 + "value", 152 + "alpha", 153 + "one" 154 + ] 155 + }, 156 + { 157 + "markdown": "# Metrics\n\nintro alpha\n\n| Name | Value |\n| --- | --- |\n| beta | two |\n", 158 + "tokens": [ 159 + "metrics", 160 + "intro", 161 + "alpha", 162 + "name", 163 + "value", 164 + "beta", 165 + "two" 166 + ] 167 + } 168 + ], 169 + "id": "intro_table", 170 + "input": "# Metrics\n\nintro alpha\n\n| Name | Value |\n| --- | --- |\n| alpha | one |\n| beta | two |\n", 171 + "warnings": [] 172 + }, 173 + { 174 + "chunk_count": 1, 175 + "chunks": [ 176 + { 177 + "markdown": "# Definitions\n\n- **alpha:** value one\n- ordinary note.\n- **beta:** value two\n- ordinary other.\n", 178 + "tokens": [ 179 + "definitions", 180 + "alpha", 181 + "value", 182 + "one", 183 + "ordinary", 184 + "note", 185 + "beta", 186 + "value", 187 + "two", 188 + "ordinary", 189 + "other" 190 + ] 191 + } 192 + ], 193 + "id": "definition_2_of_4", 194 + "input": "# Definitions\n\n- **alpha:** value one\n- ordinary note.\n- **beta:** value two\n- ordinary other.\n", 195 + "warnings": [] 196 + }, 197 + { 198 + "chunk_count": 5, 199 + "chunks": [ 200 + { 201 + "markdown": "# Boundary\n\n- **alpha:** value one\n", 202 + "tokens": [ 203 + "boundary", 204 + "alpha", 205 + "value", 206 + "one" 207 + ] 208 + }, 209 + { 210 + "markdown": "# Boundary\n\n- ordinary note.\n", 211 + "tokens": [ 212 + "boundary", 213 + "ordinary", 214 + "note" 215 + ] 216 + }, 217 + { 218 + "markdown": "# Boundary\n\n- **beta:** value two\n", 219 + "tokens": [ 220 + "boundary", 221 + "beta", 222 + "value", 223 + "two" 224 + ] 225 + }, 226 + { 227 + "markdown": "# Boundary\n\n- ordinary other.\n", 228 + "tokens": [ 229 + "boundary", 230 + "ordinary", 231 + "other" 232 + ] 233 + }, 234 + { 235 + "markdown": "# Boundary\n\n- ordinary final.\n", 236 + "tokens": [ 237 + "boundary", 238 + "ordinary", 239 + "final" 240 + ] 241 + } 242 + ], 243 + "id": "definition_2_of_5", 244 + "input": "# Boundary\n\n- **alpha:** value one\n- ordinary note.\n- **beta:** value two\n- ordinary other.\n- ordinary final.\n", 245 + "warnings": [] 246 + }, 247 + { 248 + "chunk_count": 2, 249 + "chunks": [ 250 + { 251 + "markdown": "# Boundary\n\n- **alpha:** value one\n", 252 + "tokens": [ 253 + "boundary", 254 + "alpha", 255 + "value", 256 + "one" 257 + ] 258 + }, 259 + { 260 + "markdown": "# Boundary\n\n- ordinary note.\n", 261 + "tokens": [ 262 + "boundary", 263 + "ordinary", 264 + "note" 265 + ] 266 + } 267 + ], 268 + "id": "definition_1_of_2", 269 + "input": "# Boundary\n\n- **alpha:** value one\n- ordinary note.\n", 270 + "warnings": [] 271 + }, 272 + { 273 + "chunk_count": 3, 274 + "chunks": [ 275 + { 276 + "markdown": "# Matrix\n\n| Name | Value |\n| --- | --- |\n| alpha | one |\n", 277 + "tokens": [ 278 + "matrix", 279 + "name", 280 + "value", 281 + "alpha", 282 + "one" 283 + ] 284 + }, 285 + { 286 + "markdown": "# Matrix\n\n| Name | Value |\n| --- | --- |\n| beta | two |\n", 287 + "tokens": [ 288 + "matrix", 289 + "name", 290 + "value", 291 + "beta", 292 + "two" 293 + ] 294 + }, 295 + { 296 + "markdown": "# Matrix\n\n| Name | Value |\n| --- | --- |\n| gamma | three |\n", 297 + "tokens": [ 298 + "matrix", 299 + "name", 300 + "value", 301 + "gamma", 302 + "three" 303 + ] 304 + } 305 + ], 306 + "id": "multi_row_table", 307 + "input": "# Matrix\n\n| Name | Value |\n| --- | --- |\n| alpha | one |\n| beta | two |\n| gamma | three |\n", 308 + "warnings": [] 309 + }, 310 + { 311 + "chunk_count": 1, 312 + "chunks": [ 313 + { 314 + "markdown": "# Code\n\n```python\nprint('alpha')\n```\n", 315 + "tokens": [ 316 + "code", 317 + "python", 318 + "print", 319 + "alpha" 320 + ] 321 + } 322 + ], 323 + "id": "fenced_code_info", 324 + "input": "# Code\n\n```python\nprint('alpha')\n```\n", 325 + "warnings": [] 326 + }, 327 + { 328 + "chunk_count": 1, 329 + "chunks": [ 330 + { 331 + "markdown": "# Quote\n\n> alpha quote\n> \n> beta quote\n", 332 + "tokens": [ 333 + "quote", 334 + "alpha", 335 + "quote", 336 + "beta", 337 + "quote" 338 + ] 339 + } 340 + ], 341 + "id": "blockquote_multi_paragraph", 342 + "input": "# Quote\n\n> alpha quote\n>\n> beta quote\n", 343 + "warnings": [] 344 + }, 345 + { 346 + "chunk_count": 1, 347 + "chunks": [ 348 + { 349 + "markdown": "# Long\n\nkept alpha\n", 350 + "tokens": [ 351 + "long", 352 + "kept", 353 + "alpha" 354 + ] 355 + } 356 + ], 357 + "id": "overlong_line", 358 + "input": "# Long\n\nzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\n\nkept alpha\n", 359 + "warnings": [ 360 + "Dropped 1 line(s) exceeding 2048 chars during markdown sanitization" 361 + ] 362 + }, 363 + { 364 + "chunk_count": 1, 365 + "chunks": [ 366 + { 367 + "markdown": "# Big\n\n\n[Content too large to index: 5,407 chars]", 368 + "normalizations": [ 369 + "oversized_size" 370 + ], 371 + "tokens": [ 372 + "big", 373 + "content", 374 + "too", 375 + "large", 376 + "to", 377 + "index", 378 + "normalizedsize", 379 + "chars" 380 + ] 381 + } 382 + ], 383 + "id": "oversized_chunk", 384 + "input": "# Big\n\nalpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha \nalpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha \nalpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha \n", 385 + "warnings": [] 386 + }, 387 + { 388 + "chunk_count": 1, 389 + "chunks": [ 390 + { 391 + "markdown": "# Nested\n\n- parent alpha\n\n - child beta\n", 392 + "tokens": [ 393 + "nested", 394 + "parent", 395 + "alpha", 396 + "child", 397 + "beta" 398 + ] 399 + } 400 + ], 401 + "id": "loose_nested_list", 402 + "input": "# Nested\n\n- parent alpha\n\n - child beta\n", 403 + "warnings": [] 404 + }, 405 + { 406 + "chunk_count": 1, 407 + "chunks": [ 408 + { 409 + "markdown": "# Loose\n\n- first alpha\n\n second beta\n", 410 + "tokens": [ 411 + "loose", 412 + "first", 413 + "alpha", 414 + "second", 415 + "beta" 416 + ] 417 + } 418 + ], 419 + "id": "two_paragraph_list_item", 420 + "input": "# Loose\n\n- first alpha\n\n second beta\n", 421 + "warnings": [] 422 + }, 423 + { 424 + "chunk_count": 1, 425 + "chunks": [ 426 + { 427 + "markdown": "# Item Code\n\n- alpha before\n\n ```python\n print('beta')\n ```\n", 428 + "tokens": [ 429 + "item", 430 + "code", 431 + "alpha", 432 + "before", 433 + "python", 434 + "print", 435 + "beta" 436 + ] 437 + } 438 + ], 439 + "id": "list_item_fenced_code", 440 + "input": "# Item Code\n\n- alpha before\n\n ```python\n print('beta')\n ```\n", 441 + "warnings": [] 442 + }, 443 + { 444 + "chunk_count": 1, 445 + "chunks": [ 446 + { 447 + "markdown": "# Link\n\n[alpha](https://example.com/path/to-beta?q=gamma)\n", 448 + "tokens": [ 449 + "link", 450 + "alpha", 451 + "https", 452 + "example", 453 + "com", 454 + "path", 455 + "to", 456 + "beta", 457 + "q", 458 + "gamma" 459 + ] 460 + } 461 + ], 462 + "id": "inline_link", 463 + "input": "# Link\n\n[alpha](https://example.com/path/to-beta?q=gamma)\n", 464 + "warnings": [] 465 + }, 466 + { 467 + "chunk_count": 1, 468 + "chunks": [ 469 + { 470 + "markdown": "# Image\n\n![alt text](images/pic-alpha.png \"title beta\")\n", 471 + "tokens": [ 472 + "image", 473 + "alt", 474 + "text", 475 + "images", 476 + "pic", 477 + "alpha", 478 + "png", 479 + "title", 480 + "beta" 481 + ] 482 + } 483 + ], 484 + "id": "inline_image", 485 + "input": "# Image\n\n![alt text](images/pic-alpha.png \"title beta\")\n", 486 + "warnings": [] 487 + }, 488 + { 489 + "chunk_count": 1, 490 + "chunks": [ 491 + { 492 + "markdown": "# Auto\n\n<https://example.com/path?q=gamma>\n", 493 + "tokens": [ 494 + "auto", 495 + "https", 496 + "example", 497 + "com", 498 + "path", 499 + "q", 500 + "gamma" 501 + ] 502 + } 503 + ], 504 + "id": "autolink", 505 + "input": "# Auto\n\n<https://example.com/path?q=gamma>\n", 506 + "warnings": [] 507 + }, 508 + { 509 + "chunk_count": 1, 510 + "chunks": [ 511 + { 512 + "markdown": "# Reference\n\n[alpha][ref]\n", 513 + "tokens": [ 514 + "reference", 515 + "alpha", 516 + "ref" 517 + ] 518 + } 519 + ], 520 + "id": "reference_link", 521 + "input": "# Reference\n\n[alpha][ref]\n\n[ref]: https://example.com/path \"title beta\"\n", 522 + "warnings": [] 523 + }, 524 + { 525 + "chunk_count": 1, 526 + "chunks": [ 527 + { 528 + "markdown": "# Html\n\nalpha <span>beta</span> gamma\n", 529 + "tokens": [ 530 + "html", 531 + "alpha", 532 + "span", 533 + "beta", 534 + "span", 535 + "gamma" 536 + ] 537 + } 538 + ], 539 + "id": "inline_html", 540 + "input": "# Html\n\nalpha <span>beta</span> gamma\n", 541 + "warnings": [] 542 + } 543 + ], 544 + "constraints": { 545 + "ascii_only": true, 546 + "max_chunk_chars": 4096, 547 + "max_line_chars": 2048, 548 + "normalizations": { 549 + "oversized_size": "replace content-too-large size number tokens with normalizedsize" 550 + }, 551 + "tokenizer": "sqlite fts5(content) with fts5vocab(chunks, 'instance') ordered by doc, offset" 552 + }, 553 + "fixture": "solstone-markdown-chunks", 554 + "fixture_version": 1, 555 + "generated_by": "make core-fixtures" 556 + }
+3
docs/PORTING.md
··· 103 103 `scripts/build_core_fixtures.py` generates Rust-facing fixtures under 104 104 `core/fixtures/`. 105 105 106 + `core/fixtures/markdown_chunks.json` pins Python markdown chunking/token output 107 + for the Rust markdown indexer port. 108 + 106 109 `tests/verify_indexer_differential.py` runs the indexer differential harness and 107 110 writes its report under the harness work directory unless `--report` is supplied. 108 111
+253
scripts/build_core_fixtures.py
··· 9 9 import argparse 10 10 import hashlib 11 11 import json 12 + import logging 12 13 import sqlite3 13 14 import sys 14 15 from pathlib import Path 15 16 from typing import Any 16 17 17 18 from solstone.convey.contract.assemble import CALLOSUM_REGISTRY 19 + from solstone.think import markdown as markdown_formatter 18 20 from solstone.think.cogitate_contract import ( 19 21 COGITATE_ACCESS_TIERS, 20 22 COGITATE_READ_TOOL_NAMES, ··· 30 32 CALLOSUM_ARTIFACT_PATH = FIXTURE_DIR / "callosum_registry.json" 31 33 COGITATE_ARTIFACT_PATH = FIXTURE_DIR / "cogitate_contract.json" 32 34 EDGE_SCHEMA_ARTIFACT_PATH = FIXTURE_DIR / "edge_schema.json" 35 + MARKDOWN_CHUNKS_ARTIFACT_PATH = FIXTURE_DIR / "markdown_chunks.json" 36 + OVERSIZED_SIZE_NORMALIZATION = "oversized_size" 37 + OVERSIZED_SIZE_TOKEN = "normalizedsize" 33 38 34 39 35 40 def build_callosum_registry_fixture() -> dict[str, Any]: ··· 116 121 conn.close() 117 122 118 123 124 + def _markdown_fixture_cases() -> list[dict[str, str]]: 125 + long_line = "z" * (markdown_formatter._MAX_LINE_CHARS + 1) 126 + oversized_line = "alpha " * 300 127 + oversized_body = "\n".join([oversized_line] * 3) 128 + return [ 129 + {"id": "empty", "input": ""}, 130 + {"id": "whitespace_only", "input": " \n\t\n"}, 131 + {"id": "heading_only", "input": "# Heading\n"}, 132 + {"id": "thematic_break_only", "input": "---\n"}, 133 + { 134 + "id": "header_only_table", 135 + "input": "| Name | Value |\n| --- | --- |\n", 136 + }, 137 + { 138 + "id": "nested_heading_context", 139 + "input": "# Root\n\n## Child\n\nalpha paragraph\n\n### Leaf\n\nbeta paragraph\n", 140 + }, 141 + { 142 + "id": "ordinary_paragraphs", 143 + "input": "# Notes\n\nalpha paragraph\n\nbeta paragraph\n", 144 + }, 145 + { 146 + "id": "ordinary_list", 147 + "input": "# Tasks\n\n- alpha item\n- beta item\n", 148 + }, 149 + { 150 + "id": "intro_list", 151 + "input": "# Tasks\n\nintro alpha\n\n- alpha item\n- beta item\n", 152 + }, 153 + { 154 + "id": "intro_table", 155 + "input": ( 156 + "# Metrics\n\nintro alpha\n\n" 157 + "| Name | Value |\n| --- | --- |\n| alpha | one |\n| beta | two |\n" 158 + ), 159 + }, 160 + { 161 + "id": "definition_2_of_4", 162 + "input": ( 163 + "# Definitions\n\n" 164 + "- **alpha:** value one\n" 165 + "- ordinary note.\n" 166 + "- **beta:** value two\n" 167 + "- ordinary other.\n" 168 + ), 169 + }, 170 + { 171 + "id": "definition_2_of_5", 172 + "input": ( 173 + "# Boundary\n\n" 174 + "- **alpha:** value one\n" 175 + "- ordinary note.\n" 176 + "- **beta:** value two\n" 177 + "- ordinary other.\n" 178 + "- ordinary final.\n" 179 + ), 180 + }, 181 + { 182 + "id": "definition_1_of_2", 183 + "input": "# Boundary\n\n- **alpha:** value one\n- ordinary note.\n", 184 + }, 185 + { 186 + "id": "multi_row_table", 187 + "input": ( 188 + "# Matrix\n\n" 189 + "| Name | Value |\n| --- | --- |\n" 190 + "| alpha | one |\n| beta | two |\n| gamma | three |\n" 191 + ), 192 + }, 193 + { 194 + "id": "fenced_code_info", 195 + "input": "# Code\n\n```python\nprint('alpha')\n```\n", 196 + }, 197 + { 198 + "id": "blockquote_multi_paragraph", 199 + "input": "# Quote\n\n> alpha quote\n>\n> beta quote\n", 200 + }, 201 + { 202 + "id": "overlong_line", 203 + "input": f"# Long\n\n{long_line}\n\nkept alpha\n", 204 + }, 205 + { 206 + "id": "oversized_chunk", 207 + "input": f"# Big\n\n{oversized_body}\n", 208 + }, 209 + { 210 + "id": "loose_nested_list", 211 + "input": "# Nested\n\n- parent alpha\n\n - child beta\n", 212 + }, 213 + { 214 + "id": "two_paragraph_list_item", 215 + "input": "# Loose\n\n- first alpha\n\n second beta\n", 216 + }, 217 + { 218 + "id": "list_item_fenced_code", 219 + "input": "# Item Code\n\n- alpha before\n\n ```python\n print('beta')\n ```\n", 220 + }, 221 + { 222 + "id": "inline_link", 223 + "input": "# Link\n\n[alpha](https://example.com/path/to-beta?q=gamma)\n", 224 + }, 225 + { 226 + "id": "inline_image", 227 + "input": '# Image\n\n![alt text](images/pic-alpha.png "title beta")\n', 228 + }, 229 + { 230 + "id": "autolink", 231 + "input": "# Auto\n\n<https://example.com/path?q=gamma>\n", 232 + }, 233 + { 234 + "id": "reference_link", 235 + "input": '# Reference\n\n[alpha][ref]\n\n[ref]: https://example.com/path "title beta"\n', 236 + }, 237 + { 238 + "id": "inline_html", 239 + "input": "# Html\n\nalpha <span>beta</span> gamma\n", 240 + }, 241 + ] 242 + 243 + 244 + class _WarningCapture(logging.Handler): 245 + def __init__(self) -> None: 246 + super().__init__(level=logging.WARNING) 247 + self.messages: list[str] = [] 248 + 249 + def emit(self, record: logging.LogRecord) -> None: 250 + self.messages.append(record.getMessage()) 251 + 252 + 253 + def _format_markdown_with_warnings(text: str) -> tuple[list[dict[str, Any]], list[str]]: 254 + logger = logging.getLogger(markdown_formatter.__name__) 255 + handler = _WarningCapture() 256 + logger.addHandler(handler) 257 + try: 258 + chunks, _meta = markdown_formatter.format_markdown(text) 259 + finally: 260 + logger.removeHandler(handler) 261 + return chunks, handler.messages 262 + 263 + 264 + def _fts5_tokens(chunks: list[str]) -> list[list[str]]: 265 + tokens: list[list[str]] = [[] for _chunk in chunks] 266 + conn = sqlite3.connect(":memory:") 267 + try: 268 + conn.execute("CREATE VIRTUAL TABLE chunks USING fts5(content)") 269 + conn.executemany( 270 + "INSERT INTO chunks(content) VALUES (?)", 271 + [(chunk,) for chunk in chunks], 272 + ) 273 + conn.execute("CREATE VIRTUAL TABLE vocab USING fts5vocab(chunks, 'instance')") 274 + rows = conn.execute( 275 + "SELECT doc, offset, term FROM vocab ORDER BY doc, offset" 276 + ).fetchall() 277 + finally: 278 + conn.close() 279 + 280 + for doc, _offset, term in rows: 281 + tokens[int(doc) - 1].append(str(term)) 282 + return tokens 283 + 284 + 285 + def _normalize_oversized_size_tokens(tokens: list[str]) -> list[str]: 286 + normalized: list[str] = [] 287 + i = 0 288 + while i < len(tokens): 289 + if i + 5 < len(tokens) and tokens[i : i + 5] == [ 290 + "content", 291 + "too", 292 + "large", 293 + "to", 294 + "index", 295 + ]: 296 + normalized.extend(tokens[i : i + 5]) 297 + j = i + 5 298 + while j < len(tokens) and tokens[j] != "chars": 299 + j += 1 300 + if j < len(tokens): 301 + normalized.append(OVERSIZED_SIZE_TOKEN) 302 + normalized.append("chars") 303 + i = j + 1 304 + continue 305 + normalized.append(tokens[i]) 306 + i += 1 307 + return normalized 308 + 309 + 310 + def _normalize_tokens(tokens: list[str], normalizations: list[str]) -> list[str]: 311 + if OVERSIZED_SIZE_NORMALIZATION in normalizations: 312 + tokens = _normalize_oversized_size_tokens(tokens) 313 + return tokens 314 + 315 + 316 + def build_markdown_chunks_fixture() -> dict[str, Any]: 317 + cases = [] 318 + for case in _markdown_fixture_cases(): 319 + if not case["input"].isascii(): 320 + raise RuntimeError(f"markdown fixture case is not ASCII-only: {case['id']}") 321 + chunks, warnings = _format_markdown_with_warnings(case["input"]) 322 + rendered = [chunk["markdown"] for chunk in chunks] 323 + tokens_by_chunk = _fts5_tokens(rendered) 324 + chunk_entries = [] 325 + for markdown, tokens in zip(rendered, tokens_by_chunk, strict=True): 326 + normalizations = ( 327 + [OVERSIZED_SIZE_NORMALIZATION] 328 + if "[Content too large to index:" in markdown 329 + else [] 330 + ) 331 + entry: dict[str, Any] = { 332 + "markdown": markdown, 333 + "tokens": _normalize_tokens(tokens, normalizations), 334 + } 335 + if normalizations: 336 + entry["normalizations"] = normalizations 337 + chunk_entries.append(entry) 338 + cases.append( 339 + { 340 + "id": case["id"], 341 + "input": case["input"], 342 + "chunk_count": len(chunks), 343 + "warnings": warnings, 344 + "chunks": chunk_entries, 345 + } 346 + ) 347 + 348 + return { 349 + "fixture": "solstone-markdown-chunks", 350 + "fixture_version": 1, 351 + "generated_by": "make core-fixtures", 352 + "constraints": { 353 + "ascii_only": True, 354 + "max_line_chars": markdown_formatter._MAX_LINE_CHARS, 355 + "max_chunk_chars": markdown_formatter._MAX_CHUNK_CHARS, 356 + "normalizations": { 357 + OVERSIZED_SIZE_NORMALIZATION: ( 358 + "replace content-too-large size number tokens with " 359 + f"{OVERSIZED_SIZE_TOKEN}" 360 + ) 361 + }, 362 + "tokenizer": ( 363 + "sqlite fts5(content) with fts5vocab(chunks, 'instance') " 364 + "ordered by doc, offset" 365 + ), 366 + }, 367 + "cases": cases, 368 + } 369 + 370 + 119 371 def render_json(payload: dict[str, Any]) -> str: 120 372 return json.dumps(payload, indent=2, sort_keys=True) + "\n" 121 373 ··· 125 377 CALLOSUM_ARTIFACT_PATH: render_json(build_callosum_registry_fixture()), 126 378 COGITATE_ARTIFACT_PATH: render_json(build_cogitate_contract_fixture()), 127 379 EDGE_SCHEMA_ARTIFACT_PATH: render_json(build_edge_schema_fixture()), 380 + MARKDOWN_CHUNKS_ARTIFACT_PATH: render_json(build_markdown_chunks_fixture()), 128 381 } 129 382 130 383
+147
tests/_indexer_differential_fixtures.py
··· 35 35 "indexer/journal.sqlite-shm", 36 36 ) 37 37 38 + MARKDOWN_PARITY_CORPUS_FILES = ( 39 + { 40 + "fixture_path": "chronicle/20240102/talents/parity_intro_list_01.md", 41 + "index_path": "20240102/talents/parity_intro_list_01.md", 42 + "structure": "intro paragraph before ordinary list", 43 + }, 44 + { 45 + "fixture_path": "chronicle/20240102/talents/parity_intro_list_02.md", 46 + "index_path": "20240102/talents/parity_intro_list_02.md", 47 + "structure": "intro paragraph before ordinary list", 48 + }, 49 + { 50 + "fixture_path": "chronicle/20240102/talents/parity_intro_list_03.md", 51 + "index_path": "20240102/talents/parity_intro_list_03.md", 52 + "structure": "intro paragraph before ordinary list", 53 + }, 54 + { 55 + "fixture_path": "chronicle/20240102/talents/parity_intro_table_01.md", 56 + "index_path": "20240102/talents/parity_intro_table_01.md", 57 + "structure": "intro paragraph before three-row table", 58 + }, 59 + { 60 + "fixture_path": "chronicle/20240102/talents/parity_intro_table_02.md", 61 + "index_path": "20240102/talents/parity_intro_table_02.md", 62 + "structure": "intro paragraph before three-row table", 63 + }, 64 + { 65 + "fixture_path": "chronicle/20240102/talents/parity_intro_table_03.md", 66 + "index_path": "20240102/talents/parity_intro_table_03.md", 67 + "structure": "intro paragraph before three-row table", 68 + }, 69 + { 70 + "fixture_path": "chronicle/20240102/talents/parity_definition_2_of_4.md", 71 + "index_path": "20240102/talents/parity_definition_2_of_4.md", 72 + "structure": "definition-list 2-of-4 grouping boundary", 73 + }, 74 + { 75 + "fixture_path": "chronicle/20240102/talents/parity_definition_2_of_5.md", 76 + "index_path": "20240102/talents/parity_definition_2_of_5.md", 77 + "structure": "definition-list 2-of-5 non-grouping boundary", 78 + }, 79 + { 80 + "fixture_path": "chronicle/20240102/talents/parity_blockquote.md", 81 + "index_path": "20240102/talents/parity_blockquote.md", 82 + "structure": "multi-paragraph blockquote", 83 + }, 84 + { 85 + "fixture_path": "chronicle/20240102/talents/parity_fenced_code.md", 86 + "index_path": "20240102/talents/parity_fenced_code.md", 87 + "structure": "fenced code with info string", 88 + }, 89 + { 90 + "fixture_path": "chronicle/20240102/talents/parity_loose_nested_list.md", 91 + "index_path": "20240102/talents/parity_loose_nested_list.md", 92 + "structure": "loose nested list", 93 + }, 94 + { 95 + "fixture_path": "chronicle/20240102/talents/parity_multiblock_item.md", 96 + "index_path": "20240102/talents/parity_multiblock_item.md", 97 + "structure": "list item containing two paragraphs", 98 + }, 99 + { 100 + "fixture_path": "chronicle/20240102/talents/parity_list_item_code.md", 101 + "index_path": "20240102/talents/parity_list_item_code.md", 102 + "structure": "list item containing a fenced code block", 103 + }, 104 + { 105 + "fixture_path": "chronicle/20240102/talents/parity_overlong_line.md", 106 + "index_path": "20240102/talents/parity_overlong_line.md", 107 + "structure": "single over-2048-char neutral line plus searchable paragraph", 108 + }, 109 + { 110 + "fixture_path": "facets/work/news/parity_news_intro_list.md", 111 + "index_path": "facets/work/news/parity_news_intro_list.md", 112 + "structure": "work/news intro paragraph before ordinary list", 113 + }, 114 + { 115 + "fixture_path": "facets/work/news/parity_news_intro_table.md", 116 + "index_path": "facets/work/news/parity_news_intro_table.md", 117 + "structure": "work/news intro paragraph before three-row table", 118 + }, 119 + { 120 + "fixture_path": "facets/work/news/parity_news_definition.md", 121 + "index_path": "facets/work/news/parity_news_definition.md", 122 + "structure": "work/news definition-list 2-of-4 grouping boundary", 123 + }, 124 + { 125 + "fixture_path": "facets/work/news/parity_news_code.md", 126 + "index_path": "facets/work/news/parity_news_code.md", 127 + "structure": "work/news fenced code with info string", 128 + }, 129 + ) 130 + 131 + MARKDOWN_PARITY_FULLTEXT_QUERY_CASES = ( 132 + { 133 + "name": "markdown_parity_single_term", 134 + "query": "paritysignal", 135 + "filters": {}, 136 + "reference_total": 35, 137 + "reference_distinct_paths": 18, 138 + "rationale": "single-term query over all markdown parity corpus files", 139 + }, 140 + { 141 + "name": "markdown_parity_and", 142 + "query": "paritysignal AND matrixanchor", 143 + "filters": {}, 144 + "reference_total": 35, 145 + "reference_distinct_paths": 18, 146 + "rationale": "explicit AND query over repeated parity corpus vocabulary", 147 + }, 148 + { 149 + "name": "markdown_parity_phrase", 150 + "query": '"chunk balance"', 151 + "filters": {}, 152 + "reference_total": 35, 153 + "reference_distinct_paths": 18, 154 + "rationale": "quoted phrase query over parity corpus markdown structures", 155 + }, 156 + { 157 + "name": "markdown_parity_prefix_code_info", 158 + "query": "paritycode*", 159 + "filters": {}, 160 + "reference_total": 3, 161 + "reference_distinct_paths": 3, 162 + "rationale": "prefix query proving fenced-code info strings remain searchable", 163 + }, 164 + { 165 + "name": "markdown_parity_work_news", 166 + "query": "paritysignal", 167 + "filters": {"facet": "work", "agent": "news"}, 168 + "reference_total": 8, 169 + "reference_distinct_paths": 4, 170 + "rationale": "query plus real work/news metadata filters on parity corpus", 171 + }, 172 + ) 173 + 174 + MARKDOWN_PARITY_METADATA_FILTER_CASES = ( 175 + { 176 + "name": "markdown_parity_work_news", 177 + "query": "paritysignal", 178 + "filters": {"facet": "work", "agent": "news"}, 179 + "reference_total": 8, 180 + "reference_distinct_paths": 4, 181 + "rationale": "metadata path-set case combining parity query, facet, and agent", 182 + }, 183 + ) 184 + 38 185 FULLTEXT_QUERY_CASES = ( 39 186 { 40 187 "name": "single_term_authentication",
+8
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_blockquote.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Blockquote 5 + 6 + > paritysignal matrixanchor chunk balance quote alpha 7 + > 8 + > paritysignal matrixanchor chunk balance quote beta
+9
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_definition_2_of_4.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Definition 2 Of 4 5 + 6 + - **paritysignal:** matrixanchor chunk balance definition alpha 7 + - ordinary alpha note. 8 + - **matrixanchor:** paritysignal chunk balance definition beta 9 + - ordinary beta note.
+10
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_definition_2_of_5.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Definition 2 Of 5 5 + 6 + - **paritysignal:** matrixanchor chunk balance boundary alpha 7 + - ordinary alpha note. 8 + - **matrixanchor:** paritysignal chunk balance boundary beta 9 + - ordinary beta note. 10 + - ordinary gamma note.
+10
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_fenced_code.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Fenced Code 5 + 6 + paritysignal matrixanchor chunk balance code context. 7 + 8 + ```paritycode 9 + print("neutral alpha") 10 + ```
+10
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_01.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Intro List 01 5 + 6 + paritysignal matrixanchor chunk balance guides intro list alpha. 7 + 8 + - roster alpha ready 9 + - ledger alpha ready 10 + - review alpha ready
+10
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_02.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Intro List 02 5 + 6 + paritysignal matrixanchor chunk balance guides intro list beta. 7 + 8 + - roster beta ready 9 + - ledger beta ready 10 + - review beta ready
+10
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_list_03.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Intro List 03 5 + 6 + paritysignal matrixanchor chunk balance guides intro list gamma. 7 + 8 + - roster gamma ready 9 + - ledger gamma ready 10 + - review gamma ready
+12
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_01.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Intro Table 01 5 + 6 + paritysignal matrixanchor chunk balance guides table alpha. 7 + 8 + | Step | Status | 9 + | --- | --- | 10 + | alpha one | ready | 11 + | alpha two | waiting | 12 + | alpha three | blocked |
+12
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_02.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Intro Table 02 5 + 6 + paritysignal matrixanchor chunk balance guides table beta. 7 + 8 + | Step | Status | 9 + | --- | --- | 10 + | beta one | ready | 11 + | beta two | waiting | 12 + | beta three | blocked |
+12
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_intro_table_03.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Intro Table 03 5 + 6 + paritysignal matrixanchor chunk balance guides table gamma. 7 + 8 + | Step | Status | 9 + | --- | --- | 10 + | gamma one | ready | 11 + | gamma two | waiting | 12 + | gamma three | blocked |
+10
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_list_item_code.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity List Item Code 5 + 6 + - paritysignal matrixanchor chunk balance item before code. 7 + 8 + ```paritycode 9 + echo neutral beta 10 + ```
+8
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_loose_nested_list.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Loose Nested List 5 + 6 + - paritysignal matrixanchor chunk balance parent alpha 7 + 8 + - paritysignal matrixanchor chunk balance child alpha
+8
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_multiblock_item.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Multiblock Item 5 + 6 + - paritysignal matrixanchor chunk balance first paragraph. 7 + 8 + paritysignal matrixanchor chunk balance second paragraph.
+8
tests/fixtures/markdown_parity/chronicle/20240102/talents/parity_overlong_line.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity Overlong Line 5 + 6 + paritysignal matrixanchor chunk balance survives sanitize. 7 + 8 + <!-- zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill zzfill -->
+10
tests/fixtures/markdown_parity/facets/work/news/parity_news_code.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity News Code 5 + 6 + paritysignal matrixanchor chunk balance work news code context. 7 + 8 + ```paritycode 9 + print("neutral news") 10 + ```
+9
tests/fixtures/markdown_parity/facets/work/news/parity_news_definition.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity News Definition 5 + 6 + - **paritysignal:** matrixanchor chunk balance news alpha 7 + - ordinary news note. 8 + - **matrixanchor:** paritysignal chunk balance news beta 9 + - ordinary news beta.
+10
tests/fixtures/markdown_parity/facets/work/news/parity_news_intro_list.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity News Intro List 5 + 6 + paritysignal matrixanchor chunk balance guides work news list. 7 + 8 + - news roster ready 9 + - news ledger ready 10 + - news review ready
+12
tests/fixtures/markdown_parity/facets/work/news/parity_news_intro_table.md
··· 1 + <!-- SPDX-License-Identifier: AGPL-3.0-only --> 2 + <!-- Copyright (c) 2026 sol pbc --> 3 + 4 + # Parity News Intro Table 5 + 6 + paritysignal matrixanchor chunk balance guides work news table. 7 + 8 + | Signal | State | 9 + | --- | --- | 10 + | news one | ready | 11 + | news two | waiting | 12 + | news three | blocked |
+110
tests/integration/test_indexer_markdown_differential.py
··· 1 + # SPDX-License-Identifier: AGPL-3.0-only 2 + # Copyright (c) 2026 sol pbc 3 + 4 + from __future__ import annotations 5 + 6 + import json 7 + import shlex 8 + import shutil 9 + import subprocess 10 + import sys 11 + from pathlib import Path 12 + 13 + import pytest 14 + 15 + from tests import verify_indexer_differential as harness 16 + from tests._indexer_differential_fixtures import ( 17 + FULLTEXT_TOP10_JACCARD_MIN, 18 + MARKDOWN_PARITY_CORPUS_FILES, 19 + MARKDOWN_PARITY_FULLTEXT_QUERY_CASES, 20 + MARKDOWN_PARITY_METADATA_FILTER_CASES, 21 + ) 22 + 23 + ROOT = Path(__file__).resolve().parents[2] 24 + MARKDOWN_PARITY_FIXTURE = ROOT / "tests" / "fixtures" / "markdown_parity" 25 + pytestmark = pytest.mark.integration 26 + 27 + 28 + def _quote_command(*parts: str | Path) -> str: 29 + return " ".join(shlex.quote(str(part)) for part in parts) 30 + 31 + 32 + def _build_native_binary() -> Path: 33 + if shutil.which("cargo") is None: 34 + pytest.skip("cargo is not installed") 35 + subprocess.run( 36 + [ 37 + "cargo", 38 + "build", 39 + "--manifest-path", 40 + "core/Cargo.toml", 41 + "--release", 42 + "-p", 43 + "solstone-core", 44 + ], 45 + cwd=ROOT, 46 + check=True, 47 + ) 48 + binary = ROOT / "core" / "target" / "release" / "solstone-core" 49 + assert binary.exists() 50 + return binary 51 + 52 + 53 + def _build_markdown_parity_corpus(dst: Path) -> Path: 54 + for entry in MARKDOWN_PARITY_CORPUS_FILES: 55 + source = MARKDOWN_PARITY_FIXTURE / entry["fixture_path"] 56 + target = dst / entry["fixture_path"] 57 + assert source.exists(), entry["fixture_path"] 58 + target.parent.mkdir(parents=True, exist_ok=True) 59 + shutil.copy2(source, target) 60 + return dst 61 + 62 + 63 + @pytest.mark.timeout(600) 64 + def test_native_markdown_chunker_matches_python_functional_parity( 65 + tmp_path: Path, 66 + ) -> None: 67 + structures = {entry["structure"] for entry in MARKDOWN_PARITY_CORPUS_FILES} 68 + assert { 69 + "intro paragraph before ordinary list", 70 + "intro paragraph before three-row table", 71 + "definition-list 2-of-4 grouping boundary", 72 + "definition-list 2-of-5 non-grouping boundary", 73 + "multi-paragraph blockquote", 74 + "fenced code with info string", 75 + "loose nested list", 76 + "list item containing two paragraphs", 77 + "list item containing a fenced code block", 78 + "single over-2048-char neutral line plus searchable paragraph", 79 + } <= structures 80 + assert any( 81 + case["reference_distinct_paths"] > 10 82 + for case in MARKDOWN_PARITY_FULLTEXT_QUERY_CASES 83 + ) 84 + 85 + journal_bin = Path(sys.executable).with_name("journal") 86 + if not journal_bin.exists(): 87 + pytest.skip("journal entry point is not installed") 88 + 89 + source_journal = _build_markdown_parity_corpus(tmp_path / "source-journal") 90 + report = harness.run_differential( 91 + journal=source_journal, 92 + command_a=_quote_command(journal_bin, "indexer", "--rescan-full"), 93 + command_b=_quote_command(_build_native_binary(), "indexer", "--rescan-full"), 94 + work_root=tmp_path / "work", 95 + mode="functional", 96 + copy_mode="full", 97 + fulltext_cases=MARKDOWN_PARITY_FULLTEXT_QUERY_CASES, 98 + metadata_cases=MARKDOWN_PARITY_METADATA_FILTER_CASES, 99 + ) 100 + 101 + assert report["classification"] == "functionally-equal", json.dumps( 102 + report, 103 + indent=2, 104 + sort_keys=True, 105 + ) 106 + assert report["functional"]["failed_components"] == [] 107 + assert all( 108 + case["jaccard"] >= FULLTEXT_TOP10_JACCARD_MIN 109 + for case in report["functional"]["fulltext"]["cases"] 110 + )
+20
tests/test_core_fixtures.py
··· 9 9 10 10 from scripts import build_core_fixtures 11 11 from solstone.convey.contract.assemble import CALLOSUM_REGISTRY 12 + from solstone.think import markdown as markdown_formatter 12 13 from solstone.think.cogitate_contract import ( 13 14 COGITATE_ACCESS_TIERS, 14 15 COGITATE_READ_TOOL_NAMES, ··· 54 55 } 55 56 56 57 58 + def test_markdown_core_fixture_matches_formatter_contract() -> None: 59 + fixture = build_core_fixtures.build_markdown_chunks_fixture() 60 + 61 + assert fixture["constraints"]["ascii_only"] is True 62 + assert ( 63 + fixture["constraints"]["max_line_chars"] == markdown_formatter._MAX_LINE_CHARS 64 + ) 65 + assert ( 66 + fixture["constraints"]["max_chunk_chars"] == markdown_formatter._MAX_CHUNK_CHARS 67 + ) 68 + assert {case["id"] for case in fixture["cases"]} == { 69 + case["id"] for case in build_core_fixtures._markdown_fixture_cases() 70 + } 71 + 72 + 57 73 def test_committed_core_fixtures_are_current() -> None: 58 74 for path, expected in build_core_fixtures.expected_outputs().items(): 59 75 assert path.read_text(encoding="utf-8") == expected ··· 69 85 callosum_path = fixture_dir / "callosum_registry.json" 70 86 cogitate_path = fixture_dir / "cogitate_contract.json" 71 87 edge_schema_path = fixture_dir / "edge_schema.json" 88 + markdown_chunks_path = fixture_dir / "markdown_chunks.json" 72 89 73 90 monkeypatch.setattr(build_core_fixtures, "ROOT", root) 74 91 monkeypatch.setattr(build_core_fixtures, "FIXTURE_DIR", fixture_dir) ··· 76 93 monkeypatch.setattr(build_core_fixtures, "COGITATE_ARTIFACT_PATH", cogitate_path) 77 94 monkeypatch.setattr( 78 95 build_core_fixtures, "EDGE_SCHEMA_ARTIFACT_PATH", edge_schema_path 96 + ) 97 + monkeypatch.setattr( 98 + build_core_fixtures, "MARKDOWN_CHUNKS_ARTIFACT_PATH", markdown_chunks_path 79 99 ) 80 100 81 101 build_core_fixtures.write_outputs()
+20
tests/test_indexer_differential.py
··· 371 371 assert classified["unclassified"] == [] 372 372 373 373 374 + def test_stderr_classifier_allows_native_markdown_sanitize_warning() -> None: 375 + stderr = "\n".join( 376 + [ 377 + "warning: Dropped 1 line(s) exceeding 2048 chars during markdown sanitization", 378 + "warning: Dropped 5 line(s) exceeding 2048 chars during markdown sanitization", 379 + ] 380 + ) 381 + classified = harness.classify_stderr(stderr) 382 + native_markdown_rule = next( 383 + rule 384 + for rule in classified["rules"] 385 + if rule["name"] == harness.NATIVE_MARKDOWN_SANITIZE_RULE 386 + ) 387 + assert native_markdown_rule["count"] == 2 388 + assert native_markdown_rule["examples"] == stderr.splitlines() 389 + assert classified["unclassified"] == [] 390 + 391 + 374 392 def test_stderr_classifier_rejects_markdown_near_misses() -> None: 375 393 stderr = "\n".join( 376 394 [ 377 395 "ERROR:solstone.think.markdown:Dropped 1 line(s) exceeding 2048 chars during markdown sanitization", 378 396 "WARNING:solstone.think.other:Dropped 1 line(s) exceeding 2048 chars during markdown sanitization", 379 397 "WARNING:solstone.think.markdown:Some unrelated warning", 398 + "warning: Dropped 1 line(s) exceeding 4096 chars during markdown sanitization", 399 + "warning: dropped 1 line(s) exceeding 2048 chars during markdown sanitization", 380 400 "WARNING:some.other.module:generic warning", 381 401 ] 382 402 )
+38 -9
tests/verify_indexer_differential.py
··· 90 90 r"^WARNING:solstone\.think\.markdown:" 91 91 r"Dropped \d+ line\(s\) exceeding \d+ chars during markdown sanitization$" 92 92 ) 93 + NATIVE_MARKDOWN_SANITIZE_RULE = "native_markdown_sanitize_drop" 94 + NATIVE_MARKDOWN_SANITIZE_RE = re.compile( 95 + r"^warning: Dropped \d+ line\(s\) exceeding 2048 chars during markdown sanitization$" 96 + ) 93 97 EXCLUDED_SHADOW_TABLES = [ 94 98 "chunks_config", 95 99 "chunks_content", ··· 286 290 def classify_stderr(stderr: str) -> dict[str, Any]: 287 291 edge_rule = {"name": EDGE_SKIP_RULE, "count": 0, "examples": []} 288 292 markdown_rule = {"name": MARKDOWN_SANITIZE_RULE, "count": 0, "examples": []} 293 + native_markdown_rule = { 294 + "name": NATIVE_MARKDOWN_SANITIZE_RULE, 295 + "count": 0, 296 + "examples": [], 297 + } 289 298 unclassified: list[str] = [] 290 299 for line in stderr.splitlines(): 291 300 if not line.strip(): ··· 294 303 _record_rule_hit(edge_rule, line) 295 304 elif MARKDOWN_SANITIZE_RE.match(line): 296 305 _record_rule_hit(markdown_rule, line) 306 + elif NATIVE_MARKDOWN_SANITIZE_RE.match(line): 307 + _record_rule_hit(native_markdown_rule, line) 297 308 else: 298 309 unclassified.append(line) 299 - return {"rules": [edge_rule, markdown_rule], "unclassified": unclassified} 310 + return { 311 + "rules": [edge_rule, markdown_rule, native_markdown_rule], 312 + "unclassified": unclassified, 313 + } 300 314 301 315 302 316 def _database_check(journal: Path) -> dict[str, Any]: ··· 821 835 822 836 823 837 def _compare_metadata_filters( 824 - left_journal: Path, right_journal: Path 838 + left_journal: Path, 839 + right_journal: Path, 840 + cases: tuple[dict[str, Any], ...] = METADATA_FILTER_CASES, 825 841 ) -> dict[str, Any]: 826 842 cases = [ 827 - _compare_metadata_case(left_journal, right_journal, case) 828 - for case in METADATA_FILTER_CASES 843 + _compare_metadata_case(left_journal, right_journal, case) for case in cases 829 844 ] 830 845 return {"passed": all(case["equal"] for case in cases), "cases": cases} 831 846 ··· 903 918 return report 904 919 905 920 906 - def _compare_fulltext(left_journal: Path, right_journal: Path) -> dict[str, Any]: 921 + def _compare_fulltext( 922 + left_journal: Path, 923 + right_journal: Path, 924 + cases: tuple[dict[str, Any], ...] = FULLTEXT_QUERY_CASES, 925 + ) -> dict[str, Any]: 907 926 cases = [ 908 - _compare_fulltext_case(left_journal, right_journal, case) 909 - for case in FULLTEXT_QUERY_CASES 927 + _compare_fulltext_case(left_journal, right_journal, case) for case in cases 910 928 ] 911 929 return {"passed": all(case["passed"] for case in cases), "cases": cases} 912 930 ··· 915 933 left_db: Path, 916 934 right_db: Path, 917 935 scratch_root: Path, 936 + *, 937 + fulltext_cases: tuple[dict[str, Any], ...] = FULLTEXT_QUERY_CASES, 938 + metadata_cases: tuple[dict[str, Any], ...] = METADATA_FILTER_CASES, 918 939 ) -> dict[str, Any]: 919 940 left_direct, left_search = _snapshot_functional_side( 920 941 left_db, ··· 929 950 930 951 files = _compare_functional_files(left_direct, right_direct) 931 952 chunk_coverage = _compare_functional_coverage(left_direct, right_direct) 932 - metadata_filters = _compare_metadata_filters(left_search, right_search) 933 - fulltext = _compare_fulltext(left_search, right_search) 953 + metadata_filters = _compare_metadata_filters( 954 + left_search, 955 + right_search, 956 + metadata_cases, 957 + ) 958 + fulltext = _compare_fulltext(left_search, right_search, fulltext_cases) 934 959 edges = _compare_functional_edges(left_direct, right_direct) 935 960 936 961 component_passes = { ··· 971 996 seed: int | None = None, 972 997 mode: str = "byte", 973 998 copy_mode: str = "git", 999 + fulltext_cases: tuple[dict[str, Any], ...] = FULLTEXT_QUERY_CASES, 1000 + metadata_cases: tuple[dict[str, Any], ...] = METADATA_FILTER_CASES, 974 1001 ) -> dict[str, Any]: 975 1002 if mode not in MODES: 976 1003 raise ValueError(f"unknown differential mode: {mode!r}") ··· 1026 1053 Path(commands[0]["checks"]["database"]["db_path"]), 1027 1054 Path(commands[1]["checks"]["database"]["db_path"]), 1028 1055 work_root / "functional", 1056 + fulltext_cases=fulltext_cases, 1057 + metadata_cases=metadata_cases, 1029 1058 ) 1030 1059 report["classification"] = comparison["classification"] 1031 1060 report["functional"] = comparison["functional"]