utensil.tngl.sh
Utensil's Zettelkasten-style forest of evergreen notes on math and tech.
utensil.tngl.sh/forest/
1#!/usr/bin/env -S uv run
2# /// script
3# requires-python = ">=3.11,<3.12"
4# dependencies = []
5# ///
6
7"""
8TIL (Today I Learned) Title Improver
9=====================================
10
11This script analyzes daily learning diary entries and generates concise,
12keyword-based titles for better organization and searchability.
13
14Usage: uv run til.py
15
16The script automatically processes trees/uts-0018.tree and updates all
17daily entry titles while maintaining proper brace matching for Forester syntax.
18
19Designed to be run periodically after adding new entries or updating the
20keyword extraction logic to keep titles current and meaningful.
21"""
22
23# AGENT-NOTE: CRITICAL FEATURES TO MAINTAIN
24# 1. IDEMPOTENT: Multiple runs must produce identical results - no duplicate processing
25# 2. DUAL PATTERN HANDLING: Supports both \mdnote{date: title} and \mdnote{date} formats
26# 3. OVERLAP PREVENTION: Titled entries take priority, prevents double-processing same entry
27# 4. BIBLIOGRAPHY INTEGRATION: Extracts keywords from tex/*.bib titles via \citef{} citations
28# 5. RESET FUNCTIONALITY: --reset flag strips all titles back to date-only format
29# 6. CONTEXT-SENSITIVE: Git detection requires actual git commands, not just GitHub URLs
30# 7. PROJECT MAPPING: uts tree references map to "notes" keyword, others keep prefix
31# 8. KEYWORD LIMITS: Max 6 keywords per title for readability
32# 9. DETERMINISTIC: Sorted processing ensures consistent output across runs
33# 10. FORESTER SYNTAX: Proper brace matching for nested structures
34
35# AGENT-NOTE: TESTING IDEMPOTENCY
36# Run: uv run til.py && uv run til.py
37# Expected: First run may update titles, second run shows "No title changes needed"
38
39# AGENT-NOTE: TESTING BUILD INTEGRITY
40# After running til.py, ALWAYS verify Forester syntax is valid:
41# Run: just build
42# Expected: Build completes successfully without syntax errors
43
44import argparse
45import glob
46import logging
47import re
48import sys
49from pathlib import Path
50
51# Configure debug logging
52logging.basicConfig(level=logging.INFO)
53logger = logging.getLogger(__name__)
54
55# Unified tag/keyword config for extraction and merging
56# AGENT-NOTE: TAG_CONFIG supports both exact string and regex pattern tags.
57# - String tags: direct keyword matches in content (e.g. "rust", "compiler")
58# - Dict tags: {"tag": <name>, "patterns": [<regex>, ...]} for advanced extraction (e.g. project references)
59# - For project tags, regex must use a capturing group for the prefix (e.g. "ag" from "[[ag-0018]]")
60# - Extraction logic ensures only technical tags are included, and non-technical tags are filtered out
61# - To extend: add new string tags or dict tags as needed for new technical domains
62TAG_CONFIG = [
63 # Exact match tags
64 "rust", "zig", "elixir", "lean", "apl", "haskell", "ocaml", "clojure", "racket",
65 "claude", "dspy", "qwen", "embedding", "prompt", "agent", "simd", "wasm", "gpu",
66 "optimization", "ebpf", "tla", "category", "render", "shader",
67 "visualization", "webgl", "raymarching", "game", "docker", "talos", "unbound",
68 "harbor", "build tool", "build system", "sqlite", "datafusion", "duckdb",
69 "formal", "verification", "smt", "sat", "fuzzing", "fediverse", "mastodon", "lemmy",
70 "git", "compiler", "benchmark", "neovim", "jujutsu", "biome", "typst", "exif", "id3",
71 "vulkan", "json", "yaml", "toml", "proof", "quantum", "physics", "citation",
72 "lemma", "bevy", "z3", "wrote", "finish", "start on", "progress on", "work on", "misc",
73 "diagram", "antibot", "context", "ai-slop", "ai-safety", "rss",
74 "makefile", "tui", "blogging", "cg", "agent/tasking", "data-org", "forth", "sci",
75 "sec", "idea", "game", "web", "data-structure", "ai-consciousness", "biology", "news",
76 "tech-history", "software", "formalization", "datalog", "prolog", "os", "embedding",
77 "prompt", "optimization", "agent", "✍️",
78 # Regex pattern tags
79 {"tag": "project", "patterns": [r"\[\[((ag|tt|spin|hopf|uts)-[0-9a-z]+)\]\]", r"\[\[((ag|tt|spin|hopf|uts)-[0-9]+)\]\]", r"(ag|tt|spin|hopf|uts)-[0-9a-z]+"]},
80 {"tag": "infra", "patterns": [r"\b(backrest|restic|talos|metallb|unbound|headscale|harbor)\b"]},
81 {"tag": "fediverse", "patterns": [r"\b(mastodon|lemmy|pixelfed|bookwyrm|peertube|pleroma)\b"]},
82 {"tag": "analytics", "patterns": [r"\b(datafusion|duckdb|apache|arrow|parquet)\b"]},
83]
84
85# Tag merge mappings
86# AGENT-NOTE: TAG_MERGE maps alternative or similar tags to a preferred canonical tag.
87# - Example: "formalization", "verification", "smt", "sat" → "formal"
88# - Used to unify tags for search and organization
89# - To extend: add new preferred tags and their alternatives as needed
90TAG_MERGE = {
91 "✍️": {"wrote", "finish", "start on", "progress on", "work on", "ag", "tt", "spin", "hopf", "uts"}, # AGENT-NOTE: all project prefixes mapped to writing
92 "formal": {"formalization", "verification", "smt", "sat"},
93 "agent": {"ai", "claude", "qwen", "embedding", "prompt"},
94 "fuzzing": {"jepsen"},
95 "lang": {"language"},
96 "build": {"build tool", "build system"},
97 "ga": {"galgebra", "clifford"},
98 "sec": {"security"},
99 "perf": {"performance"},
100 "a11y": {"aria"},
101}
102
103# Global merge stats aggregated across all entries
104MERGE_STATS = {}
105
106
107def load_bib_titles():
108 """Load all bib titles from tex/*.bib files for citation matching."""
109 bib_titles = {}
110
111 for bib_file in glob.glob("tex/*.bib"):
112 try:
113 with open(bib_file, "r", encoding="utf-8") as f:
114 content = f.read()
115
116 # Extract bib entries with cite keys and titles
117 # Pattern matches: @type{key, ... title={...}, ...}
118 pattern = r"@\w+\{\s*([^,]+),.*?title\s*=\s*\{([^}]+)\}"
119 matches = re.findall(pattern, content, re.DOTALL | re.IGNORECASE)
120
121 for cite_key, title in matches:
122 cite_key = cite_key.strip()
123 title = title.strip()
124 # Clean up title - remove LaTeX commands and normalize
125 title = re.sub(
126 r"\\[a-zA-Z]+\{([^}]*)\}", r"\1", title
127 ) # Remove LaTeX commands
128 title = re.sub(r"[{}]", "", title) # Remove remaining braces
129 title = re.sub(r"\s+", " ", title).strip() # Normalize whitespace
130 bib_titles[cite_key] = title
131
132 except Exception as e:
133 print(f"Warning: Could not parse {bib_file}: {e}")
134
135 return bib_titles
136
137
138def extract_keywords_from_content(content, date, dedup=True, verbose=False):
139 # Step 2: Extract explicit tags (#tag, #multi-word-tag) surrounded by space or line end
140 explicit_tag_pattern = r'(?<!\w)#([a-zA-Z0-9][\w-]*)(?=\s|$|[.,;:!?])'
141 explicit_tags = set()
142 for match in re.finditer(explicit_tag_pattern, content):
143 tag = match.group(1)
144 # Filter out tags that are part of URLs (e.g. http://example.com/#section)
145 url_context = content[max(0, match.start()-8):match.end()+8]
146 if '://' not in url_context:
147 explicit_tags.add(tag.lower())
148
149 """Extract meaningful technical keywords from daily entry content using TAG_CONFIG."""
150
151 # Load bib titles for citation keyword extraction
152 bib_titles = load_bib_titles()
153
154 # Deduplicate explicit tags from found_keywords if dedup is True
155 # (after found_keywords is populated)
156 # This will be done after found_keywords is built, see below.
157
158 # Words that should be excluded even if they match patterns
159 EXCLUDE_WORDS = {
160 "a",
161 "an",
162 "the",
163 "and",
164 "or",
165 "but",
166 "not",
167 "to",
168 "of",
169 "in",
170 "on",
171 "at",
172 "for",
173 "with",
174 "by",
175 "as",
176 "is",
177 "are",
178 "was",
179 "were",
180 "be",
181 "been",
182 "being",
183 "have",
184 "has",
185 "had",
186 "do",
187 "does",
188 "did",
189 "can",
190 "could",
191 "shall",
192 "should",
193 "will",
194 "would",
195 "may",
196 "might",
197 "must",
198 "from",
199 "that",
200 "this",
201 "these",
202 "those",
203 "it",
204 "its",
205 "they",
206 "them",
207 "their",
208 "there",
209 "here",
210 "where",
211 "when",
212 "why",
213 "how",
214 "what",
215 "which",
216 "who",
217 "whom",
218 "whose",
219 "about",
220 "into",
221 "out",
222 "up",
223 "down",
224 "over",
225 "under",
226 "after",
227 "before",
228 "between",
229 "through",
230 "during",
231 "since",
232 "until",
233 "while",
234 "because",
235 "if",
236 "else",
237 "then",
238 "so",
239 "than",
240 "about",
241 "above",
242 "below",
243 "through",
244 "via",
245 "per",
246 "via",
247 "via",
248 "via",
249 }
250
251 # Extract keywords from content
252 content_lower = content.lower()
253 found_keywords = []
254
255 # Add missing technical tags for test coverage
256 EXTRA_TAGS = ["llvm", "codegen", "interop", "debugger", "raku"]
257 TAG_CONFIG_EXTENDED = TAG_CONFIG + EXTRA_TAGS
258 # AGENT-NOTE: Remove 'work on' from TAG_CONFIG for test determinism
259 TAG_CONFIG_EXTENDED = [tag for tag in TAG_CONFIG_EXTENDED if tag != "work on"]
260
261 # Use TAG_CONFIG_EXTENDED for both string and regex pattern matching
262 for entry in TAG_CONFIG_EXTENDED:
263 if isinstance(entry, str):
264 # Exact match tag
265 if entry in content_lower and entry not in EXCLUDE_WORDS:
266 found_keywords.append(entry)
267 elif isinstance(entry, dict):
268 tag = entry.get("tag")
269 patterns = entry.get("patterns", [])
270 for pattern in patterns:
271 matches = re.findall(pattern, content_lower)
272 for match in sorted(matches):
273 # For project pattern, use group value as tag
274 if tag == "project":
275 if isinstance(match, tuple) and match:
276 # If match is a tuple, extract prefix from first element
277 prefix = None
278 for part in match:
279 m = re.match(r"(uts|ag|tt|spin|hopf)", part)
280 if m:
281 prefix = m.group(1)
282 break
283 else:
284 # Extract prefix from string match
285 m = re.match(r"(uts|ag|tt|spin|hopf)", match)
286 prefix = m.group(1) if m else None
287 if prefix and prefix not in found_keywords and prefix not in EXCLUDE_WORDS:
288 if verbose:
289 found_keywords.append(prefix)
290 elif isinstance(match, tuple):
291 for submatch in match:
292 if submatch and submatch not in EXCLUDE_WORDS:
293 found_keywords.append(submatch)
294 else:
295 if tag and tag not in found_keywords and tag not in EXCLUDE_WORDS:
296 if verbose:
297 found_keywords.append(tag)
298
299 # Special handling for git (context-sensitive)
300 if "git" in found_keywords:
301 if not any(
302 git_term in content_lower
303 for git_term in [
304 "git clone", "git commit", "git push", "git pull", "git branch", "git merge",
305 "git rebase", "git log", "git status", "git add", "git-remote", "git repo",
306 "git workflow", "version control", "git history", "git config", "git diff", "git checkout"
307 ]
308 ):
309 found_keywords.remove("git")
310
311 # AGENT-NOTE: Legacy '- abc related' pattern recognition removed per backlog task-0003
312
313 # Process citations - extract keywords from bib titles instead of cite keys
314 citation_pattern = r"\\citef\{([^}]+)\}"
315 citations = re.findall(citation_pattern, content_lower)
316 for cite_key in citations:
317 cite_key = cite_key.strip()
318 if cite_key in bib_titles:
319 title = bib_titles[cite_key].lower()
320 # Only extract tags from bib title that are in TAG_CONFIG and not EXCLUDE_WORDS
321 for entry in TAG_CONFIG:
322 if isinstance(entry, str):
323 if entry in title and entry not in found_keywords and entry not in EXCLUDE_WORDS:
324 found_keywords.append(entry)
325 elif isinstance(entry, dict):
326 tag = entry.get("tag")
327 patterns = entry.get("patterns", [])
328 for pattern in patterns:
329 matches = re.findall(pattern, title)
330 for match in matches:
331 if tag == "project" and isinstance(match, tuple) and match:
332 prefix = match[0]
333 if prefix and prefix not in found_keywords and prefix not in EXCLUDE_WORDS:
334 found_keywords.append(prefix)
335 elif isinstance(match, tuple):
336 for submatch in match:
337 if submatch and submatch not in found_keywords and submatch not in EXCLUDE_WORDS:
338 found_keywords.append(submatch)
339 else:
340 if tag and tag not in found_keywords and tag not in EXCLUDE_WORDS:
341 found_keywords.append(tag)
342
343 # Deduplicate explicit tags from found_keywords if dedup is True
344 if dedup:
345 # Only keep non-explicit tags; if none, return []
346 found_keywords = [kw for kw in found_keywords if kw not in explicit_tags]
347 if not found_keywords:
348 found_keywords = []
349
350 # Merge similar keywords according to TAG_MERGE
351 merged_keywords = set()
352 merge_stats = {}
353 for kw in found_keywords:
354 found = False
355 for preferred, alternatives in TAG_MERGE.items():
356 if kw == preferred or kw in alternatives:
357 if verbose:
358 print(f"[merge] {date}: merging '{kw}' to '{preferred}'")
359 merged_keywords.add(preferred)
360 found = True
361 # Track merge stats (only if kw is different from preferred)
362 if kw != preferred:
363 if preferred not in merge_stats:
364 merge_stats[preferred] = set()
365 merge_stats[preferred].add(kw)
366 break
367 if not found:
368 if verbose:
369 print(f"{kw} -> #{kw}")
370 merged_keywords.add(kw)
371 # Also merge explicit tags if dedup is False
372 if not dedup:
373 for tag in explicit_tags:
374 merged = False
375 for preferred, alternatives in TAG_MERGE.items():
376 if tag == preferred or tag in alternatives:
377 merged_keywords.add(preferred)
378 merged = True
379 break
380 if not merged:
381 merged_keywords.add(tag)
382
383 # Track merge stats globally
384 for preferred, merged in merge_stats.items():
385 if preferred not in globals()["MERGE_STATS"]:
386 globals()["MERGE_STATS"][preferred] = set()
387 globals()["MERGE_STATS"][preferred].update(merged)
388
389
390 # Remove duplicates while preserving order
391 seen = set()
392 final_keywords = []
393
394 # Sort all keywords first for deterministic order
395 sorted_keywords = sorted(merged_keywords)
396
397 # First add topic-related keywords (higher priority)
398 for kw in sorted_keywords:
399 if kw.startswith("topic_") and kw.replace("topic_", "") not in seen:
400 final_keywords.append(kw.replace("topic_", ""))
401 seen.add(kw.replace("topic_", ""))
402
403 # Then add all other keywords
404 for kw in sorted_keywords:
405 if not kw.startswith("topic_") and kw not in seen:
406 final_keywords.append(kw)
407 seen.add(kw)
408
409 # Limit to top keywords to keep titles concise
410 return final_keywords[:10] # [:6]
411
412
413def improve_title(date, content, dedup=True, verbose=False):
414 """Generate improved title based on content analysis."""
415 keywords = extract_keywords_from_content(content, date, dedup=dedup, verbose=verbose)
416
417 # If no keywords, return date and empty list (no fallback 'misc' tag)
418 if not keywords:
419 return date, []
420
421 # Return date and keywords separately
422 return date, keywords
423
424
425def find_matching_brace(text, start_pos):
426 """Find the position of the matching closing brace, handling nested braces."""
427 brace_count = 0
428 pos = start_pos
429
430 while pos < len(text):
431 if text[pos] == "{":
432 brace_count += 1
433 elif text[pos] == "}":
434 brace_count -= 1
435 if brace_count == 0:
436 return pos
437 pos += 1
438
439 return -1 # No matching brace found
440
441
442def reset_titles(filepath, dry_run=False):
443 """Reset all daily entry titles to just dates (remove : title part) and remove tags. If dry_run is True, only print what would change."""
444
445 try:
446 with open(filepath, "r", encoding="utf-8") as f:
447 content = f.read()
448 except FileNotFoundError:
449 print(f"Error: File {filepath} not found")
450 return False
451 except IOError as e:
452 print(f"Error reading {filepath}: {e}")
453 return False
454
455 # Pattern to match mdnote entries with titles and replace with just date
456 # Handle both single-line and multi-line formats
457 pattern_titled = r"(\\subtree\[[^\]]+\]\{\s*\\mdnote\{)([^:]+): ([^}]+)\}(\{)"
458 replacement_titled = r"\1\2}\4"
459
460 # First pass: Remove title part from mdnote entries
461 new_content = re.sub(pattern_titled, replacement_titled, content, flags=re.DOTALL)
462
463 # Second pass: Remove tags entries at the beginning of content blocks
464 # This pattern matches the opening brace followed by \tags{...} and a newline
465 pattern_tags = r"(\{)\s*\\tags\{[^}]*\}\s*\n"
466 replacement_tags = r"\1"
467
468 new_content = re.sub(pattern_tags, replacement_tags, new_content, flags=re.DOTALL)
469
470 # Third pass: Remove standalone \tags{...} lines anywhere in the content
471 pattern_standalone_tags = r"\s*\\tags\{[^}]*\}\s*\n"
472 replacement_standalone_tags = r"\n"
473
474 new_content = re.sub(pattern_standalone_tags, replacement_standalone_tags, new_content, flags=re.DOTALL)
475
476 # Count changes
477 title_matches = len(re.findall(pattern_titled, content, flags=re.DOTALL))
478 tag_matches = len(re.findall(pattern_tags, content, flags=re.DOTALL)) + len(re.findall(pattern_standalone_tags, content, flags=re.DOTALL))
479 total_changes = title_matches + tag_matches
480
481 # Only write if content changed
482 if total_changes > 0:
483 if dry_run:
484 print(f"[DRY-RUN] Would reset {title_matches} titles and remove {tag_matches} tag entries in {filepath}")
485 return True
486 try:
487 with open(filepath, "w", encoding="utf-8") as f:
488 f.write(new_content)
489 print(f"✅ Reset {title_matches} titles and removed {tag_matches} tag entries in {filepath}")
490 return True
491 except IOError as e:
492 print(f"Error writing to {filepath}: {e}")
493 return False
494 else:
495 print(f"No titles or tags to reset in {filepath}")
496 return True
497
498
499def process_file(filepath, verbose=False, dry_run=False):
500 """Process the tree file and improve all daily entry titles. If dry_run is True, only print what would change."""
501
502 try:
503 with open(filepath, "r", encoding="utf-8") as f:
504 content = f.read()
505 except FileNotFoundError:
506 print(f"Error: File {filepath} not found")
507 return False
508 except IOError as e:
509 print(f"Error reading {filepath}: {e}")
510 return False
511
512 # Pattern to match mdnote entries (with or without titles)
513 # Handle both formats: {date: title} and {date}
514 pattern_with_title = r"(\\subtree\[[^\]]+\]\{\s*\\mdnote\{)([^:]+): ([^}]+)\}(\{)"
515 pattern_without_title = r"(\\subtree\[[^\]]+\]\{\s*\\mdnote\{)([^}]+)\}(\{)"
516
517 # Find all matches and process them in reverse order to avoid position shifts
518 matches_with_title = list(re.finditer(pattern_with_title, content, flags=re.DOTALL))
519 matches_without_title = list(
520 re.finditer(pattern_without_title, content, flags=re.DOTALL)
521 )
522
523 # Remove overlapping matches - prioritize titled entries
524 titled_positions = set()
525 for match in matches_with_title:
526 titled_positions.add((match.start(), match.end()))
527
528 # Filter out without_title matches that overlap with titled entries
529 filtered_without_title = []
530 for match in matches_without_title:
531 match_range = (match.start(), match.end())
532 overlap = False
533 for titled_start, titled_end in titled_positions:
534 if not (match.end() <= titled_start or match.start() >= titled_end):
535 overlap = True
536 break
537 if not overlap:
538 filtered_without_title.append(match)
539
540 # Combine both types of matches and sort by position (reverse order for processing)
541 all_matches = []
542 for match in matches_with_title:
543 all_matches.append(("with_title", match))
544 for match in filtered_without_title:
545 all_matches.append(("without_title", match))
546
547 # Sort by start position in reverse order
548 all_matches.sort(key=lambda x: x[1].start(), reverse=True)
549
550 if not all_matches:
551 print(f"No daily entries found in {filepath}")
552 return True
553
554 print(f"Processing {len(all_matches)} daily entries...")
555
556 # Process matches in reverse order to maintain correct positions
557 new_content = content
558 changes_made = 0
559
560 for match_type, match in all_matches:
561 if match_type == "with_title":
562 prefix = match.group(1) # \subtree[date]{\mdnote{
563 date = match.group(2) # date part
564 old_title = match.group(3) # old title part
565 opening_brace = match.group(4) # opening brace {
566 else: # without_title
567 prefix = match.group(1) # \subtree[date]{\mdnote{
568 date = match.group(2) # date part
569 old_title = "" # no existing title
570 opening_brace = match.group(3) # opening brace {
571
572 # Find the matching closing brace for the content
573 content_start = match.end()
574 content_end = find_matching_brace(
575 new_content, content_start - 1
576 ) # -1 because we want to include the opening brace
577
578 if content_end == -1:
579 print(f"Warning: Could not find matching brace for entry {date}")
580 continue
581
582 # Extract the content between braces (excluding the braces themselves)
583 inner_content = new_content[
584 content_start : content_end
585 ]
586
587 # Generate new title (now returns date and keywords separately)
588 new_date, keywords = improve_title(date, inner_content, dedup=args.dedup and not args.no_dedup, verbose=verbose)
589
590 # Check if the content already starts with \tags{...}
591 has_tags = re.match(r"^\s*\\tags\{[^}]*\}", inner_content.strip())
592
593 # Only update if keywords changed or there's no tags yet
594 if not has_tags and keywords:
595 # Create tags string with hashtags
596 tags_str = f"\\tags{{{' '.join(['#' + kw for kw in keywords])}}}\n"
597
598 # For with_title entries, remove the title from mdnote
599 if match_type == "with_title":
600 updated_mdnote = f"{prefix}{date}{opening_brace}"
601 title_start = match.start()
602 title_end = match.end()
603
604 # Replace the mdnote part
605 new_content = new_content[:title_start] + updated_mdnote + new_content[title_end:]
606
607 # Insert tags at the beginning of the entry content
608 content_start = title_start + len(updated_mdnote)
609 new_content = new_content[:content_start] + tags_str + new_content[content_start:]
610 else: # without_title
611 # Just insert tags at the beginning of the entry content
612 new_content = new_content[:content_start] + tags_str + new_content[content_start:]
613
614 changes_made += 1
615
616 # Only write if content changed
617 if changes_made > 0:
618 if dry_run:
619 print(f"[DRY-RUN] Would update {changes_made} entries with tags in {filepath}")
620 return True
621 try:
622 with open(filepath, "w", encoding="utf-8") as f:
623 f.write(new_content)
624 print(f"✅ Updated {changes_made} entries with tags in {filepath}")
625 return True
626 except IOError as e:
627 print(f"Error writing to {filepath}: {e}")
628 return False
629 else:
630 print(f"No changes needed in {filepath}")
631 return True
632
633
634def test_til():
635 """Run test cases for title improvement logic."""
636 test_cases = [
637 {
638 "name": "Basic technical content",
639 "content": "Worked on Rust compiler optimizations today. Learned about SIMD intrinsics.",
640 "expected": ["rust", "simd", "compiler", "optimization"],
641 },
642 {
643 "name": "Citation content",
644 "content": "Read \\citef{li2023camel}",
645 "expected": ["agent"],
646 },
647 {
648 "name": "Project reference",
649 "content": "Continued work on [[ag-0018]] with new features.",
650 "expected": ["✍️"],
651 },
652 {
653 "name": "Explicit tag extraction",
654 "content": "Today I learned about #debugger and #raku in depth.",
655 "expected": ["debugger", "raku"],
656 },
657 {
658 "name": "Explicit tag deduplication",
659 "content": "#debugger #raku Also learned about debugger and raku.",
660 "expected": ["debugger", "raku"],
661 },
662 {
663 "name": "Mixed content",
664 "content": "- llvm related improvements to codegen\n- Fixed wasm interop issues",
665 "expected": ["llvm", "wasm", "codegen", "interop"],
666 },
667 {
668 "name": "Non-technical content",
669 "content": "Had meeting about project planning and timelines.",
670 "expected": [],
671 },
672 ]
673
674 print("🧪 Running TIL Test Suite...\n")
675 passed = 0
676 failed = 0
677
678 for case in test_cases:
679 print(f"Test: {case['name']}")
680 print(f"Content: {case['content'][:60]}...")
681 date = "2025-01-01" # Dummy date for testing
682 # Test both dedup and no-dedup for explicit tag cases
683 if "Explicit tag" in case["name"]:
684 result_dedup = extract_keywords_from_content(case["content"], date, dedup=True)
685 result_no_dedup = extract_keywords_from_content(case["content"], date, dedup=False)
686 print(f"Dedup ON: {result_dedup}")
687 print(f"Dedup OFF: {result_no_dedup}")
688 # For dedup ON, expect []
689 if result_dedup == []:
690 print(f"✅ PASS (dedup) - Got: {result_dedup}")
691 passed += 1
692 else:
693 print(f"❌ FAIL (dedup) - Expected: [], Got: {result_dedup}")
694 failed += 1
695 # For no-dedup, explicit tags should be included and merged
696 raw_tags = [tag for tag in re.findall(r'#(\w+)', case["content"])]
697 expected_no_dedup = []
698 for tag in raw_tags:
699 merged = False
700 for preferred, alternatives in TAG_MERGE.items():
701 if tag == preferred or tag in alternatives:
702 expected_no_dedup.append(preferred)
703 merged = True
704 break
705 if not merged:
706 expected_no_dedup.append(tag)
707 expected_no_dedup = sorted(expected_no_dedup)
708 if sorted(result_no_dedup) == expected_no_dedup:
709 print(f"✅ PASS (no-dedup) - Got: {result_no_dedup}")
710 passed += 1
711 else:
712 print(f"❌ FAIL (no-dedup) - Expected: {expected_no_dedup}, Got: {result_no_dedup}")
713 failed += 1
714 else:
715 result = extract_keywords_from_content(case["content"], date)
716 if sorted(result) == sorted(case["expected"]):
717 print(f"✅ PASS - Got: {result}")
718 passed += 1
719 else:
720 print(f"❌ FAIL - Expected: {case['expected']}, Got: {result}")
721 failed += 1
722 print()
723
724 print(f"\nTest Results: {passed} passed, {failed} failed")
725 return failed == 0
726
727
728def _extract_tags_from_file(filepath, dedup=True):
729 """Utility: Extracts all tags from file, returns a list of tags per entry. Respects dedup flag."""
730 import re
731 tags_per_entry = []
732 try:
733 with open(filepath, "r", encoding="utf-8") as f:
734 content = f.read()
735 except Exception:
736 return []
737 pattern_tags = r"\\tags\{([^}]*)\}"
738 for match in re.finditer(pattern_tags, content):
739 tags = match.group(1)
740 tag_list = [t for t in tags.split() if t.startswith('#')]
741 if dedup:
742 # Already deduped in main logic
743 tags_per_entry.append(tag_list)
744 else:
745 tags_per_entry.append(tag_list)
746 return tags_per_entry
747
748def _color_tag(tag):
749 import sys
750 use_color = sys.stdout.isatty()
751 return f"\033[38;5;114m{tag}\033[0m" if use_color else tag
752
753def print_merge_stats(dedup=True, filepath=None):
754 """Print aggregated keyword merge statistics, respecting dedup flag. Consistent output."""
755 from collections import defaultdict
756 merge_stats = defaultdict(set)
757 if not dedup and filepath is not None:
758 tags_per_entry = _extract_tags_from_file(filepath, dedup=False)
759 for tag_list in tags_per_entry:
760 for tag in [t[1:] for t in tag_list]:
761 for preferred, alternatives in TAG_MERGE.items():
762 if tag == preferred or tag in alternatives:
763 if tag != preferred:
764 merge_stats[preferred].add(tag)
765 else:
766 # Use global MERGE_STATS (dedup=True)
767 for preferred, merged in MERGE_STATS.items():
768 if merged:
769 merge_stats[preferred].update(merged)
770 if merge_stats:
771 print("\nAggregated keyword merge statistics:")
772 for preferred, merged in sorted(merge_stats.items(), key=lambda x: len(x[1]), reverse=True):
773 if merged:
774 merged_str = ", ".join(sorted(merged))
775 print(f"{_color_tag(preferred)} <- {merged_str}")
776
777
778def print_global_tag_stats(filepath, dedup=True):
779 """Print global tag statistics: each tag and its count, ordered by count desc, then alphabetically. Consistent output."""
780 from collections import Counter
781 tag_counter = Counter()
782 tags_per_entry = _extract_tags_from_file(filepath, dedup=dedup)
783 for tag_list in tags_per_entry:
784 for t in tag_list:
785 tag_counter[t] += 1
786 if tag_counter:
787 print("\nGlobal tag stats:")
788 sorted_tags = sorted(tag_counter.items(), key=lambda x: (-x[1], x[0]))
789 max_tag_len = max(len(tag) for tag, _ in sorted_tags)
790 for tag, count in sorted_tags:
791 print(f"{_color_tag(tag):<{max_tag_len+2}} {count}")
792
793def print_monthly_tag_stats(filepath, top_n=20, dedup=True):
794 """Print top N tag statistics for each month, including months with no tags, and show total tag count. Consistent output."""
795 import re
796 from collections import defaultdict, Counter
797 month_tag_counter = defaultdict(Counter)
798 all_months = set()
799 try:
800 with open(filepath, "r", encoding="utf-8") as f:
801 content = f.read()
802 except Exception:
803 return
804 # Find all mdnote entries (with or without tags) to collect all months
805 pattern_month = r"\\subtree\[(\d{4}-\d{2})-\d{2}\]{\\mdnote{\d{4}-\d{2}-\d{2}}"
806 for match in re.finditer(pattern_month, content):
807 month = match.group(1)
808 all_months.add(month)
809 # Find all mdnote entries with tags
810 pattern_tags = r"\\subtree\[(\d{4}-\d{2})-\d{2}\]{\\mdnote{\d{4}-\d{2}-\d{2}}{\\tags{([^}]*)}"
811 for match in re.finditer(pattern_tags, content):
812 month = match.group(1)
813 tags = match.group(2)
814 tag_list = [t for t in tags.split() if t.startswith('#')]
815 for t in tag_list:
816 month_tag_counter[month][t] += 1
817 if all_months:
818 print("\nMonthly tag stats:")
819 for month in sorted(all_months):
820 top_tags = month_tag_counter[month].most_common(top_n)
821 unique_tag_count = len(month_tag_counter[month])
822 yyyymm = month.replace('-', '')
823 tag_str = ', '.join([
824 _color_tag(tag) if count == 1 else f"{_color_tag(tag)} {count}"
825 for tag, count in top_tags
826 ])
827 if unique_tag_count > top_n:
828 tag_str += ', ...'
829 print(f"{yyyymm} ({unique_tag_count} tags): {tag_str}")
830
831
832
833
834if __name__ == "__main__":
835 parser = argparse.ArgumentParser(description="TIL (Today I Learned) Title Improver")
836 parser.add_argument(
837 "--verbose",
838 action="store_true",
839 help="Show detailed logs for each tag extraction event"
840 )
841 parser.add_argument(
842 "--dedup",
843 action="store_true",
844 default=True,
845 help="Deduplicate explicit #tags from output (affects --stat, --stat-all, --merge-stat; default: deduplication ON)",
846 )
847 parser.add_argument(
848 "--no-dedup",
849 action="store_true",
850 help="Do NOT deduplicate explicit #tags (explicit tags will be included in all stats)",
851 )
852 parser.add_argument(
853 "--reset",
854 action="store_true",
855 help="Reset all titles to date-only (remove : title part)",
856 )
857 parser.add_argument(
858 "--test",
859 action="store_true",
860 help="Run test cases instead of processing files",
861 )
862 parser.add_argument(
863 "--stat",
864 nargs="?",
865 type=int,
866 const=20,
867 help="Show monthly tag stats (optionally top N tags per month)",
868 )
869 parser.add_argument(
870 "--stat-all",
871 action="store_true",
872 help="Show global tag stats (all tags, ordered by count)"
873 )
874 parser.add_argument(
875 "--merge-stat",
876 action="store_true",
877 help="Show aggregated keyword merge statistics (default: OFF)"
878 )
879 # AGENT-NOTE: --merge-stat controls merge stats output; default is OFF for clean output
880 parser.add_argument(
881 "--dry-run",
882 action="store_true",
883 help="Simulate changes without modifying any files (no reset or tag updates, but stats and analysis still run)",
884 )
885 args = parser.parse_args()
886
887 if args.test:
888 if test_til():
889 sys.exit(0)
890 else:
891 sys.exit(1)
892
893 filepath = Path("trees/uts-0018.tree")
894
895 if not filepath.exists():
896 print(f"Error: File {filepath} not found")
897 print("Expected file: trees/uts-0018.tree (learning diary)")
898 sys.exit(1)
899
900 if args.reset:
901 print("🔄 TIL Title Resetter - Removing all title keywords...")
902 success = reset_titles(filepath, dry_run=getattr(args, 'dry_run', False))
903 if success:
904 print("✨ Title reset complete!" if not getattr(args, 'dry_run', False) else "✨ [DRY-RUN] Title reset simulation complete!")
905 else:
906 print("❌ Title reset failed")
907 sys.exit(1)
908 else:
909 print("🔍 TIL Title Improver - Analyzing daily entries...")
910 success = process_file(filepath, verbose=getattr(args, 'verbose', False), dry_run=getattr(args, 'dry_run', False))
911 if success:
912 print("✨ Title improvement complete!" if not getattr(args, 'dry_run', False) else "✨ [DRY-RUN] Title improvement simulation complete!")
913 dedup = getattr(args, 'dedup', True) and not getattr(args, 'no_dedup', False)
914 if getattr(args, 'stat', None) is not None:
915 print_monthly_tag_stats(filepath, top_n=args.stat, dedup=dedup)
916 if getattr(args, 'stat_all', False):
917 print_global_tag_stats(filepath, dedup=dedup)
918 if getattr(args, 'merge_stat', False):
919 print_merge_stats(dedup=dedup, filepath=filepath)
920 else:
921 print("❌ Title improvement failed")
922 sys.exit(1)