utensil.tngl.sh
Utensil's Zettelkasten-style forest of evergreen notes on math and tech.
utensil.tngl.sh/forest/
1#!/usr/bin/env -S uv run
2# /// script
3# requires-python = ">=3.11,<3.12"
4# dependencies = ["lxml>=5.2.0"]
5# ///
6"""
7XML to HTML Incremental Build Script
8===================================
9
10This script incrementally converts XML files in the output/ directory to HTML using XSLT (output/uts-forest.xsl).
11It is designed to be idempotent, robust, and fast, supporting parallel execution and accurate dependency checking.
12
13Key Features:
14- Only rebuilds HTML files if the corresponding XML or the XSL file is newer, or if the HTML is missing.
15- Uses Python's concurrent.futures for parallelism, maximizing CPU utilization while avoiding overload.
16- Reports progress and the number of files actually updated.
17- Handles errors gracefully, printing clear error messages but continuing the build.
18- Designed for integration with build systems (e.g., mise, just) and for deterministic, reproducible builds.
19
20Usage:
21 uv run convert_xml_to_html.py
22
23Extension Points:
24- To support additional output formats or XSLT parameters, extend the convert_one() function.
25- To change the input/output directories, modify OUTPUT_DIR and XSL_PATH.
26- For CLI options, add argument parsing to main().
27
28AGENT-NOTE: CRITICAL FEATURES TO MAINTAIN
291. IDEMPOTENT: Multiple runs must produce identical results
302. ERROR HANDLING: Graceful degradation for missing data
313. BUILD INTEGRATION: Validates syntax after processing
324. DETERMINISTIC: Sorted processing ensures consistent output
33"""
34
35import os
36import sys
37import pathlib
38import concurrent.futures
39import multiprocessing
40from lxml import etree
41from typing import List
42
43# AGENT-NOTE: perf-hot-path; parallel conversion, progress bar, and update count must be robust
44#
45# This script is called by mise-tasks/build/xml_to_html for incremental HTML builds.
46# It is safe to run repeatedly and will only update files as needed.
47
48OUTPUT_DIR = pathlib.Path("output")
49XSL_PATH = OUTPUT_DIR / "uts-forest.xsl"
50BAK_DIR = OUTPUT_DIR / ".bak"
51
52
53def get_max_workers() -> int:
54 """
55 Determine the number of worker threads to use for parallel processing.
56 Leaves 2 CPUs free to avoid overloading the system.
57 Returns:
58 int: Number of worker threads to use (minimum 2).
59 """
60 try:
61 cpu_count = multiprocessing.cpu_count()
62 return max(2, cpu_count - 2)
63 except Exception:
64 return 2
65
66
67def get_xml_files() -> List[pathlib.Path]:
68 """
69 Find all XML files in the output directory.
70 Returns:
71 List[pathlib.Path]: Sorted list of XML file paths.
72 """
73 return sorted(OUTPUT_DIR.glob("*.xml"))
74
75
76def needs_rebuild(xml_path: pathlib.Path, html_path: pathlib.Path, xsl_path: pathlib.Path) -> bool:
77 """
78 Determine if the HTML file needs to be rebuilt.
79 Args:
80 xml_path (pathlib.Path): Path to the XML source file.
81 html_path (pathlib.Path): Path to the HTML output file.
82 xsl_path (pathlib.Path): Path to the XSLT stylesheet.
83 Returns:
84 bool: True if HTML should be rebuilt, False otherwise.
85 """
86 if not html_path.exists():
87 # print(f"[DEBUG] {html_path} does not exist; will rebuild.")
88 return True
89 if xml_path.stat().st_mtime > html_path.stat().st_mtime:
90 # print(f"[DEBUG] {xml_path} is newer than {html_path}; will rebuild.")
91 return True
92 if xsl_path.exists() and xsl_path.stat().st_mtime > html_path.stat().st_mtime:
93 # print(f"[DEBUG] {xsl_path} is newer than {html_path}; will rebuild.")
94 return True
95 return False
96
97
98def convert_one(xml_path: pathlib.Path, xsl_path: pathlib.Path, html_path: pathlib.Path, bak_dir: pathlib.Path) -> bool:
99 """
100 Convert a single XML file to HTML using the provided XSLT stylesheet.
101 Only overwrite the HTML file if the output differs from the backup.
102 Args:
103 xml_path (pathlib.Path): Path to the XML source file.
104 xsl_path (pathlib.Path): Path to the XSLT stylesheet.
105 html_path (pathlib.Path): Path to the HTML output file.
106 bak_dir (pathlib.Path): Path to the backup directory.
107 Returns:
108 bool: True if HTML was updated (content changed), False otherwise.
109 """
110 try:
111 # Parse XML and XSLT files
112 dom = etree.parse(str(xml_path))
113 xslt = etree.parse(str(xsl_path))
114 transform = etree.XSLT(xslt)
115 # Apply the XSLT transformation
116 newdom = transform(dom)
117 html_path.parent.mkdir(parents=True, exist_ok=True)
118 new_html = str(newdom)
119 bak_path = bak_dir / html_path.name
120 # Compare with backup if it exists
121 if bak_path.exists():
122 old_html = bak_path.read_text(encoding="utf-8")
123 if old_html == new_html:
124 return False # No change
125 # Write the HTML output
126 html_path.write_text(new_html, encoding="utf-8")
127 return True
128 except Exception as e:
129 # Print error but do not stop the build
130 print(f"[ERROR] Failed to convert {xml_path} -> {html_path}: {e}", file=sys.stderr)
131 return False
132
133
134def main():
135 """
136 Main entry point for the incremental XML-to-HTML build process.
137 - Scans for XML files and checks for the XSLT stylesheet.
138 - Uses a thread pool to process files in parallel.
139 - Only rebuilds HTML files when necessary and content changes.
140 - Prints progress and a summary of updated files.
141 """
142 xml_files = get_xml_files()
143 if not xml_files:
144 print("No XML files found in output/.")
145 sys.exit(0)
146 if not XSL_PATH.exists():
147 print(f"XSL file not found: {XSL_PATH}", file=sys.stderr)
148 sys.exit(1)
149
150 max_workers = get_max_workers()
151 print(f"Max jobs: {max_workers}")
152 updated_count = 0
153 total_files = len(xml_files)
154 progress = 0
155 progress_step = max(1, total_files // 20)
156 print("Progress: ", end="", flush=True)
157
158 def process(xml_path: pathlib.Path) -> bool:
159 """
160 Worker function for a single XML file.
161 Returns True if the file was rebuilt (HTML content changed), False otherwise.
162 """
163 basename = xml_path.stem
164 html_path = OUTPUT_DIR / f"{basename}.html"
165 bak_path = BAK_DIR / html_path.name
166 try:
167 # If backup is missing, always rebuild (skip needs_rebuild)
168 if not bak_path.exists():
169 dom = etree.parse(str(xml_path))
170 xslt = etree.parse(str(XSL_PATH))
171 transform = etree.XSLT(xslt)
172 new_html = str(transform(dom))
173 html_path.parent.mkdir(parents=True, exist_ok=True)
174 html_path.write_text(new_html, encoding="utf-8")
175 return True
176 # If backup exists, only rebuild if needed and content differs
177 if not needs_rebuild(xml_path, html_path, XSL_PATH):
178 return False
179 dom = etree.parse(str(xml_path))
180 xslt = etree.parse(str(XSL_PATH))
181 transform = etree.XSLT(xslt)
182 new_html = str(transform(dom))
183 old_html = bak_path.read_text(encoding="utf-8")
184 if old_html == new_html:
185 return False
186 html_path.parent.mkdir(parents=True, exist_ok=True)
187 html_path.write_text(new_html, encoding="utf-8")
188 return True
189 except Exception as e:
190 print(f"[ERROR] Failed to convert {xml_path} -> {html_path}: {e}", file=sys.stderr)
191 return False
192
193 # Use ThreadPoolExecutor for parallel processing of XML files
194 with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
195 futures = {executor.submit(process, xml_path): i for i, xml_path in enumerate(xml_files)}
196 for i, future in enumerate(concurrent.futures.as_completed(futures)):
197 try:
198 if future.result():
199 updated_count += 1
200 except Exception as e:
201 print(f"[ERROR] Exception in worker: {e}", file=sys.stderr)
202 # AGENT-NOTE: progress bar logic
203 if (i + 1) % progress_step == 0 or (i + 1) == total_files:
204 print("█", end="", flush=True)
205 print()
206 print(f"📝 Updated {updated_count} HTML file(s) out of {total_files}")
207
208if __name__ == "__main__":
209 # Entry point for script execution
210 main()