utensil.tngl.sh
Utensil's Zettelkasten-style forest of evergreen notes on math and tech.
utensil.tngl.sh/forest/
1#!/usr/bin/env -S uv run
2# /// script
3# requires-python = ">=3.11,<3.12"
4# dependencies = ["bibtexparser"]
5# ///
6import json
7import os
8import pathlib
9import shutil
10import subprocess
11import sys
12from pathlib import Path
13import bibtexparser
14from bibtexparser.bwriter import BibTexWriter, SortingStrategy
15import re
16
17BIB_TEX_WRITER = BibTexWriter()
18BIB_TEX_WRITER.display_order = [
19 'title', 'author', 'year', 'date', 'isbn', 'doi', 'url', 'urldate', 'howpublished',
20 'journal', 'journaltitle', 'booktitle', 'edition', 'series',
21 'editor', 'volume', 'number', 'pages', 'publisher', 'institution', 'address'
22]
23BIB_TEX_WRITER.display_order_sorting = SortingStrategy.PRESERVE # ALPHABETICAL_ASC ALPHABETICAL_DESC
24
25
26# Get the script directory
27script_dir = Path(__file__).resolve().parent
28# Set the project root directory
29project_root = script_dir
30
31# Set the bib directory
32tex_dir = project_root / 'tex'
33bib_dir = project_root / 'trees' / 'refs'
34# Set the bib file name
35bib_filename = sys.argv[1] if len(sys.argv) > 1 else 'references'
36# create a directory to store the split files
37generated_dir = bib_dir / 'generated'
38os.makedirs(generated_dir, exist_ok=True)
39
40bib_file = pathlib.Path(tex_dir) / f'{bib_filename}.bib'
41csljson_file = generated_dir / f'{bib_filename}.json'
42
43print(f'📚 {bib_file.relative_to(project_root)} -> {csljson_file.relative_to(project_root)}')
44# Run the pandoc command
45subprocess.run(['pandoc', f'{bib_file}', '-s', '-f', 'biblatex', '-t', 'csljson', '-o', csljson_file], check=True)
46
47# Load the BibTeX file
48with open(bib_file, encoding='utf-8') as f:
49 bib_db = bibtexparser.load(f)
50
51csljson_file_name = csljson_file.stem
52
53with open(csljson_file, encoding='utf-8') as f:
54 references = json.load(f)
55
56TREE_TEMPLATE = r"""% {bib_filenames}
57\title{{{title}}}
58\date{{{date}}}
59{authors}
60\taxon{{reference}}
61{meta_doi}{meta_external}
62\meta{{bibtex}}{{\startverb
63{original_bibtex}\stopverb}}
64"""
65
66def format_author(author):
67 if 'literal' in author:
68 return author['literal']
69 elif 'given' in author and 'family' in author:
70 author_name = []
71 author_name.append(author['given'])
72 if 'dropping-particle' in author:
73 author_name.append(author['dropping-particle'])
74 author_name.append(author['family'])
75 return ' '.join(author_name)
76 else:
77 return ''
78
79# format a number with leading zeros
80def format_number(number, length=2):
81 format_string = "{:0" + str(length) + "}"
82 return format_string.format(number)
83
84def format_date(date_parts):
85 return '-'.join([format_number(part) for part in date_parts])
86
87def format_doi(reference):
88 doi = reference.get('DOI', None)
89 return f'\\meta{{doi}}{{{doi}}}\n' if doi else ''
90
91def format_external(reference):
92 url = reference.get('URL', None)
93 if url is None:
94 publisher = reference.get('publisher', None)
95 # if publisher is a URL, use regex
96 if publisher is not None:
97 url = re.search('(https?://[^\s]+)', publisher)
98 if url is not None:
99 url = url.group(1)
100 print(f'🔵 {url}')
101
102 return f'\\meta{{external}}{{{url}}}\n' if url else ''
103
104def parse_frontmatter(line):
105 # strip beginning whitespace and %
106 line = line.lstrip().lstrip('%').lstrip()
107 # try parse it as JSON
108 try:
109 ret = json.loads(line)
110 # check if it's an array
111 if isinstance(ret, list):
112 return ret
113 else:
114 return []
115 except json.JSONDecodeError:
116 return []
117
118# print(f'📚 Splitting {csljson_file.relative_to(project_root)}')
119csl_file = bib_dir / 'forest.csl'
120for i, reference in enumerate(references):
121 citekey = reference['id']
122 csljson_file_i = generated_dir / f'{citekey}.json'
123 tree_file_i = bib_dir / f'{citekey}.tree'
124 print(f' {csljson_file_i.relative_to(project_root)} -> {tree_file_i.relative_to(project_root)}')
125
126 bibtex_entry = bib_db.entries_dict[citekey]
127 entrydb = bibtexparser.bibdatabase.BibDatabase()
128 entrydb.entries = [bibtex_entry]
129 original_bibtex = bibtexparser.dumps(entrydb, BIB_TEX_WRITER)
130
131 with open(csljson_file_i, 'w', encoding='utf-8') as f:
132 # reference['title'] = reference.get('title', '').replace('\n', ' ')
133 # reference['title_short'] = reference.get('title', '').split(' ')[0].lower()
134
135 # print(original_bibtex)
136 reference['original_bibtex'] = original_bibtex
137 json.dump([reference], f)
138 f.flush()
139
140 bib_filenames_i = [bib_filename]
141
142 # if tree_file_i exists
143 if os.path.exists(tree_file_i):
144 # read the first line
145 with open(tree_file_i, 'r', encoding='utf-8') as f:
146 first_line = f.readline()
147 first_line_json = parse_frontmatter(first_line)
148 # remove bib_filename from it
149 bib_filenames_i = [
150 filename for filename in first_line_json
151 if filename != bib_filename
152 ]
153 # add bib_filename to the end of it
154 bib_filenames_i.append(bib_filename)
155 # sort the list
156 bib_filenames_i.sort()
157
158 # detect duplication
159 if len(bib_filenames_i) > 1:
160 # {tree_file_i.relative_to(project_root)}:
161 print(f'🟡 {bib_filenames_i}')
162
163 formatted = TREE_TEMPLATE.format(
164 bib_filenames=json.dumps(bib_filenames_i),
165 title=reference['title'],
166 date=format_date(reference['issued']['date-parts'][0]),
167 meta_doi=format_doi(reference),
168 meta_external=format_external(reference),
169 authors=''.join([f'\\author{{{format_author(author)}}}' for author in reference['author']]),
170 original_bibtex=original_bibtex)
171
172 if tree_file_i.exists():
173 with open(tree_file_i, 'r', encoding='utf-8') as f:
174 existing = f.read()
175 # get the first line
176 existing_first_line, existing_rest = existing.split('\n', 1)
177 existing_bib_filenames_i = parse_frontmatter(existing_first_line)
178 # if it's the same or shorter
179 if len(existing_bib_filenames_i) >= len(bib_filenames_i) or set(existing_bib_filenames_i) == set(bib_filenames_i):
180 formatted_first_line, formatted_rest = formatted.split('\n', 1)
181 if existing_rest == formatted_rest:
182 continue
183 # if manually modified, skip
184 if 'manual' in existing_bib_filenames_i:
185 print(f"🦘Skipping {tree_file_i.relative_to(project_root)} as it's manually modified")
186 continue
187
188 with open(tree_file_i, 'w', encoding='utf-8') as f:
189 f.write(formatted)
190 f.flush()