feat(import): deterministic timestamp resolution before model detection · solpbc.org/solstone@92cf447

+24 -2

docs/openapi/convey-clients.json

··· 1293 1293 "schema": { 1294 1294 "additionalProperties": true, 1295 1295 "properties": { 1296 + "deterministic_only": { 1297 + "type": "boolean" 1298 + }, 1296 1299 "facet": { 1297 1300 "type": "string" 1298 1301 }, ··· 1327 1330 "facet": null, 1328 1331 "path": "/journal/imports/20260618_143022/source.txt", 1329 1332 "setting": null, 1330 - "timestamp": "20260618_143022" 1333 + "timestamp": "20260618_143022", 1334 + "timestamp_detection_method": "deterministic", 1335 + "timestamp_detection_model_called": false, 1336 + "timestamp_detection_no_match_reason": null 1331 1337 }, 1332 1338 "schema": { 1333 1339 "additionalProperties": true, ··· 1349 1355 }, 1350 1356 "timestamp": { 1351 1357 "type": "string" 1358 + }, 1359 + "timestamp_detection_method": { 1360 + "description": "Timestamp detection method: deterministic, model, upload_fallback, or explicit.", 1361 + "type": "string" 1362 + }, 1363 + "timestamp_detection_model_called": { 1364 + "type": "boolean" 1365 + }, 1366 + "timestamp_detection_no_match_reason": { 1367 + "type": [ 1368 + "string", 1369 + "null" 1370 + ] 1352 1371 } 1353 1372 }, 1354 1373 "required": [ 1355 1374 "path", 1356 1375 "timestamp", 1357 1376 "facet", 1358 - "setting" 1377 + "setting", 1378 + "timestamp_detection_method", 1379 + "timestamp_detection_model_called", 1380 + "timestamp_detection_no_match_reason" 1359 1381 ], 1360 1382 "type": "object" 1361 1383 }

+24

solstone/apps/import/contract.py

View file

··· 46 46 FieldSpec("setting", "string"), 47 47 FieldSpec("imported_via", "string"), 48 48 FieldSpec("observer_handle", "string"), 49 + FieldSpec("deterministic_only", "boolean"), 49 50 ), 50 51 description="Multipart body with either file or text.", 51 52 ), ··· 68 69 required=True, 69 70 raw_schema=_NULLABLE_STRING, 70 71 ), 72 + FieldSpec( 73 + "timestamp_detection_method", 74 + "string", 75 + required=True, 76 + description=( 77 + "Timestamp detection method: deterministic, model, " 78 + "upload_fallback, or explicit." 79 + ), 80 + ), 81 + FieldSpec( 82 + "timestamp_detection_model_called", 83 + "boolean", 84 + required=True, 85 + ), 86 + FieldSpec( 87 + "timestamp_detection_no_match_reason", 88 + "string", 89 + required=True, 90 + raw_schema=_NULLABLE_STRING, 91 + ), 71 92 ), 72 93 example={ 73 94 "path": "/journal/imports/20260618_143022/source.txt", 74 95 "timestamp": "20260618_143022", 75 96 "facet": None, 76 97 "setting": None, 98 + "timestamp_detection_method": "deterministic", 99 + "timestamp_detection_model_called": False, 100 + "timestamp_detection_no_match_reason": None, 77 101 }, 78 102 ), 79 103 _json_error(

+56 -12

solstone/apps/import/routes.py

View file

··· 31 31 respond_collection, 32 32 success_response, 33 33 ) 34 - from solstone.think.detect_created import detect_created 34 + from solstone.think.detect_created import detect_created, resolve_created_deterministic 35 35 from solstone.think.importers.utils import ( 36 36 build_import_info, 37 37 generate_content_manifest, ··· 218 218 ) 219 219 220 220 221 + def _form_bool(value: str | None) -> bool: 222 + return value.strip().lower() in {"true", "1", "yes"} if value else False 223 + 224 + 221 225 @import_bp.route("/api/save", methods=["POST"]) 222 226 def import_save() -> Any: 223 227 from datetime import datetime ··· 226 230 text = request.form.get("text", "").strip() 227 231 facet = request.form.get("facet", "").strip() or None 228 232 setting = request.form.get("setting", "").strip() or None 233 + deterministic_only = _form_bool(request.form.get("deterministic_only")) 229 234 230 235 # Generate timestamp for folder name 231 236 timestamp_ms = now_ms() ··· 241 246 # Detect timestamp from content first (need temporary save for detection) 242 247 ts = None 243 248 detection_result = None 249 + timestamp_detection_method = "upload_fallback" 250 + timestamp_detection_model_called = False 251 + timestamp_detection_no_match_reason = None 244 252 245 253 # Create temporary file for detection if needed 246 254 if upload: ··· 264 272 temp_path = tmp.name 265 273 266 274 try: 267 - # Pass original filename for better timestamp detection 268 - original_name = upload.filename if upload else None 269 - detection_result = detect_created(temp_path, original_filename=original_name) 270 - if ( 271 - detection_result 272 - and detection_result.get("day") 273 - and detection_result.get("time") 274 - ): 275 - ts = f"{detection_result['day']}_{detection_result['time']}" 276 - except Exception: 277 - ts = None 275 + try: 276 + original_name = upload.filename if upload else None 277 + detection_result = resolve_created_deterministic( 278 + temp_path, 279 + original_filename=original_name, 280 + ) 281 + if ( 282 + detection_result 283 + and detection_result.get("day") 284 + and detection_result.get("time") 285 + ): 286 + ts = f"{detection_result['day']}_{detection_result['time']}" 287 + timestamp_detection_method = "deterministic" 288 + except Exception: 289 + detection_result = None 290 + 291 + if not ts: 292 + if deterministic_only: 293 + timestamp_detection_no_match_reason = "no_deterministic_match" 294 + else: 295 + try: 296 + # Pass original filename for better timestamp detection 297 + original_name = upload.filename if upload else None 298 + detection_result = detect_created( 299 + temp_path, 300 + original_filename=original_name, 301 + ) 302 + timestamp_detection_model_called = True 303 + if ( 304 + detection_result 305 + and detection_result.get("day") 306 + and detection_result.get("time") 307 + ): 308 + ts = f"{detection_result['day']}_{detection_result['time']}" 309 + timestamp_detection_method = "model" 310 + else: 311 + timestamp_detection_no_match_reason = "model_no_match" 312 + except Exception: 313 + detection_result = None 314 + timestamp_detection_model_called = True 315 + timestamp_detection_no_match_reason = "model_no_match" 278 316 finally: 279 317 # Clean up temporary file 280 318 Path(temp_path).unlink(missing_ok=True) ··· 320 358 "detection_result": detection_result, 321 359 "detected_timestamp": ts, 322 360 "user_timestamp": folder_timestamp, # The timestamp used for the folder 361 + "timestamp_detection_method": timestamp_detection_method, 362 + "timestamp_detection_model_called": timestamp_detection_model_called, 363 + "timestamp_detection_no_match_reason": timestamp_detection_no_match_reason, 323 364 "file_size": file_path.stat().st_size if file_path.exists() else 0, 324 365 "mime_type": upload.content_type if upload else "text/plain", 325 366 "facet": facet, # Include selected facet ··· 358 399 "timestamp": folder_timestamp, 359 400 "facet": facet, 360 401 "setting": setting, 402 + "timestamp_detection_method": timestamp_detection_method, 403 + "timestamp_detection_model_called": timestamp_detection_model_called, 404 + "timestamp_detection_no_match_reason": timestamp_detection_no_match_reason, 361 405 } 362 406 if dedup: 363 407 result["dedup"] = dedup

+187

solstone/think/detect_created.py

View file

··· 6 6 from __future__ import annotations 7 7 8 8 import json 9 + import re 9 10 import subprocess 10 11 from datetime import datetime, timezone 11 12 from pathlib import Path ··· 13 14 14 15 from .prompts import load_prompt 15 16 17 + DETERMINISTIC_SOURCE_FILENAME = "filename_local" 18 + DETERMINISTIC_SOURCE_FILENAME_UTC = "filename_utc" 19 + DETERMINISTIC_SOURCE_METADATA_LOCAL = "metadata_local" 20 + DETERMINISTIC_SOURCE_METADATA_UTC = "metadata_utc" 21 + 22 + _LIMITLESS_FILENAME_RE = re.compile( 23 + r"^limitless_pendant_(\d{4})-(\d{2})-(\d{2})T" 24 + r"(\d{2})-(\d{2})-(\d{2})_to_.*$" 25 + ) 26 + _LOCAL_FILENAME_RES = ( 27 + re.compile(r"^(\d{4})-(\d{2})-(\d{2})_(\d{2})_(\d{2})_(\d{2})(?!\d)"), 28 + re.compile(r"^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})(?!\d)"), 29 + re.compile(r"^(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})(?!\d)"), 30 + re.compile(r"^(\d{4})(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2})(?!\d)"), 31 + re.compile(r"^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})(?!\d)"), 32 + ) 33 + _METADATA_CREATION_FIELDS = ( 34 + "SubSecCreateDate", 35 + "SubSecDateTimeOriginal", 36 + "CreateDate", 37 + "CreationDate", 38 + "DateTimeOriginal", 39 + "MediaCreateDate", 40 + "TrackCreateDate", 41 + "ContentCreateDate", 42 + ) 43 + _OFFSET_RE = re.compile(r"(Z|[+-]\d{2}:\d{2})$") 44 + _SUBSECOND_RE = re.compile(r"(\d{2}:\d{2}:\d{2})\.\d+") 45 + 16 46 _SCHEMA = json.loads( 17 47 (Path(__file__).parent / "detect_created.schema.json").read_text(encoding="utf-8") 18 48 ) ··· 35 65 return proc.stdout 36 66 except Exception as exc: # pragma: no cover - exiftool optional 37 67 return f"Error extracting metadata: {exc}" 68 + 69 + 70 + def _extract_metadata_json(path: str) -> dict: 71 + """Return JSON metadata for *path* using exiftool if available.""" 72 + cmd = [ 73 + "exiftool", 74 + "-json", 75 + path, 76 + ] 77 + try: 78 + proc = subprocess.run(cmd, capture_output=True, text=True, check=True) 79 + metadata = json.loads(proc.stdout) 80 + except Exception: # pragma: no cover - exiftool optional 81 + return {} 82 + if not isinstance(metadata, list) or not metadata: 83 + return {} 84 + first = metadata[0] 85 + return first if isinstance(first, dict) else {} 86 + 87 + 88 + def _result_from_datetime( 89 + value: datetime, 90 + *, 91 + source: str, 92 + utc: bool, 93 + ) -> dict: 94 + return { 95 + "day": value.strftime("%Y%m%d"), 96 + "time": value.strftime("%H%M%S"), 97 + "confidence": "high", 98 + "source": source, 99 + "utc": utc, 100 + } 101 + 102 + 103 + def _parse_datetime_parts(parts: tuple[str, ...]) -> datetime | None: 104 + try: 105 + return datetime(*(int(part) for part in parts)) 106 + except ValueError: 107 + return None 108 + 109 + 110 + def _filename_stem(path: str, original_filename: Optional[str]) -> str: 111 + name = original_filename if original_filename else path 112 + return Path(Path(name).name).stem 113 + 114 + 115 + def _resolve_limitless_filename(stem: str) -> dict | None: 116 + match = _LIMITLESS_FILENAME_RE.match(stem) 117 + if match is None: 118 + return None 119 + parsed = _parse_datetime_parts(match.groups()) 120 + if parsed is None: 121 + return None 122 + utc_dt = parsed.replace(tzinfo=timezone.utc) 123 + return _result_from_datetime( 124 + utc_dt.astimezone(), 125 + source=DETERMINISTIC_SOURCE_FILENAME_UTC, 126 + utc=True, 127 + ) 128 + 129 + 130 + def _resolve_local_filename(stem: str) -> dict | None: 131 + for pattern in _LOCAL_FILENAME_RES: 132 + match = pattern.match(stem) 133 + if match is None: 134 + continue 135 + parsed = _parse_datetime_parts(match.groups()) 136 + if parsed is None: 137 + return None 138 + return _result_from_datetime( 139 + parsed, 140 + source=DETERMINISTIC_SOURCE_FILENAME, 141 + utc=False, 142 + ) 143 + return None 144 + 145 + 146 + def _normalize_metadata_datetime(value: object) -> tuple[str, str, bool] | None: 147 + if not isinstance(value, str): 148 + return None 149 + raw = value.strip() 150 + if not raw or raw.startswith("0000:") or raw.startswith("0000-"): 151 + return None 152 + normalized = raw.replace("T", " ") 153 + normalized = _SUBSECOND_RE.sub(r"\1", normalized) 154 + if len(normalized) >= 10 and normalized[4] == ":" and normalized[7] == ":": 155 + normalized = f"{normalized[:4]}-{normalized[5:7]}-{normalized[8:]}" 156 + normalized = normalized.replace(" ", "T", 1) 157 + 158 + offset_bearing = bool(_OFFSET_RE.search(normalized)) 159 + if normalized.endswith("Z"): 160 + normalized = f"{normalized[:-1]}+00:00" 161 + 162 + try: 163 + parsed = datetime.fromisoformat(normalized) 164 + except ValueError: 165 + return None 166 + if parsed.year == 0: 167 + return None 168 + if offset_bearing: 169 + parsed = parsed.astimezone() 170 + return parsed.strftime("%Y%m%d"), parsed.strftime("%H%M%S"), offset_bearing 171 + 172 + 173 + def _resolve_metadata(path: str) -> dict | None: 174 + try: 175 + metadata = _extract_metadata_json(path) 176 + except Exception: 177 + return None 178 + pairs: set[tuple[str, str]] = set() 179 + saw_offset = False 180 + for field in _METADATA_CREATION_FIELDS: 181 + parsed = _normalize_metadata_datetime(metadata.get(field)) 182 + if parsed is None: 183 + continue 184 + day, time, offset_bearing = parsed 185 + pairs.add((day, time)) 186 + saw_offset = saw_offset or offset_bearing 187 + if len(pairs) != 1: 188 + return None 189 + day, time = next(iter(pairs)) 190 + return { 191 + "day": day, 192 + "time": time, 193 + "confidence": "high", 194 + "source": ( 195 + DETERMINISTIC_SOURCE_METADATA_UTC 196 + if saw_offset 197 + else DETERMINISTIC_SOURCE_METADATA_LOCAL 198 + ), 199 + "utc": saw_offset, 200 + } 201 + 202 + 203 + def resolve_created_deterministic( 204 + path: str, original_filename: Optional[str] = None 205 + ) -> Optional[dict]: 206 + """Return deterministic creation time information for *path* when unambiguous. 207 + 208 + Direct-source timestamps, such as Plaud recording start times, bypass this resolver 209 + by passing an explicit timestamp into the importer. 210 + """ 211 + stem = _filename_stem(path, original_filename) 212 + limitless_candidate = _resolve_limitless_filename(stem) 213 + if limitless_candidate is not None: 214 + return limitless_candidate 215 + 216 + filename_candidate = _resolve_local_filename(stem) 217 + metadata_candidate = _resolve_metadata(path) 218 + if filename_candidate is not None and metadata_candidate is not None: 219 + filename_pair = (filename_candidate["day"], filename_candidate["time"]) 220 + metadata_pair = (metadata_candidate["day"], metadata_candidate["time"]) 221 + if filename_pair != metadata_pair: 222 + return None 223 + return filename_candidate 224 + return filename_candidate or metadata_candidate 38 225 39 226 40 227 def detect_created(

+8

solstone/think/import_client.py

View file

··· 30 30 "--source": "http-client", 31 31 "--force": "http-client", 32 32 "--auto": "http-client", 33 + "--deterministic-only": "http-client", 33 34 "--dry-run": "reject-journal-host", 34 35 "--json": "client-output", 35 36 "-v/--verbose": "client-logging", ··· 69 70 const=True, 70 71 default=None, 71 72 help="Accept the server-detected timestamp", 73 + ) 74 + parser.add_argument( 75 + "--deterministic-only", 76 + action="store_true", 77 + help="Use only deterministic timestamp detection; skip model detection", 72 78 ) 73 79 parser.add_argument( 74 80 "--dry-run", ··· 169 175 }.items() 170 176 if value is not None 171 177 } 178 + if args.deterministic_only: 179 + data["deterministic_only"] = "true" 172 180 if media_path.exists() and media_path.is_file(): 173 181 return client.upload( 174 182 f"{IMPORT_API}/save",

+32 -5

solstone/think/importers/cli.py

View file

··· 14 14 from typing import Any 15 15 16 16 from solstone.think.callosum import CallosumConnection 17 - from solstone.think.detect_created import detect_created 17 + from solstone.think.detect_created import detect_created, resolve_created_deterministic 18 18 from solstone.think.importers.audio import _get_audio_duration, prepare_audio_segments 19 19 from solstone.think.importers.shared import ( 20 20 _get_relative_path, ··· 375 375 json_output: bool = False, 376 376 verbose: bool = False, 377 377 wait_for_processing: bool = True, 378 + deterministic_only: bool = False, 378 379 ) -> dict[str, Any] | None: 379 380 """When False, returns after segment creation without awaiting transcription completion; 380 381 failed_segments is omitted from the result and created_segments is the durable ··· 392 393 json=json_output, 393 394 verbose=verbose, 394 395 wait_for_processing=wait_for_processing, 396 + deterministic_only=deterministic_only, 395 397 ) 396 398 return _import_one_from_args(args) 397 399 ··· 474 476 # File importers don't need an external timestamp — auto-generate for metadata 475 477 args.timestamp = dt.datetime.now().strftime("%Y%m%d_%H%M%S") 476 478 elif not args.timestamp: 477 - # If no timestamp provided, detect it 478 - # Pass the original filename for better detection 479 - detection_result = detect_created( 479 + detection_result = resolve_created_deterministic( 480 480 args.media, 481 481 original_filename=os.path.basename(args.media), 482 - guidance=args.auto if isinstance(args.auto, str) else None, 483 482 ) 483 + if ( 484 + not detection_result 485 + or not detection_result.get("day") 486 + or not detection_result.get("time") 487 + ): 488 + if args.deterministic_only: 489 + print( 490 + "No deterministic timestamp found. Provide --timestamp " 491 + "YYYYMMDD_HHMMSS, or omit --deterministic-only to use model " 492 + "detection." 493 + ) 494 + return { 495 + "skipped": True, 496 + "reason": "no_deterministic_match", 497 + } 498 + # If no deterministic timestamp exists, fall back to model detection. 499 + # Pass the original filename for better detection. 500 + detection_result = detect_created( 501 + args.media, 502 + original_filename=os.path.basename(args.media), 503 + guidance=args.auto if isinstance(args.auto, str) else None, 504 + ) 484 505 if ( 485 506 detection_result 486 507 and detection_result.get("day") ··· 1346 1367 help="Show what would be imported without writing to the journal", 1347 1368 ) 1348 1369 parser.add_argument( 1370 + "--deterministic-only", 1371 + action="store_true", 1372 + help="Use only deterministic timestamp detection; skip model detection", 1373 + ) 1374 + parser.add_argument( 1349 1375 "--backends", 1350 1376 action="store_true", 1351 1377 help="List syncable importer backends", ··· 1470 1496 dry_run=args.dry_run, 1471 1497 json_output=args.json, 1472 1498 verbose=args.verbose, 1499 + deterministic_only=args.deterministic_only, 1473 1500 ) 1474 1501 except Exception as exc: 1475 1502 raise SystemExit(str(exc)) from exc

+227

tests/test_detect_created_deterministic.py

View file

··· 1 + # SPDX-License-Identifier: AGPL-3.0-only 2 + # Copyright (c) 2026 sol pbc 3 + 4 + import importlib 5 + import os 6 + import time 7 + 8 + import pytest 9 + 10 + detect_created_mod = importlib.import_module("solstone.think.detect_created") 11 + 12 + 13 + @pytest.fixture 14 + def tz_los_angeles(monkeypatch): 15 + original_tz = os.environ.get("TZ") 16 + monkeypatch.setenv("TZ", "America/Los_Angeles") 17 + time.tzset() 18 + yield 19 + if original_tz is None: 20 + os.environ.pop("TZ", None) 21 + else: 22 + os.environ["TZ"] = original_tz 23 + time.tzset() 24 + 25 + 26 + @pytest.mark.parametrize( 27 + "filename", 28 + [ 29 + "2024-01-15_10_30_00_copy.m4a", 30 + "2024-01-15 10:30:00.wav", 31 + "2024-01-15_10-30-00.mp3", 32 + "20240115_103000_2.m4a", 33 + "20240115103000.mov", 34 + ], 35 + ) 36 + def test_filename_local_formats_and_suffixes(monkeypatch, filename): 37 + monkeypatch.setattr(detect_created_mod, "_extract_metadata_json", lambda path: {}) 38 + 39 + result = detect_created_mod.resolve_created_deterministic( 40 + "/tmp/source", 41 + original_filename=filename, 42 + ) 43 + 44 + assert result == { 45 + "day": "20240115", 46 + "time": "103000", 47 + "confidence": "high", 48 + "source": detect_created_mod.DETERMINISTIC_SOURCE_FILENAME, 49 + "utc": False, 50 + } 51 + 52 + 53 + @pytest.mark.parametrize( 54 + "filename", 55 + [ 56 + "2024-01-15.m4a", 57 + "01-15.m4a", 58 + "2024-13-15_10_30_00.m4a", 59 + "2024-02-30_10_30_00.m4a", 60 + "2024-01-15_25_30_00.m4a", 61 + ], 62 + ) 63 + def test_filename_non_matches_and_invalid_values(monkeypatch, filename): 64 + monkeypatch.setattr(detect_created_mod, "_extract_metadata_json", lambda path: {}) 65 + 66 + assert ( 67 + detect_created_mod.resolve_created_deterministic( 68 + "/tmp/source", 69 + original_filename=filename, 70 + ) 71 + is None 72 + ) 73 + 74 + 75 + def test_limitless_filename_utc_converts_to_local(monkeypatch, tz_los_angeles): 76 + def fail_metadata(path): 77 + raise AssertionError("metadata should not be read for limitless filenames") 78 + 79 + monkeypatch.setattr(detect_created_mod, "_extract_metadata_json", fail_metadata) 80 + 81 + result = detect_created_mod.resolve_created_deterministic( 82 + "/tmp/source", 83 + original_filename=( 84 + "limitless_pendant_2024-01-15T18-30-00_to_2024-01-15T18-45-00.m4a" 85 + ), 86 + ) 87 + 88 + assert result == { 89 + "day": "20240115", 90 + "time": "103000", 91 + "confidence": "high", 92 + "source": detect_created_mod.DETERMINISTIC_SOURCE_FILENAME_UTC, 93 + "utc": True, 94 + } 95 + 96 + 97 + def test_limitless_rule_is_source_scoped(monkeypatch): 98 + monkeypatch.setattr(detect_created_mod, "_extract_metadata_json", lambda path: {}) 99 + 100 + result = detect_created_mod.resolve_created_deterministic( 101 + "/tmp/source", 102 + original_filename="other_2024-01-15T18-30-00_to_2024-01-15T18-45-00.m4a", 103 + ) 104 + 105 + assert result is None 106 + 107 + 108 + def test_metadata_local_unambiguous(monkeypatch): 109 + monkeypatch.setattr( 110 + detect_created_mod, 111 + "_extract_metadata_json", 112 + lambda path: { 113 + "CreateDate": "2024:01:15 10:30:00", 114 + "DateTimeOriginal": "2024:01:15 10:30:00", 115 + }, 116 + ) 117 + 118 + result = detect_created_mod.resolve_created_deterministic( 119 + "/tmp/source", 120 + original_filename="voice.m4a", 121 + ) 122 + 123 + assert result == { 124 + "day": "20240115", 125 + "time": "103000", 126 + "confidence": "high", 127 + "source": detect_created_mod.DETERMINISTIC_SOURCE_METADATA_LOCAL, 128 + "utc": False, 129 + } 130 + 131 + 132 + def test_metadata_utc_offset_unambiguous(monkeypatch, tz_los_angeles): 133 + monkeypatch.setattr( 134 + detect_created_mod, 135 + "_extract_metadata_json", 136 + lambda path: {"CreationDate": "2024:01:15 18:30:00+00:00"}, 137 + ) 138 + 139 + result = detect_created_mod.resolve_created_deterministic( 140 + "/tmp/source", 141 + original_filename="voice.m4a", 142 + ) 143 + 144 + assert result == { 145 + "day": "20240115", 146 + "time": "103000", 147 + "confidence": "high", 148 + "source": detect_created_mod.DETERMINISTIC_SOURCE_METADATA_UTC, 149 + "utc": True, 150 + } 151 + 152 + 153 + def test_metadata_ambiguous_voice_memo_falls_through(monkeypatch, tz_los_angeles): 154 + monkeypatch.setattr( 155 + detect_created_mod, 156 + "_extract_metadata_json", 157 + lambda path: { 158 + "CreationDate": "2024:01:15 10:30:00-08:00", 159 + "CreateDate": "2024:01:15 18:30:00", 160 + }, 161 + ) 162 + 163 + assert ( 164 + detect_created_mod.resolve_created_deterministic( 165 + "/tmp/source", 166 + original_filename="voice.m4a", 167 + ) 168 + is None 169 + ) 170 + 171 + 172 + def test_filename_metadata_conflict_falls_through(monkeypatch): 173 + monkeypatch.setattr( 174 + detect_created_mod, 175 + "_extract_metadata_json", 176 + lambda path: {"CreateDate": "2024:01:15 10:30:01"}, 177 + ) 178 + 179 + assert ( 180 + detect_created_mod.resolve_created_deterministic( 181 + "/tmp/source", 182 + original_filename="2024-01-15_10_30_00.m4a", 183 + ) 184 + is None 185 + ) 186 + 187 + 188 + def test_filename_metadata_agree_keeps_filename_source(monkeypatch): 189 + monkeypatch.setattr( 190 + detect_created_mod, 191 + "_extract_metadata_json", 192 + lambda path: {"CreateDate": "2024:01:15 10:30:00"}, 193 + ) 194 + 195 + result = detect_created_mod.resolve_created_deterministic( 196 + "/tmp/source", 197 + original_filename="2024-01-15_10_30_00.m4a", 198 + ) 199 + 200 + assert result["source"] == detect_created_mod.DETERMINISTIC_SOURCE_FILENAME 201 + assert result["day"] == "20240115" 202 + assert result["time"] == "103000" 203 + 204 + 205 + def test_metadata_failure_does_not_block_filename(monkeypatch): 206 + def fail_metadata(path): 207 + raise AssertionError("metadata failed") 208 + 209 + monkeypatch.setattr(detect_created_mod, "_extract_metadata_json", fail_metadata) 210 + 211 + result = detect_created_mod.resolve_created_deterministic( 212 + "/tmp/source", 213 + original_filename="2024-01-15_10_30_00.m4a", 214 + ) 215 + 216 + assert result["source"] == detect_created_mod.DETERMINISTIC_SOURCE_FILENAME 217 + assert result["day"] == "20240115" 218 + assert result["time"] == "103000" 219 + 220 + 221 + def test_extract_metadata_json_returns_empty_on_failure(monkeypatch): 222 + def fail_run(*args, **kwargs): 223 + raise FileNotFoundError("exiftool") 224 + 225 + monkeypatch.setattr(detect_created_mod.subprocess, "run", fail_run) 226 + 227 + assert detect_created_mod._extract_metadata_json("/tmp/source") == {}

+16

tests/test_import_client.py

View file

··· 66 66 "--source": "http-client", 67 67 "--force": "http-client", 68 68 "--auto": "http-client", 69 + "--deterministic-only": "http-client", 69 70 "--dry-run": "reject-journal-host", 70 71 "--json": "client-output", 71 72 "-v/--verbose": "client-logging", ··· 183 184 "setting": "office", 184 185 "source": "ics", 185 186 } 187 + 188 + 189 + def test_deterministic_only_forwards_only_on_save_data(tmp_path: Path) -> None: 190 + media = tmp_path / "sample.txt" 191 + media.write_text("hello", encoding="utf-8") 192 + client = FakeClient() 193 + 194 + code = import_client.main( 195 + [str(media), "--deterministic-only"], 196 + client=client, # type: ignore[arg-type] 197 + ) 198 + 199 + assert code == 0 200 + assert client.uploads[0]["data"] == {"deterministic_only": "true"} 201 + assert "deterministic_only" not in client.requests[0]["json"] 186 202 187 203 188 204 def test_json_output_shape(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:

+232 -3

tests/test_importer.py

View file

··· 101 101 monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) 102 102 think_utils._journal_path_cache = None 103 103 monkeypatch.setattr(import_routes, "detect_created", lambda *args, **kwargs: None) 104 + monkeypatch.setattr( 105 + import_routes, "resolve_created_deterministic", lambda *args, **kwargs: None 106 + ) 104 107 monkeypatch.setattr(import_routes, "now_ms", lambda: 1_765_000_000_000) 105 108 106 109 stamped = identity or ConveyIdentity( ··· 133 136 ) 134 137 135 138 139 + def _read_import_metadata(journal_root: Path, timestamp: str) -> dict: 140 + return json.loads( 141 + (journal_root / "imports" / timestamp / "import.json").read_text( 142 + encoding="utf-8" 143 + ) 144 + ) 145 + 146 + 147 + def test_importer_deterministic_success_skips_model_without_flag(tmp_path, monkeypatch): 148 + mod = importlib.import_module("solstone.think.importers.cli") 149 + media = tmp_path / "note.txt" 150 + media.write_text("meeting notes", encoding="utf-8") 151 + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) 152 + think_utils._journal_path_cache = None 153 + monkeypatch.setattr( 154 + mod, 155 + "resolve_created_deterministic", 156 + lambda *args, **kwargs: {"day": "20240115", "time": "103000"}, 157 + ) 158 + 159 + def fail_detect(*args, **kwargs): 160 + raise AssertionError("model detection should not be called") 161 + 162 + monkeypatch.setattr(mod, "detect_created", fail_detect) 163 + 164 + result = mod.import_one(media) 165 + 166 + assert result == { 167 + "skipped": True, 168 + "reason": "timestamp_required", 169 + "detected_timestamp": "20240115_103000", 170 + } 171 + 172 + 173 + def test_importer_deterministic_only_no_match_never_calls_model( 174 + tmp_path, monkeypatch, capsys 175 + ): 176 + mod = importlib.import_module("solstone.think.importers.cli") 177 + media = tmp_path / "note.txt" 178 + media.write_text("meeting notes", encoding="utf-8") 179 + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) 180 + think_utils._journal_path_cache = None 181 + monkeypatch.setattr( 182 + mod, "resolve_created_deterministic", lambda *args, **kwargs: None 183 + ) 184 + 185 + def fail_detect(*args, **kwargs): 186 + raise AssertionError("model detection should not be called") 187 + 188 + monkeypatch.setattr(mod, "detect_created", fail_detect) 189 + 190 + result = mod.import_one(media, deterministic_only=True) 191 + 192 + assert result == {"skipped": True, "reason": "no_deterministic_match"} 193 + assert "No deterministic timestamp found" in capsys.readouterr().out 194 + 195 + 196 + def test_importer_falls_back_to_model_when_deterministic_missing(tmp_path, monkeypatch): 197 + mod = importlib.import_module("solstone.think.importers.cli") 198 + media = tmp_path / "note.txt" 199 + media.write_text("meeting notes", encoding="utf-8") 200 + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) 201 + think_utils._journal_path_cache = None 202 + monkeypatch.setattr( 203 + mod, "resolve_created_deterministic", lambda *args, **kwargs: None 204 + ) 205 + monkeypatch.setattr( 206 + mod, 207 + "detect_created", 208 + lambda *args, **kwargs: {"day": "20240115", "time": "103000"}, 209 + ) 210 + 211 + result = mod.import_one(media) 212 + 213 + assert result == { 214 + "skipped": True, 215 + "reason": "timestamp_required", 216 + "detected_timestamp": "20240115_103000", 217 + } 218 + 219 + 220 + def test_import_one_explicit_timestamp_bypasses_resolver_and_model( 221 + tmp_path, monkeypatch 222 + ): 223 + mod = importlib.import_module("solstone.think.importers.cli") 224 + transcript = "hello\nworld" 225 + txt = tmp_path / "sample.txt" 226 + txt.write_text(transcript, encoding="utf-8") 227 + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) 228 + think_utils._journal_path_cache = None 229 + _configure_text_import_runtime(monkeypatch, mod) 230 + 231 + def fail_detect(*args, **kwargs): 232 + raise AssertionError("timestamp detection should not be called") 233 + 234 + monkeypatch.setattr(mod, "resolve_created_deterministic", fail_detect) 235 + monkeypatch.setattr(mod, "detect_created", fail_detect) 236 + 237 + result = mod.import_one(txt, timestamp="20240101_120000") 238 + 239 + assert result["processed_timestamp"] == "20240101_120000" 240 + 241 + 136 242 def test_import_save_stamps_web_dashboard_provenance(tmp_path, monkeypatch): 137 243 client = _import_route_client(tmp_path, monkeypatch) 138 244 ··· 185 291 assert metadata["link_id"] == fingerprint 186 292 187 293 294 + def test_import_save_deterministic_success_skips_model_and_audits( 295 + tmp_path, monkeypatch 296 + ): 297 + client = _import_route_client(tmp_path, monkeypatch) 298 + import_routes = importlib.import_module("solstone.apps.import.routes") 299 + deterministic_result = { 300 + "day": "20240115", 301 + "time": "103000", 302 + "confidence": "high", 303 + "source": "filename_local", 304 + "utc": False, 305 + } 306 + monkeypatch.setattr( 307 + import_routes, 308 + "resolve_created_deterministic", 309 + lambda *args, **kwargs: deterministic_result, 310 + ) 311 + 312 + def fail_detect(*args, **kwargs): 313 + raise AssertionError("model detection should not be called") 314 + 315 + monkeypatch.setattr(import_routes, "detect_created", fail_detect) 316 + 317 + response = _post_import_save(client, {}) 318 + 319 + assert response.status_code == 200 320 + body = response.get_json() 321 + assert body["timestamp"] == "20240115_103000" 322 + assert body["timestamp_detection_method"] == "deterministic" 323 + assert body["timestamp_detection_model_called"] is False 324 + assert body["timestamp_detection_no_match_reason"] is None 325 + metadata = _read_import_metadata(tmp_path, body["timestamp"]) 326 + assert metadata["detection_result"] == deterministic_result 327 + assert metadata["detected_timestamp"] == "20240115_103000" 328 + assert metadata["timestamp_detection_method"] == "deterministic" 329 + assert metadata["timestamp_detection_model_called"] is False 330 + assert metadata["timestamp_detection_no_match_reason"] is None 331 + 332 + 333 + def test_import_save_deterministic_only_no_match_uses_upload_fallback_and_audit( 334 + tmp_path, monkeypatch 335 + ): 336 + client = _import_route_client(tmp_path, monkeypatch) 337 + import_routes = importlib.import_module("solstone.apps.import.routes") 338 + monkeypatch.setattr( 339 + import_routes, "resolve_created_deterministic", lambda *args, **kwargs: None 340 + ) 341 + 342 + def fail_detect(*args, **kwargs): 343 + raise AssertionError("model detection should not be called") 344 + 345 + monkeypatch.setattr(import_routes, "detect_created", fail_detect) 346 + 347 + response = _post_import_save(client, {"deterministic_only": "true"}) 348 + 349 + assert response.status_code == 200 350 + body = response.get_json() 351 + expected_timestamp = dt.datetime.fromtimestamp(1_765_000_000_000 / 1000).strftime( 352 + "%Y%m%d_%H%M%S" 353 + ) 354 + assert body["timestamp"] == expected_timestamp 355 + assert body["timestamp_detection_method"] == "upload_fallback" 356 + assert body["timestamp_detection_model_called"] is False 357 + assert body["timestamp_detection_no_match_reason"] == "no_deterministic_match" 358 + metadata = _read_import_metadata(tmp_path, body["timestamp"]) 359 + assert metadata["detected_timestamp"] is None 360 + assert metadata["user_timestamp"] == body["timestamp"] 361 + assert metadata["timestamp_detection_method"] == "upload_fallback" 362 + assert metadata["timestamp_detection_model_called"] is False 363 + assert metadata["timestamp_detection_no_match_reason"] == "no_deterministic_match" 364 + 365 + 366 + def test_import_save_model_success_audits(tmp_path, monkeypatch): 367 + client = _import_route_client(tmp_path, monkeypatch) 368 + import_routes = importlib.import_module("solstone.apps.import.routes") 369 + model_result = {"day": "20240115", "time": "103000"} 370 + monkeypatch.setattr( 371 + import_routes, "resolve_created_deterministic", lambda *args, **kwargs: None 372 + ) 373 + monkeypatch.setattr( 374 + import_routes, "detect_created", lambda *args, **kwargs: model_result 375 + ) 376 + 377 + response = _post_import_save(client, {}) 378 + 379 + assert response.status_code == 200 380 + body = response.get_json() 381 + assert body["timestamp"] == "20240115_103000" 382 + assert body["timestamp_detection_method"] == "model" 383 + assert body["timestamp_detection_model_called"] is True 384 + assert body["timestamp_detection_no_match_reason"] is None 385 + metadata = _read_import_metadata(tmp_path, body["timestamp"]) 386 + assert metadata["detection_result"] == model_result 387 + assert metadata["timestamp_detection_method"] == "model" 388 + assert metadata["timestamp_detection_model_called"] is True 389 + assert metadata["timestamp_detection_no_match_reason"] is None 390 + 391 + 392 + def test_import_save_model_no_match_audits_upload_fallback(tmp_path, monkeypatch): 393 + client = _import_route_client(tmp_path, monkeypatch) 394 + import_routes = importlib.import_module("solstone.apps.import.routes") 395 + monkeypatch.setattr( 396 + import_routes, "resolve_created_deterministic", lambda *args, **kwargs: None 397 + ) 398 + monkeypatch.setattr(import_routes, "detect_created", lambda *args, **kwargs: None) 399 + 400 + response = _post_import_save(client, {}) 401 + 402 + assert response.status_code == 200 403 + body = response.get_json() 404 + assert body["timestamp_detection_method"] == "upload_fallback" 405 + assert body["timestamp_detection_model_called"] is True 406 + assert body["timestamp_detection_no_match_reason"] == "model_no_match" 407 + metadata = _read_import_metadata(tmp_path, body["timestamp"]) 408 + assert metadata["detected_timestamp"] is None 409 + assert metadata["timestamp_detection_method"] == "upload_fallback" 410 + assert metadata["timestamp_detection_model_called"] is True 411 + assert metadata["timestamp_detection_no_match_reason"] == "model_no_match" 412 + 413 + 188 414 def test_cli_import_provenance_defaults(tmp_path, monkeypatch): 189 415 shared = importlib.import_module("solstone.think.importers.shared") 190 416 monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) ··· 284 510 txt.write_text(transcript) 285 511 286 512 monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) 287 - monkeypatch.setattr( 288 - mod, "detect_created", lambda p, **kw: {"day": "20240101", "time": "120000"} 289 - ) 513 + 514 + def fail_detect(*args, **kwargs): 515 + raise AssertionError("timestamp detection should not be called") 516 + 517 + monkeypatch.setattr(mod, "resolve_created_deterministic", fail_detect) 518 + monkeypatch.setattr(mod, "detect_created", fail_detect) 290 519 291 520 # Mock segment detection: returns (start_at, text) tuples with absolute times 292 521 def mock_detect_segment(text, start_time):

Configure Feed

Configure Feed