solpbc.org
personal memory agent
feat(import): byte-hash dedup + batch-safe collisions for audio/text imports
Bring the generic audio/text import path to parity with the file-importer
path on byte-hash duplicate detection, and make collisions catchable in a
batch loop.
- Re-importing the same audio/text bytes is detected as a duplicate before
the timestamp-detection LLM call, printing the same "already imported"
message the file importers give; --force still re-imports.
- import_one() directory collisions raise a catchable FileExistsError instead
of SystemExit; the human `sol import` CLI still surfaces the same message
and exit code 1 via main()'s wrapper.
- Successful audio/text imports now write a dedup manifest.json with correct
days_affected for the YYYYMMDD/stream/segment/file layout.
- write_manifest now records import_id (it already took the param) and accepts
an optional days_affected to avoid path-derivation across two layouts.
- detect_created() no longer writes gemini_debug_* files to /tmp.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
author
Jer Miller
co-author
Claude Opus 4.8 (1M context)
date
(Jun 4, 2026, 2:28 PM -0600)
commit
5cfd7c2d
5cfd7c2d1df5eca2dbae89c32710a3a3dcdf0e74
parent
b780d52e
b780d52ec08f3f00044620a508299cc7df1e2228
+208
-25
-17
solstone/think/detect_created.py
-17
solstone/think/detect_created.py
···
6
6
from __future__ import annotations
7
7
8
8
import json
9
-
import os
10
9
import subprocess
11
-
import sys
12
10
from datetime import datetime, timezone
13
11
from pathlib import Path
14
12
from typing import Optional
···
39
37
return f"Error extracting metadata: {exc}"
40
38
41
39
42
-
def _debug_write_content(content: str, path: str) -> None:
43
-
"""Write content to a debug file in /tmp for diagnosis."""
44
-
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
45
-
filename = f"gemini_debug_{timestamp}_{os.path.basename(path)}.md"
46
-
debug_path = os.path.join("/tmp", filename)
47
-
48
-
with open(debug_path, "w", encoding="utf-8") as f:
49
-
f.write(content)
50
-
51
-
print(f"Debug: Content written to {debug_path}", file=sys.stderr)
52
-
53
-
54
40
def detect_created(
55
41
path: str, original_filename: Optional[str] = None, guidance: Optional[str] = None
56
42
) -> Optional[dict]:
···
89
75
markdown = "\n".join(lines)
90
76
if guidance:
91
77
markdown += f"\n\nImportant guidance from the user: {guidance}"
92
-
93
-
# Debug: write content to temp file
94
-
_debug_write_content(markdown, path)
95
78
96
79
from solstone.think.models import generate
97
80
+42
solstone/think/importers/cli.py
+42
solstone/think/importers/cli.py
···
402
402
_file_importer = detected
403
403
import_source = detected.name
404
404
405
+
# Source-level dedup for audio/text (before timestamp detection and before
406
+
# _setup_import rewrites args.media). Mirrors the file-importer dedup at the
407
+
# file-importer branch; skipped for --force and --dry-run (which still preview).
408
+
if _file_importer is None and not args.dry_run:
409
+
from solstone.think.importers.shared import (
410
+
find_manifest_by_hash,
411
+
hash_source,
412
+
)
413
+
414
+
_source_hash = hash_source(Path(args.media))
415
+
if not args.force:
416
+
existing = find_manifest_by_hash(Path(get_journal()), _source_hash)
417
+
if existing:
418
+
imported_at = existing.get("imported_at", "unknown date")
419
+
entry_count = existing.get("entry_count", 0)
420
+
print(
421
+
f"This file was already imported on {imported_at} "
422
+
f"({entry_count} entries). Use --force to re-import."
423
+
)
424
+
return {
425
+
"skipped": True,
426
+
"reason": "already_imported",
427
+
"imported_at": imported_at,
428
+
"entry_count": entry_count,
429
+
}
430
+
405
431
# --- Timestamp resolution ---
406
432
if _file_importer is not None and not args.timestamp:
407
433
# File importers don't need an external timestamp — auto-generate for metadata
···
1086
1112
"date_range",
1087
1113
[processing_results["target_day"], processing_results["target_day"]],
1088
1114
)
1115
+
1116
+
# Write dedup manifest for audio/text imports. File importers already wrote
1117
+
# theirs in the file-importer branch above (line ~887); guard prevents a
1118
+
# double write since all branches fall through this common tail.
1119
+
if _file_importer is None:
1120
+
from solstone.think.importers.shared import write_manifest
1121
+
1122
+
write_manifest(
1123
+
journal_root,
1124
+
import_id=args.timestamp,
1125
+
source_type=import_source,
1126
+
source_hash=_source_hash,
1127
+
entry_count=len(all_created_files),
1128
+
files_created=all_created_files,
1129
+
days_affected=[day],
1130
+
)
1089
1131
1090
1132
imported_path = import_dir / "imported.json"
1091
1133
# Write imported.json with all processing metadata
+4
tests/test_detect_created_schema.py
+4
tests/test_detect_created_schema.py
···
1
1
# SPDX-License-Identifier: AGPL-3.0-only
2
2
# Copyright (c) 2026 sol pbc
3
3
4
+
import glob
4
5
import importlib
5
6
import json
6
7
from pathlib import Path
···
71
72
lambda path: "QuickTime Create Date : 2024:03:15 14:30:52",
72
73
)
73
74
75
+
before = set(glob.glob("/tmp/gemini_debug_*"))
74
76
result = detect_created_mod.detect_created("/dev/null")
77
+
after = set(glob.glob("/tmp/gemini_debug_*"))
75
78
76
79
assert captured["json_schema"] is detect_created_mod._SCHEMA
80
+
assert after == before
77
81
assert result == {
78
82
"day": "20240315",
79
83
"time": "143052",
+2
tests/test_import_dedup.py
+2
tests/test_import_dedup.py
···
119
119
# Read it back
120
120
with open(manifest_path) as f:
121
121
data = json.load(f)
122
+
assert data["import_id"] == "20260115_120000"
122
123
assert data["source_type"] == "ics"
123
124
assert data["source_hash"] == "abc123"
124
125
assert data["entry_count"] == 42
···
128
129
# Find by hash
129
130
found = find_manifest_by_hash(Path(journal), "abc123")
130
131
assert found is not None
132
+
assert found["import_id"] == "20260115_120000"
131
133
assert found["source_type"] == "ics"
132
134
133
135
# Not found for different hash
+149
tests/test_importer.py
+149
tests/test_importer.py
···
1144
1144
assert _read_action_entries(tmp_path) == []
1145
1145
1146
1146
1147
+
def test_import_one_collision_raises_catchable_file_exists_error(tmp_path, monkeypatch):
1148
+
mod = importlib.import_module("solstone.think.importers.cli")
1149
+
1150
+
timestamp = "20240101_120000"
1151
+
import_dir = tmp_path / "imports" / timestamp
1152
+
import_dir.mkdir(parents=True)
1153
+
(import_dir / "stale.txt").write_text("old import payload")
1154
+
1155
+
media = tmp_path / "note.txt"
1156
+
media.write_text("new transcript")
1157
+
1158
+
monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path))
1159
+
1160
+
with pytest.raises(FileExistsError) as ei:
1161
+
mod.import_one(media, timestamp=timestamp)
1162
+
1163
+
assert not isinstance(ei.value, SystemExit)
1164
+
1165
+
1147
1166
def test_file_importer_without_timestamp(tmp_path, monkeypatch, capsys):
1148
1167
"""File importers auto-generate timestamp and skip import setup."""
1149
1168
mod = importlib.import_module("solstone.think.importers.cli")
···
1388
1407
assert elapsed < 5
1389
1408
assert result.get("segments")
1390
1409
assert "failed_segments" not in result
1410
+
1411
+
1412
+
def test_import_one_audio_reimport_is_deduped(tmp_path, monkeypatch):
1413
+
mod = importlib.import_module("solstone.think.importers.cli")
1414
+
1415
+
audio_file = tmp_path / "test.mp3"
1416
+
audio_file.write_bytes(b"fake audio")
1417
+
calls = []
1418
+
callosum = MagicMock()
1419
+
1420
+
def fake_prepare_audio_segments(media_path, day_dir, base_dt, import_id, stream):
1421
+
calls.append(media_path)
1422
+
seg_dir = Path(day_dir) / stream / "120000_300"
1423
+
seg_dir.mkdir(parents=True, exist_ok=True)
1424
+
(seg_dir / "imported_audio.mp3").write_bytes(b"sliced audio")
1425
+
return [("120000_300", seg_dir, ["imported_audio.mp3"])]
1426
+
1427
+
monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path))
1428
+
monkeypatch.setattr(mod, "CallosumConnection", lambda **kwargs: callosum)
1429
+
monkeypatch.setattr(mod, "get_rev", lambda: "test-rev")
1430
+
monkeypatch.setattr(mod, "_status_emitter", lambda: None)
1431
+
monkeypatch.setattr(mod, "prepare_audio_segments", fake_prepare_audio_segments)
1432
+
monkeypatch.setattr(
1433
+
mod,
1434
+
"update_stream",
1435
+
lambda stream, day, seg, **kwargs: {
1436
+
"prev_day": None,
1437
+
"prev_segment": None,
1438
+
"seq": 1,
1439
+
},
1440
+
)
1441
+
monkeypatch.setattr(mod, "write_segment_stream", lambda *args, **kwargs: None)
1442
+
1443
+
result = mod.import_one(
1444
+
audio_file,
1445
+
timestamp="20260303_120000",
1446
+
source="audio",
1447
+
wait_for_processing=False,
1448
+
)
1449
+
1450
+
assert result is not None
1451
+
assert len(calls) == 1
1452
+
manifest_path = tmp_path / "imports" / "20260303_120000" / "manifest.json"
1453
+
assert manifest_path.exists()
1454
+
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
1455
+
assert manifest["source_hash"] == hashlib.sha256(b"fake audio").hexdigest()
1456
+
assert manifest["source_type"] == "audio"
1457
+
assert manifest["import_id"] == "20260303_120000"
1458
+
assert manifest["days_affected"] == ["20260303"]
1459
+
1460
+
audio_file2 = tmp_path / "same-audio.mp3"
1461
+
audio_file2.write_bytes(b"fake audio")
1462
+
result = mod.import_one(
1463
+
audio_file2,
1464
+
timestamp="20260303_120000",
1465
+
source="audio",
1466
+
wait_for_processing=False,
1467
+
)
1468
+
1469
+
assert result is not None
1470
+
assert result["reason"] == "already_imported"
1471
+
assert len(calls) == 1
1472
+
1473
+
1474
+
def test_import_one_text_reimport_is_deduped(tmp_path, monkeypatch):
1475
+
mod = importlib.import_module("solstone.think.importers.cli")
1476
+
text_mod = importlib.import_module("solstone.think.importers.text")
1477
+
1478
+
transcript = "hello\nworld"
1479
+
txt = tmp_path / "sample.txt"
1480
+
txt.write_text(transcript)
1481
+
calls = []
1482
+
1483
+
def fake_detect_segment(text, start_time):
1484
+
calls.append((text, start_time))
1485
+
return [("12:00:00", text)]
1486
+
1487
+
monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path))
1488
+
monkeypatch.setattr(
1489
+
mod, "detect_created", lambda p, **kw: {"day": "20240101", "time": "120000"}
1490
+
)
1491
+
monkeypatch.setattr(text_mod, "detect_transcript_segment", fake_detect_segment)
1492
+
monkeypatch.setattr(
1493
+
text_mod,
1494
+
"detect_transcript_json",
1495
+
lambda text, segment_start: {
1496
+
"entries": [
1497
+
{
1498
+
"start": segment_start,
1499
+
"speaker": "Unknown",
1500
+
"text": text,
1501
+
}
1502
+
],
1503
+
"topics": "",
1504
+
"setting": "",
1505
+
},
1506
+
)
1507
+
monkeypatch.setattr(mod, "CallosumConnection", lambda **kwargs: MagicMock())
1508
+
monkeypatch.setattr(mod, "get_rev", lambda: "test-rev")
1509
+
monkeypatch.setattr(mod, "_status_emitter", lambda: None)
1510
+
1511
+
result = mod.import_one(
1512
+
txt,
1513
+
timestamp="20240101_120000",
1514
+
source="text",
1515
+
)
1516
+
1517
+
assert result is not None
1518
+
assert len(calls) == 1
1519
+
manifest_path = tmp_path / "imports" / "20240101_120000" / "manifest.json"
1520
+
assert manifest_path.exists()
1521
+
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
1522
+
assert manifest["source_type"] == "text"
1523
+
assert manifest["import_id"] == "20240101_120000"
1524
+
assert manifest["days_affected"] == ["20240101"]
1525
+
segment_dir = day_path("20240101") / "import.text"
1526
+
assert len(list(segment_dir.iterdir())) == 1
1527
+
1528
+
txt2 = tmp_path / "same-sample.txt"
1529
+
txt2.write_text(transcript)
1530
+
result = mod.import_one(
1531
+
txt2,
1532
+
timestamp="20240101_120000",
1533
+
source="text",
1534
+
)
1535
+
1536
+
assert result is not None
1537
+
assert result["reason"] == "already_imported"
1538
+
assert len(calls) == 1
1539
+
assert len(list(segment_dir.iterdir())) == 1
1391
1540
1392
1541
1393
1542
def test_file_importer_indexes_created_files_in_process(tmp_path, monkeypatch):