solpbc.org / solstone
personal memory agent
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

fix(transfer): verify extracted import bytes against the manifest sha256

import_archive now verifies each extracted temp file's sha256 against the manifest before install_file promotes it. A mismatch aborts with the target file, expected hash, and actual hash, and installs nothing for that file.

The plan pre-pass also fails visibly when an archive member has no manifest hash entry or the manifest lists a file the archive does not carry.

Add a mismatch regression test covering the failed file, an earlier valid import, and an already-synced segment.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

author
Jer Miller co-author
Claude Opus 4.8 (1M context) date (Jul 3, 2026, 7:21 PM -0600) commit 9ddfa8a4 parent 08617ec9
+123 -3
+31 -3
solstone/observe/transfer.py
··· 339 339 imported = [] 340 340 with tarfile.open(archive_path, "r:gz") as tar: 341 341 members = tar.getmembers() 342 - plan: list[tuple[str, str, Path, list[tuple[tarfile.TarInfo, Path]]]] = [] 342 + plan: list[tuple[str, str, Path, list[tuple[tarfile.TarInfo, Path, str]]]] = [] 343 343 344 344 for original_arc_key, target_arc_key in validation["import_as"].items(): 345 345 _reject_if_unsafe(target_arc_key, "segment key") 346 346 target_dir = contained_path(journal, f"{day}/{target_arc_key}") 347 + manifest_by_name = { 348 + f["name"]: f["sha256"] 349 + for f in manifest["segments"][original_arc_key].get("files", []) 350 + } 347 351 348 352 planned_files = [] 353 + planned_names = set() 349 354 prefix = f"{original_arc_key}/" 350 355 for member in members: 351 356 if member.name.startswith(prefix) and member.isfile(): ··· 357 362 f"{target_arc_key!r}: {member.name!r}" 358 363 ) 359 364 _reject_if_unsafe(filename, "member filename") 365 + if filename not in manifest_by_name: 366 + raise ValueError( 367 + f"Archive member for segment {original_arc_key!r} " 368 + f"has no manifest hash entry: {filename!r}" 369 + ) 360 370 target_path = contained_path( 361 371 journal, 362 372 f"{day}/{target_arc_key}/{filename}", 363 373 ) 364 - planned_files.append((member, target_path)) 374 + planned_files.append( 375 + (member, target_path, manifest_by_name[filename]) 376 + ) 377 + planned_names.add(filename) 378 + 379 + missing = set(manifest_by_name) - planned_names 380 + if missing: 381 + missing_list = ", ".join(sorted(missing)) 382 + raise ValueError( 383 + f"Archive missing manifest-listed file(s) for segment " 384 + f"{original_arc_key!r}: {missing_list}" 385 + ) 365 386 366 387 plan.append((original_arc_key, target_arc_key, target_dir, planned_files)) 367 388 368 389 for original_arc_key, target_arc_key, target_dir, planned_files in plan: 369 390 target_dir.mkdir(parents=True, exist_ok=True) 370 391 371 - for member, target_path in planned_files: 392 + for member, target_path, expected_sha256 in planned_files: 372 393 source = tar.extractfile(member) 373 394 if source: 374 395 temp_path = None ··· 385 406 temp_path = Path(temp_handle.name) 386 407 shutil.copyfileobj(source, temp_handle) 387 408 temp_handle.close() 409 + actual_sha256 = compute_file_sha256(temp_path) 410 + if actual_sha256 != expected_sha256: 411 + raise ValueError( 412 + f"Archive content mismatch for {target_path}: " 413 + f"manifest sha256 {expected_sha256} != extracted " 414 + f"{actual_sha256}" 415 + ) 388 416 install_file(temp_path, target_path) 389 417 promoted = True 390 418 # Preserve modification time (install_file does not)
+92
tests/test_transfer.py
··· 504 504 assert first_path.read_bytes() == b"first content" 505 505 assert list(day_dir.rglob("*.tmp")) == [] 506 506 507 + def test_import_archive_rejects_member_sha256_mismatch(self, tmp_path, monkeypatch): 508 + """Test manifest/member hash mismatch aborts before promoting that file.""" 509 + import io 510 + 511 + from solstone.observe.transfer import import_archive 512 + from solstone.observe.utils import compute_bytes_sha256 513 + 514 + archive_path = tmp_path / "sha-mismatch.tgz" 515 + synced_content = b"already synced" 516 + good_content = b"good import" 517 + bad_manifest_content = b"manifest bytes" 518 + bad_member_content = b"tampered bytes" 519 + manifest = { 520 + "version": 1, 521 + "day": "20250101", 522 + "created_at": 1704067200000, 523 + "host": "test-host", 524 + "segments": { 525 + "120000_300": { 526 + "files": [ 527 + { 528 + "name": "audio.flac", 529 + "sha256": compute_bytes_sha256(synced_content), 530 + "size": len(synced_content), 531 + } 532 + ] 533 + }, 534 + "125000_300": { 535 + "files": [ 536 + { 537 + "name": "audio.flac", 538 + "sha256": compute_bytes_sha256(good_content), 539 + "size": len(good_content), 540 + } 541 + ] 542 + }, 543 + "130000_300": { 544 + "files": [ 545 + { 546 + "name": "audio.flac", 547 + "sha256": compute_bytes_sha256(bad_manifest_content), 548 + "size": len(bad_manifest_content), 549 + } 550 + ] 551 + }, 552 + }, 553 + } 554 + 555 + with tarfile.open(archive_path, "w:gz") as tar: 556 + for segment, content in ( 557 + ("120000_300", synced_content), 558 + ("125000_300", good_content), 559 + ("130000_300", bad_member_content), 560 + ): 561 + info = tarfile.TarInfo(name=f"{segment}/audio.flac") 562 + info.size = len(content) 563 + tar.addfile(info, io.BytesIO(content)) 564 + 565 + manifest_json = json.dumps(manifest).encode() 566 + manifest_info = tarfile.TarInfo(name="manifest.json") 567 + manifest_info.size = len(manifest_json) 568 + tar.addfile(manifest_info, io.BytesIO(manifest_json)) 569 + 570 + journal_path = tmp_path / "journal" 571 + synced_path = ( 572 + journal_path / "chronicle" / "20250101" / "120000_300" / "audio.flac" 573 + ) 574 + synced_path.parent.mkdir(parents=True) 575 + synced_path.write_bytes(synced_content) 576 + 577 + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal_path)) 578 + 579 + import solstone.think.utils as think_utils 580 + 581 + think_utils._journal_path_cache = None 582 + 583 + bad_target = ( 584 + journal_path / "chronicle" / "20250101" / "130000_300" / "audio.flac" 585 + ) 586 + with pytest.raises(ValueError) as excinfo: 587 + import_archive(archive_path) 588 + 589 + message = str(excinfo.value) 590 + assert "Archive content mismatch" in message 591 + assert str(bad_target) in message 592 + assert not bad_target.exists() 593 + assert ( 594 + journal_path / "chronicle" / "20250101" / "125000_300" / "audio.flac" 595 + ).read_bytes() == good_content 596 + assert synced_path.read_bytes() == synced_content 597 + assert list((journal_path / "chronicle").rglob("*.tmp")) == [] 598 + 507 599 def test_import_archive_routes_member_writes_through_install_file(self): 508 600 """Test import_archive has no raw durable member write path.""" 509 601 from solstone.observe import transfer