Enforce health raw retention decisions · solpbc.org/solstone@ee0e68a

+4 -1

docs/design/oura-import.md

··· 129 129 130 130 ``` 131 131 imports/<import_id>/ 132 - raw/oura/<endpoint>/<NNNN>.json # verbatim API page documents (save phase) 132 + raw/oura/<endpoint>.jsonl # verbatim API pages when raw_retention=retain_parsed 133 133 normalized/<YYYY-MM>.jsonl # monthly shards, schema solstone.health.oura.v1 134 134 manifest.json # shared.write_manifest, source_type "oura" 135 135 content_manifest.jsonl # shared.write_content_manifest 136 + fetch_windows.json # fetched window evidence for the chunker 136 137 imports/health-dedupe.sqlite # shared dedupe DB (existing) 137 138 imports/oura.json # sync cursor (phase O3; never tokens) 138 139 chronicle/<day>/import.oura/000000_300/day_summary_transcript.md # optional, save phase 139 140 ``` 141 + 142 + Importer-owned files under `imports/` are private (`0600`) and importer-owned directories under `imports/` are created or repaired as `0700`. Oura API sync applies the validated `raw_retention.decision`: `retain_parsed` keeps raw API pages, while `discard` writes no raw page JSONL and stores no new `raw_ref` values. 140 143 141 144 **Normalized row** (implemented in the skeleton): 142 145

+6 -2

docs/health_imports.md

View file

··· 9 9 - Preview Apple Health `export.xml` data from a directory or zip. 10 10 - Filter previews and save runs with `--date-from YYYY-MM-DD` and `--date-to YYYY-MM-DD`. 11 11 - Require the health pre-save gate before any non-dry-run Apple Health write. 12 - - Install the source export under `imports/<id>/raw/`. 12 + - Apply the approved raw-retention decision before installing raw source material. 13 13 - Write normalized monthly JSONL under `imports/<id>/normalized/`. 14 14 - Keep importer-owned record dedupe in `imports/health-dedupe.sqlite`. 15 15 - Optionally write small factual day-summary transcript files with `--with-day-summaries`. ··· 18 18 19 19 ## Apple Health Local Save Path 20 20 21 - Apple Health has a concrete importer save path for orchestrated use after privacy preflight. The importer writes only under the provided `journal_root`: raw source material under `imports/<id>/raw/`, normalized monthly JSONL under `imports/<id>/normalized/`, importer-owned dedupe rows in `imports/health-dedupe.sqlite`, and optional factual day-summary transcript files under `chronicle/YYYYMMDD/import.apple_health/000000_300/`. 21 + Apple Health has a concrete importer save path for orchestrated use after privacy preflight. The importer writes only under the provided `journal_root`: approved raw source material under `imports/<id>/raw/`, normalized monthly JSONL under `imports/<id>/normalized/`, importer-owned dedupe rows in `imports/health-dedupe.sqlite`, and optional factual day-summary transcript files under `chronicle/YYYYMMDD/import.apple_health/000000_300/`. 22 22 23 23 Dense normalized JSONL shards are not returned in `ImportResult.files_created`; only optional day-summary transcript files are returned there so indexers do not ingest per-sample health rows. 24 + 25 + Raw retention is enforced from the validated gate decision. `discard` writes no `raw/` directory and normalized rows carry no `raw_ref`. `retain_parsed` installs only `raw/export.xml` for Apple Health, whether the input was a zip or an export directory. `retain_complete` is the only Apple Health branch that copies the original zip or full export tree. Oura API sync accepts `discard` and `retain_parsed`: parsed retention keeps the raw API page JSONL files, while discard writes normalized shards, manifests, dedupe rows, fetch windows, and cursor state without raw page files or raw refs. 26 + 27 + All files written under `imports/` by the shared importer writers are installed as `0600`. Import-owned directories under `imports/` are created or repaired as `0700`. The approval-artifact directory `imports/_approvals/` remains manually owner-managed and is not created or repaired by the read-only gate. 24 28 25 29 ## Source Strategy 26 30

+48 -6

solstone/think/importers/apple_health.py

View file

··· 31 31 health_value_hash, 32 32 pick_day_sleep, 33 33 ) 34 - from solstone.think.importers.pre_save_gate import enforce_pre_save_gate 34 + from solstone.think.importers.pre_save_gate import ( 35 + RawRetentionDecision, 36 + enforce_pre_save_gate, 37 + ) 35 38 from solstone.think.importers.shared import ( 36 39 install_source_file, 40 + install_source_stream, 37 41 windowed_source_hash, 38 42 write_content_manifest, 39 43 write_jsonl_records, ··· 258 262 summary=f"Dry run only: {preview.summary}", 259 263 date_range=preview.date_range, 260 264 ) 265 + assert _gate_decision.raw_retention is not None 261 266 262 267 resolved_import_id = import_id or dt.datetime.now().strftime("%Y%m%d_%H%M%S") 263 268 result = _save_export( ··· 267 272 date_window=date_window, 268 273 with_day_summaries=with_day_summaries, 269 274 progress_callback=progress_callback, 275 + retention=_gate_decision.raw_retention, 270 276 ) 271 277 return ImportResult( 272 278 entries_written=result["entries_written"], ··· 276 282 summary=result["summary"], 277 283 segments=result["segments"] or None, 278 284 date_range=result["date_range"], 285 + raw_retention=result["raw_retention"], 279 286 ) 280 287 281 288 ··· 524 531 date_window: _DateWindow, 525 532 with_day_summaries: bool, 526 533 progress_callback: Callable | None, 534 + retention: RawRetentionDecision, 527 535 ) -> dict[str, Any]: 528 536 journal_root = Path(journal_root) 529 537 import_dir = Path(journal_root) / "imports" / import_id 530 - raw_ref = _install_raw_source(path, import_dir) 538 + raw_ref = _install_raw_source(path, import_dir, retention) 531 539 normalized_items = _parse_normalized_items( 532 540 path, 533 541 import_id=import_id, ··· 591 599 len(normalized_items), 592 600 files_created, 593 601 days_affected=sorted(summaries), 602 + raw_retention=retention.value, 594 603 ) 595 604 596 605 date_range = _date_range_from_days(summaries) ··· 599 608 "files_created": files_created, 600 609 "segments": segments, 601 610 "date_range": date_range, 611 + "raw_retention": retention.value, 602 612 "summary": ( 603 613 "Saved Apple Health import: " 604 614 f"records={len(normalized_items)}, " ··· 651 661 return files, segments 652 662 653 663 654 - def _install_raw_source(path: Path, import_dir: Path) -> str: 664 + def _install_raw_source( 665 + path: Path, 666 + import_dir: Path, 667 + retention: RawRetentionDecision, 668 + ) -> str | None: 669 + if retention == RawRetentionDecision.DISCARD: 670 + return None 671 + 655 672 raw_dir = import_dir / "raw" 673 + if retention == RawRetentionDecision.RETAIN_PARSED: 674 + raw_path = raw_dir / "export.xml" 675 + if path.is_file(): 676 + with zipfile.ZipFile(path) as archive: 677 + member = _find_export_xml_in_zip(archive.namelist()) 678 + if member is None: 679 + raise FileNotFoundError( 680 + f"No Apple Health export.xml found in {path}" 681 + ) 682 + with archive.open(member) as handle: 683 + install_source_stream(handle, raw_path) 684 + else: 685 + export_xml = _find_export_xml_in_directory(path) 686 + if export_xml is None: 687 + raise FileNotFoundError( 688 + f"No Apple Health export.xml found under {path}" 689 + ) 690 + install_source_file(export_xml, raw_path) 691 + return f"imports/{import_dir.name}/raw/export.xml" 692 + 656 693 if path.is_file(): 657 694 raw_path = raw_dir / path.name 658 695 install_source_file(path, raw_path) ··· 674 711 *, 675 712 import_id: str, 676 713 date_window: _DateWindow, 677 - raw_ref: str, 714 + raw_ref: str | None, 678 715 progress_callback: Callable | None, 679 716 ) -> list[_NormalizedItem]: 680 717 items: list[_NormalizedItem] = [] ··· 703 740 if elem.tag == "Workout": 704 741 for key, value in _workout_statistics_metadata(elem).items(): 705 742 metadata.setdefault(key, value) 743 + item_raw_ref = ( 744 + f"{raw_ref}#{elem.tag.lower()}-{scanned}" 745 + if raw_ref is not None 746 + else None 747 + ) 706 748 items.append( 707 749 _normalize_element( 708 750 elem.tag, 709 751 attrib, 710 752 import_id=import_id, 711 - raw_ref=f"{raw_ref}#{elem.tag.lower()}-{scanned}", 753 + raw_ref=item_raw_ref, 712 754 day=day or "", 713 755 metadata=metadata, 714 756 identity_metadata=identity_metadata, ··· 739 781 attrib: dict[str, str], 740 782 *, 741 783 import_id: str, 742 - raw_ref: str, 784 + raw_ref: str | None, 743 785 day: str, 744 786 metadata: dict[str, str] | None = None, 745 787 identity_metadata: dict[str, str] | None = None,

+1

solstone/think/importers/file_importer.py

View file

··· 36 36 principal_collision: dict[str, Any] | None = None 37 37 merge_log_path: str | None = None 38 38 merge_staging_path: str | None = None 39 + raw_retention: str | None = None 39 40 40 41 41 42 @runtime_checkable

+32 -11

solstone/think/importers/oura.py

View file

··· 90 90 health_value_hash, 91 91 ) 92 92 from solstone.think.importers.pre_save_gate import ( 93 + RawRetentionDecision, 93 94 ScheduledSyncConsent, 94 95 enforce_oura_sync_gate, 95 96 enforce_pre_save_gate, ··· 461 462 bundle: Mapping[str, Iterable[Mapping[str, Any]]], 462 463 *, 463 464 import_id: str, 464 - raw_ref_root: str, 465 + raw_ref_root: str | None, 465 466 owner_timezone: dt.tzinfo | None = None, 466 467 ) -> list[OuraNormalizedItem]: 467 468 """Normalize parsed endpoint items into rows with stable dedupe keys.""" ··· 470 471 items: list[OuraNormalizedItem] = [] 471 472 for endpoint in sorted(bundle): 472 473 for index, item in enumerate(bundle[endpoint], start=1): 473 - raw_ref = f"{raw_ref_root}#{endpoint}-{index}" 474 + raw_ref = ( 475 + f"{raw_ref_root}#{endpoint}-{index}" 476 + if raw_ref_root is not None 477 + else None 478 + ) 474 479 items.extend( 475 480 _normalize_item( 476 481 endpoint, ··· 488 493 item: dict[str, Any], 489 494 *, 490 495 import_id: str, 491 - raw_ref: str, 496 + raw_ref: str | None, 492 497 owner_timezone: dt.tzinfo, 493 498 ) -> list[OuraNormalizedItem]: 494 499 day = parse_oura_day(item.get(_DOCUMENT_DAY_FIELDS.get(endpoint, "day"))) or "" ··· 872 877 unit: str | None, 873 878 metadata: dict[str, Any], 874 879 import_id: str, 875 - raw_ref: str, 880 + raw_ref: str | None, 876 881 ) -> OuraNormalizedItem: 877 882 dedupe_key = health_record_dedupe_key( 878 883 HealthRecordIdentity( ··· 1518 1523 ) 1519 1524 1520 1525 assert _gate_decision is not None 1526 + assert _gate_decision.raw_retention is not None 1521 1527 1522 1528 # Quiet-run check: classify the fetch against the dedupe ledger 1523 1529 # BEFORE allocating an import id or writing anything. Dedupe keys ··· 1564 1570 scheduled_sync=_gate_decision.scheduled_sync, 1565 1571 read_artifact=False, 1566 1572 ), 1573 + raw_retention=_gate_decision.raw_retention.value, 1567 1574 quiet_run=True, 1568 1575 errors=errors, 1569 1576 ) 1570 1577 1571 1578 import_id = _new_import_id(journal_root) 1579 + raw_ref_root = ( 1580 + f"imports/{import_id}/raw/oura" 1581 + if _gate_decision.raw_retention == RawRetentionDecision.RETAIN_PARSED 1582 + else None 1583 + ) 1572 1584 items = normalize_bundle( 1573 1585 bundle, 1574 1586 import_id=import_id, 1575 - raw_ref_root=f"imports/{import_id}/raw/oura", 1587 + raw_ref_root=raw_ref_root, 1576 1588 owner_timezone=owner_timezone, 1577 1589 ) 1578 1590 saved = _save_sync_bundle( ··· 1582 1594 raw_pages=raw_pages, 1583 1595 bundle=bundle, 1584 1596 windows=windows, 1597 + retention=_gate_decision.raw_retention, 1585 1598 ) 1586 1599 1587 1600 # The cursor advances only after every bundle write succeeded; a ··· 1615 1628 updated=saved["updated"], 1616 1629 months=saved["months"], 1617 1630 cron_hint=cron_hint, 1631 + raw_retention=_gate_decision.raw_retention.value, 1618 1632 errors=errors, 1619 1633 ) 1620 1634 ··· 1715 1729 updated: int, 1716 1730 months: list[str], 1717 1731 cron_hint: str | None, 1732 + raw_retention: str | None = None, 1718 1733 known_rows: int = 0, 1719 1734 quiet_run: bool = False, 1720 1735 errors: list[str] | None = None, ··· 1766 1781 result["import_id"] = import_id 1767 1782 if cron_hint is not None: 1768 1783 result["cron_hint"] = cron_hint 1784 + if raw_retention is not None: 1785 + result["raw_retention"] = raw_retention 1769 1786 return result 1770 1787 1771 1788 ··· 1965 1982 raw_pages: Mapping[str, list[dict[str, Any]]], 1966 1983 bundle: Mapping[str, list[dict[str, Any]]], 1967 1984 windows: Mapping[str, tuple[str, str]], 1985 + retention: RawRetentionDecision, 1968 1986 ) -> dict[str, Any]: 1969 1987 """Write one sync run's import bundle, mirroring apple_health save mode. 1970 1988 ··· 1978 1996 # Raw page documents land first so every row's raw_ref points at bytes 1979 1997 # that already exist: one JSONL per endpoint, one verbatim API page 1980 1998 # per line, under imports/<id>/raw/oura/. 1981 - for endpoint in sorted(raw_pages): 1982 - pages = raw_pages[endpoint] 1983 - if pages: 1984 - write_jsonl_records( 1985 - import_dir / "raw" / "oura" / f"{endpoint}.jsonl", pages 1986 - ) 1999 + if retention == RawRetentionDecision.RETAIN_PARSED: 2000 + for endpoint in sorted(raw_pages): 2001 + pages = raw_pages[endpoint] 2002 + if pages: 2003 + write_jsonl_records( 2004 + import_dir / "raw" / "oura" / f"{endpoint}.jsonl", pages 2005 + ) 1987 2006 1988 2007 normalized_by_month: dict[str, list[dict[str, Any]]] = defaultdict(list) 1989 2008 dedupe_records: list[HealthDedupeRecord] = [] ··· 2035 2054 len(items), 2036 2055 files_created=[], 2037 2056 days_affected=sorted(days), 2057 + raw_retention=retention.value, 2038 2058 ) 2039 2059 2040 2060 return { 2041 2061 "inserted": dedupe_result.inserted, 2042 2062 "updated": dedupe_result.updated, 2043 2063 "months": [path.stem for path in normalized_paths], 2064 + "raw_retention": retention.value, 2044 2065 } 2045 2066 2046 2067

+179 -11

tests/test_apple_health_importer.py

View file

··· 29 29 pick_day_sleep, 30 30 pick_main_session, 31 31 ) 32 - from solstone.think.importers.pre_save_gate import PreSaveGateError 32 + from solstone.think.importers.pre_save_gate import ( 33 + PreSaveGateError, 34 + RawRetentionDecision, 35 + ) 33 36 from tests.conftest import write_health_approval_artifact 34 37 35 38 FIXTURE_ROOT = ( ··· 84 87 """ 85 88 86 89 87 - def _process_with_approval(path: Path, journal: Path, **kwargs: Any) -> ImportResult: 88 - write_health_approval_artifact(journal, importers=["apple_health"]) 90 + def _process_with_approval( 91 + path: Path, 92 + journal: Path, 93 + *, 94 + raw_retention_decision: str = RawRetentionDecision.RETAIN_PARSED.value, 95 + unparsed_sensitive_modalities_acknowledged: bool | None = None, 96 + **kwargs: Any, 97 + ) -> ImportResult: 98 + write_health_approval_artifact( 99 + journal, 100 + importers=["apple_health"], 101 + raw_retention_decision=raw_retention_decision, 102 + unparsed_sensitive_modalities_acknowledged=( 103 + unparsed_sensitive_modalities_acknowledged 104 + ), 105 + ) 89 106 return AppleHealthImporter().process( 90 107 path, 91 108 journal, ··· 347 364 date_to="20260102", 348 365 ) 349 366 350 - raw_export = ( 351 - journal 352 - / "imports" 353 - / "20260103_120000" 354 - / "raw" 355 - / "apple_health_export" 356 - / "export.xml" 357 - ) 367 + raw_export = journal / "imports" / "20260103_120000" / "raw" / "export.xml" 358 368 normalized = ( 359 369 journal / "imports" / "20260103_120000" / "normalized" / "2026-01.jsonl" 360 370 ) ··· 372 382 assert result.segments is None 373 383 assert result.date_range == ("20260102", "20260102") 374 384 assert raw_export.read_text(encoding="utf-8").startswith("<?xml") 385 + assert glucose_row["raw_ref"].startswith( 386 + "imports/20260103_120000/raw/export.xml#record-" 387 + ) 375 388 assert {row["day"] for row in rows} == {"20260102"} 376 389 assert {row["kind"] for row in rows} == {"record", "workout"} 377 390 assert all(row["import_id"] == "20260103_120000" for row in rows) ··· 382 395 assert not live_journal.exists() 383 396 384 397 398 + @pytest.mark.parametrize( 399 + ("source_path", "source_label"), 400 + [(ZIP_FIXTURE, "zip"), (FIXTURE_ROOT, "directory")], 401 + ) 402 + @pytest.mark.parametrize( 403 + "retention", 404 + [ 405 + RawRetentionDecision.DISCARD, 406 + RawRetentionDecision.RETAIN_PARSED, 407 + RawRetentionDecision.RETAIN_COMPLETE, 408 + ], 409 + ) 410 + def test_apple_health_retention_branches_for_zip_and_directory_inputs( 411 + tmp_path: Path, 412 + monkeypatch: pytest.MonkeyPatch, 413 + source_path: Path, 414 + source_label: str, 415 + retention: RawRetentionDecision, 416 + ) -> None: 417 + journal = tmp_path / f"journal-{source_label}-{retention.value}" 418 + import_id = f"20260103_12{len(source_label):02d}{len(retention.value):02d}" 419 + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) 420 + 421 + result = _process_with_approval( 422 + source_path, 423 + journal, 424 + import_id=import_id, 425 + dry_run=False, 426 + date_from="2026-01-02", 427 + date_to="2026-01-02", 428 + raw_retention_decision=retention.value, 429 + unparsed_sensitive_modalities_acknowledged=( 430 + True if retention == RawRetentionDecision.RETAIN_COMPLETE else None 431 + ), 432 + ) 433 + 434 + import_dir = journal / "imports" / import_id 435 + files = _relative_files(import_dir) 436 + shard = import_dir / "normalized" / "2026-01.jsonl" 437 + rows = _read_jsonl(shard) 438 + manifest = json.loads((import_dir / "manifest.json").read_text()) 439 + dedupe_row = get_health_dedupe_record(journal, rows[0]["dedupe_key"]) 440 + 441 + assert result.raw_retention == retention.value 442 + assert manifest["raw_retention"] == retention.value 443 + assert dedupe_row is not None 444 + 445 + if retention == RawRetentionDecision.DISCARD: 446 + assert not (import_dir / "raw").exists() 447 + assert all("raw_ref" not in row for row in rows) 448 + assert dedupe_row["raw_ref"] is None 449 + assert "raw/" not in json.dumps(manifest) 450 + return 451 + 452 + assert all(row["raw_ref"] == dedupe_row["raw_ref"] for row in rows[:1]) 453 + assert all("raw_ref" in row for row in rows) 454 + if retention == RawRetentionDecision.RETAIN_PARSED: 455 + assert "raw/export.xml" in files 456 + assert not any( 457 + "workout-routes" in file 458 + or "electrocardiograms" in file 459 + or file.endswith("export_cda.xml") 460 + for file in files 461 + ) 462 + assert rows[0]["raw_ref"].startswith(f"imports/{import_id}/raw/export.xml#") 463 + else: 464 + if source_label == "zip": 465 + assert f"raw/{ZIP_FIXTURE.name}" in files 466 + assert rows[0]["raw_ref"].startswith( 467 + f"imports/{import_id}/raw/{ZIP_FIXTURE.name}#" 468 + ) 469 + else: 470 + assert "raw/apple_health_export/export.xml" in files 471 + assert "raw/apple_health_export/workout-routes/synthetic-route.gpx" in files 472 + assert rows[0]["raw_ref"].startswith( 473 + f"imports/{import_id}/raw/apple_health_export/export.xml#" 474 + ) 475 + 476 + 477 + def test_apple_health_retain_parsed_excludes_cda_and_ecg_files( 478 + tmp_path: Path, 479 + monkeypatch: pytest.MonkeyPatch, 480 + ) -> None: 481 + journal = tmp_path / "journal" 482 + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) 483 + 484 + _process_with_approval( 485 + DTD_FIXTURE_ROOT, 486 + journal, 487 + import_id="20260412_120000", 488 + dry_run=False, 489 + raw_retention_decision=RawRetentionDecision.RETAIN_PARSED.value, 490 + ) 491 + 492 + files = _relative_files(journal / "imports" / "20260412_120000") 493 + 494 + assert "raw/export.xml" in files 495 + assert not any( 496 + "electrocardiograms" in file or file.endswith("export_cda.xml") 497 + for file in files 498 + ) 499 + 500 + 385 501 def test_apple_health_save_repairs_private_import_modes_under_permissive_umask( 386 502 tmp_path: Path, 387 503 monkeypatch: pytest.MonkeyPatch, ··· 420 536 Path(result.files_created[0]), 421 537 ): 422 538 assert _mode(file_path) == 0o600 539 + 540 + 541 + def test_apple_health_discard_update_preserves_historic_dedupe_raw_ref( 542 + tmp_path: Path, 543 + monkeypatch: pytest.MonkeyPatch, 544 + ) -> None: 545 + journal = tmp_path / "journal" 546 + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) 547 + 548 + _process_with_approval( 549 + FIXTURE_ROOT, 550 + journal, 551 + import_id="20260103_120000", 552 + dry_run=False, 553 + date_from="2026-01-02", 554 + date_to="2026-01-02", 555 + raw_retention_decision=RawRetentionDecision.RETAIN_PARSED.value, 556 + ) 557 + first_rows = _read_jsonl( 558 + journal / "imports" / "20260103_120000" / "normalized" / "2026-01.jsonl" 559 + ) 560 + first_glucose = next( 561 + row 562 + for row in first_rows 563 + if row["record_type"] == "HKQuantityTypeIdentifierBloodGlucose" 564 + ) 565 + historic_raw_ref = first_glucose["raw_ref"] 566 + 567 + _process_with_approval( 568 + FIXTURE_ROOT, 569 + journal, 570 + import_id="20260104_120000", 571 + dry_run=False, 572 + date_from="2026-01-02", 573 + date_to="2026-01-02", 574 + raw_retention_decision=RawRetentionDecision.DISCARD.value, 575 + ) 576 + second_rows = _read_jsonl( 577 + journal / "imports" / "20260104_120000" / "normalized" / "2026-01.jsonl" 578 + ) 579 + second_glucose = next( 580 + row 581 + for row in second_rows 582 + if row["record_type"] == "HKQuantityTypeIdentifierBloodGlucose" 583 + ) 584 + dedupe_row = get_health_dedupe_record(journal, second_glucose["dedupe_key"]) 585 + 586 + assert "raw_ref" not in second_glucose 587 + assert not (journal / "imports" / "20260104_120000" / "raw").exists() 588 + assert dedupe_row is not None 589 + assert dedupe_row["raw_ref"] == historic_raw_ref 590 + assert dedupe_row["raw_ref"].startswith("imports/20260103_120000/raw/export.xml#") 423 591 424 592 425 593 def test_workout_statistics_children_land_in_metadata_without_changing_dedupe(

+41

tests/test_health_dedupe.py

View file

··· 261 261 assert _mode(db_path.parent) == 0o700 262 262 263 263 264 + def test_batch_upsert_with_null_raw_ref_preserves_historic_raw_ref(tmp_path: Path): 265 + key = "sha256:historic-raw-ref" 266 + historic_raw_ref = "imports/20260103_120000/raw/export.xml#record-4" 267 + upsert_health_dedupe_records( 268 + tmp_path, 269 + [ 270 + HealthDedupeRecord( 271 + dedupe_key=key, 272 + source_family="apple_health", 273 + record_type="HKQuantityTypeIdentifierBloodGlucose", 274 + start_time="2026-01-02T12:30:00-07:00", 275 + first_import_id="20260103_120000", 276 + last_seen_import_id="20260103_120000", 277 + raw_ref=historic_raw_ref, 278 + ) 279 + ], 280 + ) 281 + 282 + upsert_health_dedupe_records( 283 + tmp_path, 284 + [ 285 + HealthDedupeRecord( 286 + dedupe_key=key, 287 + source_family="apple_health", 288 + record_type="HKQuantityTypeIdentifierBloodGlucose", 289 + start_time="2026-01-02T12:30:00-07:00", 290 + first_import_id="20260104_120000", 291 + last_seen_import_id="20260104_120000", 292 + normalized_ref="imports/20260104_120000/normalized/2026-01.jsonl#L1", 293 + raw_ref=None, 294 + ) 295 + ], 296 + ) 297 + 298 + row = get_health_dedupe_record(tmp_path, key) 299 + 300 + assert row is not None 301 + assert row["raw_ref"] == historic_raw_ref 302 + assert row["last_seen_import_id"] == "20260104_120000" 303 + 304 + 264 305 def test_upsert_health_dedupe_records_handles_duplicate_keys_in_batch(tmp_path: Path): 265 306 result = upsert_health_dedupe_records( 266 307 tmp_path,

+15

tests/test_importer_presave_gate.py

View file

··· 185 185 assert not (journal / "imports" / "20260102_123000").exists() 186 186 187 187 188 + def test_blocked_health_gate_creates_no_directories(tmp_path: Path): 189 + journal = tmp_path / "missing-journal" 190 + 191 + with pytest.raises(PreSaveGateError) as exc_info: 192 + enforce_pre_save_gate( 193 + "apple_health", 194 + dry_run=False, 195 + confirm_health_save=True, 196 + journal_root=journal, 197 + ) 198 + 199 + assert exc_info.value.to_dict()["gate_reason"] == "missing_approval_artifact" 200 + assert not journal.exists() 201 + 202 + 188 203 def test_cli_apple_health_save_missing_artifact_blocks_before_setup( 189 204 tmp_path: Path, 190 205 monkeypatch: pytest.MonkeyPatch,

+53

tests/test_oura_importer.py

View file

··· 1621 1621 assert manifest["source_type"] == SOURCE_OURA_API 1622 1622 assert manifest["entry_count"] == _SYNC_ROW_COUNT 1623 1623 assert manifest["days_affected"] == ["20260102", "20260103"] 1624 + assert manifest["raw_retention"] == RawRetentionDecision.RETAIN_PARSED.value 1624 1625 content_lines = (import_dir / "content_manifest.jsonl").read_text().splitlines() 1625 1626 assert json.loads(content_lines[0])["type"] == "health_normalized_month" 1626 1627 ··· 1666 1667 assert result["imported"] == 0 # updated 1667 1668 assert result["source_label"] == "Oura (API)" 1668 1669 assert "cron_hint" not in result 1670 + 1671 + 1672 + @pytest.mark.parametrize( 1673 + ("retention", "expect_raw_pages"), 1674 + [ 1675 + (RawRetentionDecision.RETAIN_PARSED, True), 1676 + (RawRetentionDecision.DISCARD, False), 1677 + ], 1678 + ) 1679 + def test_oura_sync_applies_raw_retention_choice( 1680 + tmp_path: Path, 1681 + monkeypatch, 1682 + retention: RawRetentionDecision, 1683 + expect_raw_pages: bool, 1684 + ) -> None: 1685 + journal = _use_journal(tmp_path, monkeypatch) 1686 + _write_sync_artifact( 1687 + journal, 1688 + _sync_artifact(journal, raw_retention_decision=retention.value), 1689 + ) 1690 + 1691 + result = oura.backend.sync( 1692 + journal, 1693 + dry_run=False, 1694 + confirm_health_save=True, 1695 + client=_canned_client(_fixture_transport()), 1696 + today=dt.date(2026, 1, 10), 1697 + ) 1698 + 1699 + import_dir = journal / "imports" / result["import_id"] 1700 + raw_dir = import_dir / "raw" 1701 + rows = [ 1702 + json.loads(line) 1703 + for line in (import_dir / "normalized" / "2026-01.jsonl") 1704 + .read_text() 1705 + .splitlines() 1706 + ] 1707 + manifest = json.loads((import_dir / "manifest.json").read_text()) 1708 + dedupe_row = get_health_dedupe_record(journal, rows[0]["dedupe_key"]) 1709 + 1710 + assert result["raw_retention"] == retention.value 1711 + assert manifest["raw_retention"] == retention.value 1712 + assert dedupe_row is not None 1713 + if expect_raw_pages: 1714 + assert (raw_dir / "oura" / "daily_readiness.jsonl").is_file() 1715 + assert rows[0]["raw_ref"].startswith(f"imports/{result['import_id']}/raw/oura#") 1716 + assert dedupe_row["raw_ref"] == rows[0]["raw_ref"] 1717 + else: 1718 + assert not raw_dir.exists() 1719 + assert all("raw_ref" not in row for row in rows) 1720 + assert dedupe_row["raw_ref"] is None 1721 + assert "raw/" not in json.dumps(manifest) 1669 1722 1670 1723 1671 1724 def test_oura_sync_private_modes_under_permissive_umask(

Configure Feed

Configure Feed