solpbc.org / solstone
personal memory agent
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

feat(import): one native-client staging contract (client_item_id, /meta, canonical source)

Evolve the import staging API in place so web, `sol import`, and native
clients stage through one strengthened contract.

- /save + /save-path now REQUIRE client_item_id (idempotency key). Staged
scan gives idempotent replay (same id + same content) vs 409
import_client_id_conflict (same id, different content). Replay of an
already-started item recommends do_not_start.
- Duplicates are terminal: content matching an imported manifest OR an
already-staged item returns status=duplicate / recommended_action=
do_not_start and is NOT staged again. /start refuses a saved terminal
duplicate regardless of force.
- /save returns a versioned summary: schema_version, status, replay, path,
timestamp, client_item_id, source (server-inferred canonical
audio|image|document|text), facet, setting, recommended_action, a metadata
bag (incl. flexible client bag), diagnostics (folds timestamp_detection_*),
and a duplicate object when applicable. /save-path shares the shape.
- /api/facet renamed to /api/meta (clean break, no alias).
- /start is saved-metadata-authority (ignores request source/facet/setting)
and routes via saved source_hint. recording/quick/apple never reach the
wire or the importer as source values.

Conservative PDF: canonical source reports "document" for PDFs as metadata,
but generic/quick PDF processing is unchanged (still the text path).

Intentional BREAKING changes to import.save, re-pinned via `make openapi`:
required client_item_id; top-level timestamp_detection_* moved under
diagnostics. Adds import.savePath + import.meta ops. Native-client (iOS)
owners must update.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

author
Jer Miller co-author
Claude Opus 4.8 (1M context) date (Jun 24, 2026, 8:44 PM -0600) commit 0a45a3bd parent 3571bca1
+2298 -301
+683 -24
docs/openapi/convey-clients.json
··· 63 63 "file_read_failed", 64 64 "health_report_failed", 65 65 "identity_busy", 66 + "import_client_id_conflict", 66 67 "import_conflict", 67 68 "import_metadata_failed", 68 69 "import_not_found", ··· 1304 1305 ] 1305 1306 } 1306 1307 }, 1308 + "/app/import/api/meta": { 1309 + "post": { 1310 + "description": "Update allowlisted metadata fields on a not-yet-started import.", 1311 + "operationId": "import.meta", 1312 + "requestBody": { 1313 + "content": { 1314 + "application/json": { 1315 + "example": { 1316 + "facet": "work", 1317 + "path": "/journal/imports/20260618_143022/source.m4a" 1318 + }, 1319 + "schema": { 1320 + "additionalProperties": true, 1321 + "properties": { 1322 + "client": { 1323 + "type": "object" 1324 + }, 1325 + "facet": { 1326 + "type": "string" 1327 + }, 1328 + "imported_via": { 1329 + "type": "string" 1330 + }, 1331 + "mime_type": { 1332 + "type": "string" 1333 + }, 1334 + "observer_handle": { 1335 + "type": "string" 1336 + }, 1337 + "original_filename": { 1338 + "type": "string" 1339 + }, 1340 + "path": { 1341 + "type": "string" 1342 + }, 1343 + "setting": { 1344 + "type": "string" 1345 + }, 1346 + "source_hint": { 1347 + "type": "string" 1348 + } 1349 + }, 1350 + "required": [ 1351 + "path" 1352 + ], 1353 + "type": "object" 1354 + } 1355 + } 1356 + }, 1357 + "required": true 1358 + }, 1359 + "responses": { 1360 + "200": { 1361 + "content": { 1362 + "application/json": { 1363 + "example": { 1364 + "path": "/journal/imports/20260618_143022/source.m4a", 1365 + "status": "ok", 1366 + "timestamp": "20260618_143022", 1367 + "updated": { 1368 + "facet": "work" 1369 + } 1370 + }, 1371 + "schema": { 1372 + "additionalProperties": true, 1373 + "properties": { 1374 + "path": { 1375 + "type": "string" 1376 + }, 1377 + "status": { 1378 + "type": "string" 1379 + }, 1380 + "timestamp": { 1381 + "type": "string" 1382 + }, 1383 + "updated": { 1384 + "type": "object" 1385 + } 1386 + }, 1387 + "required": [ 1388 + "status", 1389 + "path", 1390 + "timestamp", 1391 + "updated" 1392 + ], 1393 + "type": "object" 1394 + } 1395 + } 1396 + }, 1397 + "description": "Import metadata updated." 1398 + }, 1399 + "400": { 1400 + "content": { 1401 + "application/json": { 1402 + "schema": { 1403 + "$ref": "#/components/schemas/Error" 1404 + } 1405 + } 1406 + }, 1407 + "description": "The import path was missing or the import state is terminal.", 1408 + "x-reason-codes": [ 1409 + "invalid_operation_for_state", 1410 + "missing_required_field" 1411 + ] 1412 + }, 1413 + "403": { 1414 + "content": { 1415 + "application/json": { 1416 + "schema": { 1417 + "$ref": "#/components/schemas/Error" 1418 + } 1419 + } 1420 + }, 1421 + "description": "Access gate rejected a revoked paired-link identity.", 1422 + "x-reason-codes": [ 1423 + "pl_revoked" 1424 + ] 1425 + }, 1426 + "404": { 1427 + "content": { 1428 + "application/json": { 1429 + "schema": { 1430 + "$ref": "#/components/schemas/Error" 1431 + } 1432 + } 1433 + }, 1434 + "description": "Import metadata was not found.", 1435 + "x-reason-codes": [ 1436 + "import_not_found" 1437 + ] 1438 + }, 1439 + "500": { 1440 + "content": { 1441 + "application/json": { 1442 + "schema": { 1443 + "$ref": "#/components/schemas/Error" 1444 + } 1445 + } 1446 + }, 1447 + "description": "Import metadata could not be read or updated.", 1448 + "x-reason-codes": [ 1449 + "import_metadata_failed" 1450 + ] 1451 + } 1452 + }, 1453 + "summary": "Update import metadata", 1454 + "tags": [ 1455 + "import" 1456 + ] 1457 + } 1458 + }, 1307 1459 "/app/import/api/save": { 1308 1460 "post": { 1309 - "description": "Save an uploaded import file or pasted text into imports staging. Submit either file or text.", 1461 + "description": "Save an uploaded import file or pasted text into imports staging. Submit either file or text. client_item_id is required for idempotent native-client staging.", 1310 1462 "operationId": "import.save", 1311 1463 "requestBody": { 1312 1464 "content": { ··· 1314 1466 "schema": { 1315 1467 "additionalProperties": true, 1316 1468 "properties": { 1469 + "client": { 1470 + "type": "object" 1471 + }, 1472 + "client_item_id": { 1473 + "type": "string" 1474 + }, 1317 1475 "deterministic_only": { 1318 1476 "type": "boolean" 1319 1477 }, ··· 1333 1491 "setting": { 1334 1492 "type": "string" 1335 1493 }, 1494 + "source_hint": { 1495 + "type": "string" 1496 + }, 1336 1497 "text": { 1337 1498 "type": "string" 1338 1499 } 1339 1500 }, 1501 + "required": [ 1502 + "client_item_id" 1503 + ], 1340 1504 "type": "object" 1341 1505 } 1342 1506 } 1343 1507 }, 1344 - "description": "Multipart body with either file or text." 1508 + "description": "Multipart body with either file or text.", 1509 + "required": true 1345 1510 }, 1346 1511 "responses": { 1347 1512 "200": { 1348 1513 "content": { 1349 1514 "application/json": { 1350 1515 "example": { 1516 + "client_item_id": "ios-item-4f8b", 1517 + "diagnostics": { 1518 + "source_inference": "extension", 1519 + "timestamp_detection_method": "upload_fallback", 1520 + "timestamp_detection_model_called": false, 1521 + "timestamp_detection_no_match_reason": null 1522 + }, 1351 1523 "facet": null, 1352 - "path": "/journal/imports/20260618_143022/source.txt", 1524 + "metadata": { 1525 + "client": {}, 1526 + "imported_via": "ios", 1527 + "mime_type": "audio/mp4", 1528 + "observer_handle": null, 1529 + "original_filename": "source.m4a", 1530 + "source_hint": null 1531 + }, 1532 + "path": "/journal/imports/20260618_143022/source.m4a", 1533 + "recommended_action": "start", 1534 + "replay": false, 1535 + "schema_version": 1, 1353 1536 "setting": null, 1354 - "timestamp": "20260618_143022", 1355 - "timestamp_detection_method": "deterministic", 1356 - "timestamp_detection_model_called": false, 1357 - "timestamp_detection_no_match_reason": null 1537 + "source": "audio", 1538 + "status": "staged", 1539 + "timestamp": "20260618_143022" 1358 1540 }, 1359 1541 "schema": { 1360 1542 "additionalProperties": true, 1361 1543 "properties": { 1544 + "client_item_id": { 1545 + "type": "string" 1546 + }, 1547 + "diagnostics": { 1548 + "additionalProperties": true, 1549 + "properties": { 1550 + "source_inference": { 1551 + "enum": [ 1552 + "extension", 1553 + "content_type", 1554 + "default" 1555 + ], 1556 + "type": "string" 1557 + }, 1558 + "timestamp_detection_method": { 1559 + "type": "string" 1560 + }, 1561 + "timestamp_detection_model_called": { 1562 + "type": "boolean" 1563 + }, 1564 + "timestamp_detection_no_match_reason": { 1565 + "type": [ 1566 + "string", 1567 + "null" 1568 + ] 1569 + } 1570 + }, 1571 + "required": [ 1572 + "timestamp_detection_method", 1573 + "timestamp_detection_model_called", 1574 + "timestamp_detection_no_match_reason", 1575 + "source_inference" 1576 + ], 1577 + "type": "object" 1578 + }, 1579 + "duplicate": { 1580 + "additionalProperties": true, 1581 + "properties": { 1582 + "entry_count": { 1583 + "type": [ 1584 + "integer", 1585 + "null" 1586 + ] 1587 + }, 1588 + "import_id": { 1589 + "type": "string" 1590 + }, 1591 + "imported_at": { 1592 + "type": [ 1593 + "string", 1594 + "null" 1595 + ] 1596 + }, 1597 + "state": { 1598 + "enum": [ 1599 + "imported", 1600 + "staged" 1601 + ], 1602 + "type": "string" 1603 + } 1604 + }, 1605 + "required": [ 1606 + "import_id", 1607 + "imported_at", 1608 + "entry_count", 1609 + "state" 1610 + ], 1611 + "type": "object" 1612 + }, 1362 1613 "facet": { 1363 1614 "type": [ 1364 1615 "string", 1365 1616 "null" 1366 1617 ] 1367 1618 }, 1619 + "metadata": { 1620 + "additionalProperties": true, 1621 + "properties": { 1622 + "client": { 1623 + "additionalProperties": true, 1624 + "type": "object" 1625 + }, 1626 + "imported_via": { 1627 + "type": [ 1628 + "string", 1629 + "null" 1630 + ] 1631 + }, 1632 + "mime_type": { 1633 + "type": [ 1634 + "string", 1635 + "null" 1636 + ] 1637 + }, 1638 + "observer_handle": { 1639 + "type": [ 1640 + "string", 1641 + "null" 1642 + ] 1643 + }, 1644 + "original_filename": { 1645 + "type": [ 1646 + "string", 1647 + "null" 1648 + ] 1649 + }, 1650 + "source_hint": { 1651 + "type": [ 1652 + "string", 1653 + "null" 1654 + ] 1655 + } 1656 + }, 1657 + "required": [ 1658 + "original_filename", 1659 + "mime_type", 1660 + "imported_via", 1661 + "observer_handle", 1662 + "source_hint", 1663 + "client" 1664 + ], 1665 + "type": "object" 1666 + }, 1368 1667 "path": { 1369 1668 "type": "string" 1370 1669 }, 1670 + "recommended_action": { 1671 + "enum": [ 1672 + "start", 1673 + "do_not_start" 1674 + ], 1675 + "type": "string" 1676 + }, 1677 + "replay": { 1678 + "type": "boolean" 1679 + }, 1680 + "schema_version": { 1681 + "type": "integer" 1682 + }, 1371 1683 "setting": { 1372 1684 "type": [ 1373 1685 "string", 1374 1686 "null" 1375 1687 ] 1376 1688 }, 1689 + "source": { 1690 + "enum": [ 1691 + "audio", 1692 + "image", 1693 + "document", 1694 + "text" 1695 + ], 1696 + "type": "string" 1697 + }, 1698 + "status": { 1699 + "enum": [ 1700 + "staged", 1701 + "duplicate" 1702 + ], 1703 + "type": "string" 1704 + }, 1377 1705 "timestamp": { 1378 1706 "type": "string" 1707 + } 1708 + }, 1709 + "required": [ 1710 + "schema_version", 1711 + "status", 1712 + "replay", 1713 + "path", 1714 + "timestamp", 1715 + "client_item_id", 1716 + "source", 1717 + "facet", 1718 + "setting", 1719 + "recommended_action", 1720 + "metadata", 1721 + "diagnostics" 1722 + ], 1723 + "type": "object" 1724 + } 1725 + } 1726 + }, 1727 + "description": "Import source staged, replayed, or identified as a duplicate." 1728 + }, 1729 + "400": { 1730 + "content": { 1731 + "application/json": { 1732 + "schema": { 1733 + "$ref": "#/components/schemas/Error" 1734 + } 1735 + } 1736 + }, 1737 + "description": "Required fields were missing or neither file nor text was supplied.", 1738 + "x-reason-codes": [ 1739 + "ingest_no_files", 1740 + "missing_required_field" 1741 + ] 1742 + }, 1743 + "403": { 1744 + "content": { 1745 + "application/json": { 1746 + "schema": { 1747 + "$ref": "#/components/schemas/Error" 1748 + } 1749 + } 1750 + }, 1751 + "description": "Access gate rejected a revoked paired-link identity.", 1752 + "x-reason-codes": [ 1753 + "pl_revoked" 1754 + ] 1755 + }, 1756 + "409": { 1757 + "content": { 1758 + "application/json": { 1759 + "schema": { 1760 + "$ref": "#/components/schemas/Error" 1761 + } 1762 + } 1763 + }, 1764 + "description": "client_item_id already names different staged content.", 1765 + "x-reason-codes": [ 1766 + "import_client_id_conflict" 1767 + ] 1768 + } 1769 + }, 1770 + "summary": "Save import source", 1771 + "tags": [ 1772 + "import" 1773 + ] 1774 + } 1775 + }, 1776 + "/app/import/api/save-path": { 1777 + "post": { 1778 + "description": "Register a local filesystem path for import staging using the same idempotent summary response as import.save.", 1779 + "operationId": "import.savePath", 1780 + "requestBody": { 1781 + "content": { 1782 + "application/json": { 1783 + "example": { 1784 + "client": {}, 1785 + "client_item_id": "ios-path-1357", 1786 + "path": "/Users/sol/Documents/Notes", 1787 + "source_hint": "obsidian" 1788 + }, 1789 + "schema": { 1790 + "additionalProperties": true, 1791 + "properties": { 1792 + "client": { 1793 + "type": "object" 1794 + }, 1795 + "client_item_id": { 1796 + "type": "string" 1797 + }, 1798 + "facet": { 1799 + "type": "string" 1800 + }, 1801 + "imported_via": { 1802 + "type": "string" 1803 + }, 1804 + "observer_handle": { 1805 + "type": "string" 1806 + }, 1807 + "path": { 1808 + "type": "string" 1809 + }, 1810 + "setting": { 1811 + "type": "string" 1812 + }, 1813 + "source_hint": { 1814 + "type": "string" 1815 + } 1816 + }, 1817 + "required": [ 1818 + "client_item_id", 1819 + "path" 1820 + ], 1821 + "type": "object" 1822 + } 1823 + } 1824 + }, 1825 + "required": true 1826 + }, 1827 + "responses": { 1828 + "200": { 1829 + "content": { 1830 + "application/json": { 1831 + "example": { 1832 + "client_item_id": "ios-path-1357", 1833 + "diagnostics": { 1834 + "source_inference": "extension", 1835 + "timestamp_detection_method": "upload_fallback", 1836 + "timestamp_detection_model_called": false, 1837 + "timestamp_detection_no_match_reason": null 1838 + }, 1839 + "facet": null, 1840 + "metadata": { 1841 + "client": {}, 1842 + "imported_via": "ios", 1843 + "mime_type": null, 1844 + "observer_handle": null, 1845 + "original_filename": "Notes", 1846 + "source_hint": "obsidian" 1847 + }, 1848 + "path": "/Users/sol/Documents/Notes", 1849 + "recommended_action": "start", 1850 + "replay": false, 1851 + "schema_version": 1, 1852 + "setting": null, 1853 + "source": "text", 1854 + "status": "staged", 1855 + "timestamp": "20260618_143022" 1856 + }, 1857 + "schema": { 1858 + "additionalProperties": true, 1859 + "properties": { 1860 + "client_item_id": { 1861 + "type": "string" 1379 1862 }, 1380 - "timestamp_detection_method": { 1381 - "description": "Timestamp detection method: deterministic, model, upload_fallback, or explicit.", 1863 + "diagnostics": { 1864 + "additionalProperties": true, 1865 + "properties": { 1866 + "source_inference": { 1867 + "enum": [ 1868 + "extension", 1869 + "content_type", 1870 + "default" 1871 + ], 1872 + "type": "string" 1873 + }, 1874 + "timestamp_detection_method": { 1875 + "type": "string" 1876 + }, 1877 + "timestamp_detection_model_called": { 1878 + "type": "boolean" 1879 + }, 1880 + "timestamp_detection_no_match_reason": { 1881 + "type": [ 1882 + "string", 1883 + "null" 1884 + ] 1885 + } 1886 + }, 1887 + "required": [ 1888 + "timestamp_detection_method", 1889 + "timestamp_detection_model_called", 1890 + "timestamp_detection_no_match_reason", 1891 + "source_inference" 1892 + ], 1893 + "type": "object" 1894 + }, 1895 + "duplicate": { 1896 + "additionalProperties": true, 1897 + "properties": { 1898 + "entry_count": { 1899 + "type": [ 1900 + "integer", 1901 + "null" 1902 + ] 1903 + }, 1904 + "import_id": { 1905 + "type": "string" 1906 + }, 1907 + "imported_at": { 1908 + "type": [ 1909 + "string", 1910 + "null" 1911 + ] 1912 + }, 1913 + "state": { 1914 + "enum": [ 1915 + "imported", 1916 + "staged" 1917 + ], 1918 + "type": "string" 1919 + } 1920 + }, 1921 + "required": [ 1922 + "import_id", 1923 + "imported_at", 1924 + "entry_count", 1925 + "state" 1926 + ], 1927 + "type": "object" 1928 + }, 1929 + "facet": { 1930 + "type": [ 1931 + "string", 1932 + "null" 1933 + ] 1934 + }, 1935 + "metadata": { 1936 + "additionalProperties": true, 1937 + "properties": { 1938 + "client": { 1939 + "additionalProperties": true, 1940 + "type": "object" 1941 + }, 1942 + "imported_via": { 1943 + "type": [ 1944 + "string", 1945 + "null" 1946 + ] 1947 + }, 1948 + "mime_type": { 1949 + "type": [ 1950 + "string", 1951 + "null" 1952 + ] 1953 + }, 1954 + "observer_handle": { 1955 + "type": [ 1956 + "string", 1957 + "null" 1958 + ] 1959 + }, 1960 + "original_filename": { 1961 + "type": [ 1962 + "string", 1963 + "null" 1964 + ] 1965 + }, 1966 + "source_hint": { 1967 + "type": [ 1968 + "string", 1969 + "null" 1970 + ] 1971 + } 1972 + }, 1973 + "required": [ 1974 + "original_filename", 1975 + "mime_type", 1976 + "imported_via", 1977 + "observer_handle", 1978 + "source_hint", 1979 + "client" 1980 + ], 1981 + "type": "object" 1982 + }, 1983 + "path": { 1382 1984 "type": "string" 1383 1985 }, 1384 - "timestamp_detection_model_called": { 1986 + "recommended_action": { 1987 + "enum": [ 1988 + "start", 1989 + "do_not_start" 1990 + ], 1991 + "type": "string" 1992 + }, 1993 + "replay": { 1385 1994 "type": "boolean" 1386 1995 }, 1387 - "timestamp_detection_no_match_reason": { 1996 + "schema_version": { 1997 + "type": "integer" 1998 + }, 1999 + "setting": { 1388 2000 "type": [ 1389 2001 "string", 1390 2002 "null" 1391 2003 ] 2004 + }, 2005 + "source": { 2006 + "enum": [ 2007 + "audio", 2008 + "image", 2009 + "document", 2010 + "text" 2011 + ], 2012 + "type": "string" 2013 + }, 2014 + "status": { 2015 + "enum": [ 2016 + "staged", 2017 + "duplicate" 2018 + ], 2019 + "type": "string" 2020 + }, 2021 + "timestamp": { 2022 + "type": "string" 1392 2023 } 1393 2024 }, 1394 2025 "required": [ 2026 + "schema_version", 2027 + "status", 2028 + "replay", 1395 2029 "path", 1396 2030 "timestamp", 2031 + "client_item_id", 2032 + "source", 1397 2033 "facet", 1398 2034 "setting", 1399 - "timestamp_detection_method", 1400 - "timestamp_detection_model_called", 1401 - "timestamp_detection_no_match_reason" 2035 + "recommended_action", 2036 + "metadata", 2037 + "diagnostics" 1402 2038 ], 1403 2039 "type": "object" 1404 2040 } 1405 2041 } 1406 2042 }, 1407 - "description": "Import source saved." 2043 + "description": "Import path staged, replayed, or identified as a duplicate." 1408 2044 }, 1409 2045 "400": { 1410 2046 "content": { ··· 1414 2050 } 1415 2051 } 1416 2052 }, 1417 - "description": "Neither file nor text was supplied.", 2053 + "description": "client_item_id or path was missing.", 1418 2054 "x-reason-codes": [ 1419 - "ingest_no_files" 2055 + "missing_required_field" 1420 2056 ] 1421 2057 }, 1422 2058 "403": { ··· 1431 2067 "x-reason-codes": [ 1432 2068 "pl_revoked" 1433 2069 ] 2070 + }, 2071 + "404": { 2072 + "content": { 2073 + "application/json": { 2074 + "schema": { 2075 + "$ref": "#/components/schemas/Error" 2076 + } 2077 + } 2078 + }, 2079 + "description": "The local path did not exist.", 2080 + "x-reason-codes": [ 2081 + "file_not_found" 2082 + ] 2083 + }, 2084 + "409": { 2085 + "content": { 2086 + "application/json": { 2087 + "schema": { 2088 + "$ref": "#/components/schemas/Error" 2089 + } 2090 + } 2091 + }, 2092 + "description": "client_item_id already names different staged content.", 2093 + "x-reason-codes": [ 2094 + "import_client_id_conflict" 2095 + ] 1434 2096 } 1435 2097 }, 1436 - "summary": "Save import source", 2098 + "summary": "Save import source path", 1437 2099 "tags": [ 1438 2100 "import" 1439 2101 ] ··· 1441 2103 }, 1442 2104 "/app/import/api/start": { 1443 2105 "post": { 1444 - "description": "Start processing a previously saved import source.", 2106 + "description": "Start processing a previously saved import source. Saved import metadata is authoritative for facet, setting, and source routing.", 1445 2107 "operationId": "import.start", 1446 2108 "requestBody": { 1447 2109 "content": { ··· 1449 2111 "example": { 1450 2112 "force": false, 1451 2113 "path": "/journal/imports/20260618_143022/source.txt", 1452 - "source": "manual", 1453 2114 "timestamp": "20260618_143022" 1454 2115 }, 1455 2116 "schema": { ··· 1459 2120 "type": "boolean" 1460 2121 }, 1461 2122 "path": { 1462 - "type": "string" 1463 - }, 1464 - "source": { 1465 2123 "type": "string" 1466 2124 }, 1467 2125 "timestamp": { ··· 1514 2172 } 1515 2173 } 1516 2174 }, 1517 - "description": "Path or timestamp was missing.", 2175 + "description": "Path or timestamp was missing, or the import is terminal.", 1518 2176 "x-reason-codes": [ 2177 + "invalid_operation_for_state", 1519 2178 "missing_required_field" 1520 2179 ] 1521 2180 },
+251 -47
solstone/apps/import/contract.py
··· 21 21 22 22 23 23 _NULLABLE_STRING = {"type": ["string", "null"]} 24 + _NULLABLE_INTEGER = {"type": ["integer", "null"]} 25 + _FREE_OBJECT = {"type": "object", "additionalProperties": True} 26 + _STATUS_SCHEMA = {"type": "string", "enum": ["staged", "duplicate"]} 27 + _SOURCE_SCHEMA = {"type": "string", "enum": ["audio", "image", "document", "text"]} 28 + _ACTION_SCHEMA = {"type": "string", "enum": ["start", "do_not_start"]} 29 + _SOURCE_INFERENCE_SCHEMA = { 30 + "type": "string", 31 + "enum": ["extension", "content_type", "default"], 32 + } 33 + _METADATA_SCHEMA = { 34 + "type": "object", 35 + "additionalProperties": True, 36 + "properties": { 37 + "original_filename": _NULLABLE_STRING, 38 + "mime_type": _NULLABLE_STRING, 39 + "imported_via": _NULLABLE_STRING, 40 + "observer_handle": _NULLABLE_STRING, 41 + "source_hint": _NULLABLE_STRING, 42 + "client": _FREE_OBJECT, 43 + }, 44 + "required": [ 45 + "original_filename", 46 + "mime_type", 47 + "imported_via", 48 + "observer_handle", 49 + "source_hint", 50 + "client", 51 + ], 52 + } 53 + _DIAGNOSTICS_SCHEMA = { 54 + "type": "object", 55 + "additionalProperties": True, 56 + "properties": { 57 + "timestamp_detection_method": {"type": "string"}, 58 + "timestamp_detection_model_called": {"type": "boolean"}, 59 + "timestamp_detection_no_match_reason": _NULLABLE_STRING, 60 + "source_inference": _SOURCE_INFERENCE_SCHEMA, 61 + }, 62 + "required": [ 63 + "timestamp_detection_method", 64 + "timestamp_detection_model_called", 65 + "timestamp_detection_no_match_reason", 66 + "source_inference", 67 + ], 68 + } 69 + _DUPLICATE_SCHEMA = { 70 + "type": "object", 71 + "additionalProperties": True, 72 + "properties": { 73 + "import_id": {"type": "string"}, 74 + "imported_at": _NULLABLE_STRING, 75 + "entry_count": _NULLABLE_INTEGER, 76 + "state": {"type": "string", "enum": ["imported", "staged"]}, 77 + }, 78 + "required": ["import_id", "imported_at", "entry_count", "state"], 79 + } 80 + 81 + _SAVE_RESPONSE_FIELDS = ( 82 + FieldSpec("schema_version", "integer", required=True), 83 + FieldSpec("status", "string", required=True, raw_schema=_STATUS_SCHEMA), 84 + FieldSpec("replay", "boolean", required=True), 85 + FieldSpec("path", "string", required=True), 86 + FieldSpec("timestamp", "string", required=True), 87 + FieldSpec("client_item_id", "string", required=True), 88 + FieldSpec("source", "string", required=True, raw_schema=_SOURCE_SCHEMA), 89 + FieldSpec("facet", "string", required=True, raw_schema=_NULLABLE_STRING), 90 + FieldSpec("setting", "string", required=True, raw_schema=_NULLABLE_STRING), 91 + FieldSpec( 92 + "recommended_action", 93 + "string", 94 + required=True, 95 + raw_schema=_ACTION_SCHEMA, 96 + ), 97 + FieldSpec("metadata", "object", required=True, raw_schema=_METADATA_SCHEMA), 98 + FieldSpec("diagnostics", "object", required=True, raw_schema=_DIAGNOSTICS_SCHEMA), 99 + FieldSpec("duplicate", "object", raw_schema=_DUPLICATE_SCHEMA), 100 + ) 101 + 102 + _SAVE_RESPONSE_EXAMPLE = { 103 + "schema_version": 1, 104 + "status": "staged", 105 + "replay": False, 106 + "path": "/journal/imports/20260618_143022/source.m4a", 107 + "timestamp": "20260618_143022", 108 + "client_item_id": "ios-item-4f8b", 109 + "source": "audio", 110 + "facet": None, 111 + "setting": None, 112 + "recommended_action": "start", 113 + "metadata": { 114 + "original_filename": "source.m4a", 115 + "mime_type": "audio/mp4", 116 + "imported_via": "ios", 117 + "observer_handle": None, 118 + "source_hint": None, 119 + "client": {}, 120 + }, 121 + "diagnostics": { 122 + "timestamp_detection_method": "upload_fallback", 123 + "timestamp_detection_model_called": False, 124 + "timestamp_detection_no_match_reason": None, 125 + "source_inference": "extension", 126 + }, 127 + } 24 128 25 129 26 130 OPERATIONS: list[OperationSpec] = [ ··· 31 135 summary="Save import source", 32 136 description=( 33 137 "Save an uploaded import file or pasted text into imports staging. " 34 - "Submit either file or text." 138 + "Submit either file or text. client_item_id is required for " 139 + "idempotent native-client staging." 35 140 ), 36 141 request=RequestSpec( 37 142 content_type="multipart/form-data", 38 143 fields=( 144 + FieldSpec("client_item_id", "string", required=True), 39 145 FieldSpec( 40 146 "file", 41 147 "string", ··· 44 150 FieldSpec("text", "string"), 45 151 FieldSpec("facet", "string"), 46 152 FieldSpec("setting", "string"), 153 + FieldSpec("source_hint", "string"), 47 154 FieldSpec("imported_via", "string"), 48 155 FieldSpec("observer_handle", "string"), 49 156 FieldSpec("deterministic_only", "boolean"), 157 + FieldSpec("client", "object"), 50 158 ), 51 159 description="Multipart body with either file or text.", 52 160 ), 53 161 responses=( 54 162 ResponseSpec( 55 163 status=200, 56 - description="Import source saved.", 164 + description=( 165 + "Import source staged, replayed, or identified as a duplicate." 166 + ), 167 + named_fields=_SAVE_RESPONSE_FIELDS, 168 + example=_SAVE_RESPONSE_EXAMPLE, 169 + ), 170 + _json_error( 171 + 400, 172 + ("ingest_no_files", "missing_required_field"), 173 + "Required fields were missing or neither file nor text was supplied.", 174 + ), 175 + _json_error( 176 + 409, 177 + ("import_client_id_conflict",), 178 + "client_item_id already names different staged content.", 179 + ), 180 + _json_error( 181 + 403, 182 + ("pl_revoked",), 183 + "Access gate rejected a revoked paired-link identity.", 184 + ), 185 + ), 186 + ), 187 + OperationSpec( 188 + operation_id="import.savePath", 189 + method="POST", 190 + rule="/app/import/api/save-path", 191 + summary="Save import source path", 192 + description=( 193 + "Register a local filesystem path for import staging using the same " 194 + "idempotent summary response as import.save." 195 + ), 196 + request=RequestSpec( 197 + fields=( 198 + FieldSpec("client_item_id", "string", required=True), 199 + FieldSpec("path", "string", required=True), 200 + FieldSpec("facet", "string"), 201 + FieldSpec("setting", "string"), 202 + FieldSpec("source_hint", "string"), 203 + FieldSpec("imported_via", "string"), 204 + FieldSpec("observer_handle", "string"), 205 + FieldSpec("client", "object"), 206 + ), 207 + example={ 208 + "client_item_id": "ios-path-1357", 209 + "path": "/Users/sol/Documents/Notes", 210 + "source_hint": "obsidian", 211 + "client": {}, 212 + }, 213 + ), 214 + responses=( 215 + ResponseSpec( 216 + status=200, 217 + description=( 218 + "Import path staged, replayed, or identified as a duplicate." 219 + ), 220 + named_fields=_SAVE_RESPONSE_FIELDS, 221 + example={ 222 + **_SAVE_RESPONSE_EXAMPLE, 223 + "path": "/Users/sol/Documents/Notes", 224 + "client_item_id": "ios-path-1357", 225 + "source": "text", 226 + "metadata": { 227 + **_SAVE_RESPONSE_EXAMPLE["metadata"], 228 + "original_filename": "Notes", 229 + "mime_type": None, 230 + "source_hint": "obsidian", 231 + }, 232 + }, 233 + ), 234 + _json_error( 235 + 400, 236 + ("missing_required_field",), 237 + "client_item_id or path was missing.", 238 + ), 239 + _json_error( 240 + 404, 241 + ("file_not_found",), 242 + "The local path did not exist.", 243 + ), 244 + _json_error( 245 + 409, 246 + ("import_client_id_conflict",), 247 + "client_item_id already names different staged content.", 248 + ), 249 + _json_error( 250 + 403, 251 + ("pl_revoked",), 252 + "Access gate rejected a revoked paired-link identity.", 253 + ), 254 + ), 255 + ), 256 + OperationSpec( 257 + operation_id="import.meta", 258 + method="POST", 259 + rule="/app/import/api/meta", 260 + summary="Update import metadata", 261 + description="Update allowlisted metadata fields on a not-yet-started import.", 262 + request=RequestSpec( 263 + fields=( 264 + FieldSpec("path", "string", required=True), 265 + FieldSpec("facet", "string"), 266 + FieldSpec("setting", "string"), 267 + FieldSpec("original_filename", "string"), 268 + FieldSpec("mime_type", "string"), 269 + FieldSpec("source_hint", "string"), 270 + FieldSpec("observer_handle", "string"), 271 + FieldSpec("imported_via", "string"), 272 + FieldSpec("client", "object"), 273 + ), 274 + example={ 275 + "path": "/journal/imports/20260618_143022/source.m4a", 276 + "facet": "work", 277 + }, 278 + ), 279 + responses=( 280 + ResponseSpec( 281 + status=200, 282 + description="Import metadata updated.", 57 283 named_fields=( 284 + FieldSpec("status", "string", required=True), 58 285 FieldSpec("path", "string", required=True), 59 286 FieldSpec("timestamp", "string", required=True), 60 - FieldSpec( 61 - "facet", 62 - "string", 63 - required=True, 64 - raw_schema=_NULLABLE_STRING, 65 - ), 66 - FieldSpec( 67 - "setting", 68 - "string", 69 - required=True, 70 - raw_schema=_NULLABLE_STRING, 71 - ), 72 - FieldSpec( 73 - "timestamp_detection_method", 74 - "string", 75 - required=True, 76 - description=( 77 - "Timestamp detection method: deterministic, model, " 78 - "upload_fallback, or explicit." 79 - ), 80 - ), 81 - FieldSpec( 82 - "timestamp_detection_model_called", 83 - "boolean", 84 - required=True, 85 - ), 86 - FieldSpec( 87 - "timestamp_detection_no_match_reason", 88 - "string", 89 - required=True, 90 - raw_schema=_NULLABLE_STRING, 91 - ), 287 + FieldSpec("updated", "object", required=True), 92 288 ), 93 289 example={ 94 - "path": "/journal/imports/20260618_143022/source.txt", 290 + "status": "ok", 291 + "path": "/journal/imports/20260618_143022/source.m4a", 95 292 "timestamp": "20260618_143022", 96 - "facet": None, 97 - "setting": None, 98 - "timestamp_detection_method": "deterministic", 99 - "timestamp_detection_model_called": False, 100 - "timestamp_detection_no_match_reason": None, 293 + "updated": {"facet": "work"}, 101 294 }, 102 295 ), 103 296 _json_error( 104 297 400, 105 - ("ingest_no_files",), 106 - "Neither file nor text was supplied.", 298 + ("invalid_operation_for_state", "missing_required_field"), 299 + "The import path was missing or the import state is terminal.", 300 + ), 301 + _json_error( 302 + 404, 303 + ("import_not_found",), 304 + "Import metadata was not found.", 305 + ), 306 + _json_error( 307 + 500, 308 + ("import_metadata_failed",), 309 + "Import metadata could not be read or updated.", 107 310 ), 108 311 _json_error( 109 312 403, ··· 117 320 method="POST", 118 321 rule="/app/import/api/start", 119 322 summary="Start import", 120 - description="Start processing a previously saved import source.", 323 + description=( 324 + "Start processing a previously saved import source. Saved import " 325 + "metadata is authoritative for facet, setting, and source routing." 326 + ), 121 327 request=RequestSpec( 122 328 fields=( 123 329 FieldSpec("path", "string", required=True), 124 330 FieldSpec("timestamp", "string", required=True), 125 - FieldSpec("source", "string"), 126 331 FieldSpec("force", "boolean"), 127 332 ), 128 333 example={ 129 334 "path": "/journal/imports/20260618_143022/source.txt", 130 335 "timestamp": "20260618_143022", 131 - "source": "manual", 132 336 "force": False, 133 337 }, 134 338 ), ··· 144 348 ), 145 349 _json_error( 146 350 400, 147 - ("missing_required_field",), 148 - "Path or timestamp was missing.", 351 + ("invalid_operation_for_state", "missing_required_field"), 352 + "Path or timestamp was missing, or the import is terminal.", 149 353 ), 150 354 _json_error( 151 355 404,
+440 -134
solstone/apps/import/routes.py
··· 4 4 from __future__ import annotations 5 5 6 6 import json 7 - import logging 8 7 import re 9 8 import time 9 + from datetime import datetime 10 10 from pathlib import Path 11 11 from typing import Any 12 12 ··· 17 17 from solstone.convey import emit, state 18 18 from solstone.convey.reasons import ( 19 19 FILE_NOT_FOUND, 20 + IMPORT_CLIENT_ID_CONFLICT, 20 21 IMPORT_CONFLICT, 21 22 IMPORT_METADATA_FAILED, 22 23 IMPORT_NOT_FOUND, 23 24 INGEST_NO_FILES, 25 + INVALID_OPERATION_FOR_STATE, 24 26 INVALID_REQUEST_VALUE, 25 27 JOURNAL_SOURCE_PROBLEM, 26 28 MISSING_REQUIRED_FIELD, ··· 32 34 success_response, 33 35 ) 34 36 from solstone.think.detect_created import detect_created, resolve_created_deterministic 37 + from solstone.think.importers.shared import find_manifest_by_hash, hash_source 35 38 from solstone.think.importers.utils import ( 36 39 build_import_info, 40 + find_staged_by_client_item_id, 41 + find_staged_by_source_hash, 37 42 generate_content_manifest, 38 43 get_import_details, 39 44 list_import_timestamps, ··· 44 49 update_import_metadata_fields, 45 50 write_import_metadata, 46 51 ) 47 - from solstone.think.media import MEDIA_EXTENSIONS 52 + from solstone.think.media import ( 53 + MEDIA_EXTENSIONS, 54 + canonical_source, 55 + canonical_source_signal, 56 + ) 48 57 from solstone.think.utils import day_path, now_ms 49 58 50 59 from .journal_sources import ( ··· 222 231 return value.strip().lower() in {"true", "1", "yes"} if value else False 223 232 224 233 234 + CANONICAL_IMPORT_SOURCES = {"audio", "image", "document", "text"} 235 + 236 + 237 + def _clean_optional(value: Any) -> str | None: 238 + if value is None: 239 + return None 240 + if isinstance(value, str): 241 + cleaned = value.strip() 242 + else: 243 + cleaned = str(value).strip() 244 + return cleaned or None 245 + 246 + 247 + def _client_bag(value: Any) -> dict: 248 + if isinstance(value, dict): 249 + return value 250 + if isinstance(value, str) and value.strip(): 251 + try: 252 + parsed = json.loads(value) 253 + except json.JSONDecodeError: 254 + return {} 255 + return parsed if isinstance(parsed, dict) else {} 256 + return {} 257 + 258 + 259 + def _build_save_summary( 260 + metadata: dict, 261 + *, 262 + status: str, 263 + replay: bool, 264 + duplicate: dict | None, 265 + recommended_action: str | None = None, 266 + ) -> dict: 267 + """Build the versioned import staging summary response.""" 268 + client = metadata.get("client") 269 + if not isinstance(client, dict): 270 + client = {} 271 + action = recommended_action 272 + if action is None: 273 + action = "do_not_start" if status == "duplicate" else "start" 274 + 275 + summary: dict[str, Any] = { 276 + "schema_version": 1, 277 + "status": status, 278 + "replay": replay, 279 + "path": str(metadata.get("file_path", "")), 280 + "timestamp": str(metadata.get("user_timestamp", "")), 281 + "client_item_id": str(metadata.get("client_item_id", "")), 282 + "source": metadata.get("source", "text"), 283 + "facet": metadata.get("facet"), 284 + "setting": metadata.get("setting"), 285 + "recommended_action": action, 286 + "metadata": { 287 + "original_filename": metadata.get("original_filename"), 288 + "mime_type": metadata.get("mime_type"), 289 + "imported_via": metadata.get("imported_via"), 290 + "observer_handle": metadata.get("observer_handle"), 291 + "source_hint": metadata.get("source_hint"), 292 + "client": client, 293 + }, 294 + "diagnostics": { 295 + "timestamp_detection_method": metadata.get( 296 + "timestamp_detection_method", "duplicate" 297 + ), 298 + "timestamp_detection_model_called": metadata.get( 299 + "timestamp_detection_model_called", False 300 + ), 301 + "timestamp_detection_no_match_reason": metadata.get( 302 + "timestamp_detection_no_match_reason" 303 + ), 304 + "source_inference": metadata.get("source_inference", "default"), 305 + }, 306 + } 307 + if duplicate is not None: 308 + summary["duplicate"] = { 309 + "import_id": duplicate.get("import_id"), 310 + "imported_at": duplicate.get("imported_at"), 311 + "entry_count": duplicate.get("entry_count"), 312 + "state": duplicate.get("state"), 313 + } 314 + return summary 315 + 316 + 317 + def _load_import_metadata_or_none(journal_root: Path, timestamp: str) -> dict | None: 318 + try: 319 + metadata = read_import_metadata(journal_root, timestamp) 320 + except (FileNotFoundError, json.JSONDecodeError, OSError): 321 + return None 322 + return metadata if isinstance(metadata, dict) else None 323 + 324 + 325 + def _duplicate_summary_metadata( 326 + journal_root: Path, 327 + *, 328 + client_item_id: str, 329 + source: str, 330 + source_inference: str, 331 + duplicate: dict, 332 + existing_metadata: dict | None = None, 333 + ) -> dict: 334 + import_id = str(duplicate.get("import_id") or "") 335 + metadata = dict(existing_metadata or {}) 336 + if not metadata and import_id: 337 + metadata = dict(_load_import_metadata_or_none(journal_root, import_id) or {}) 338 + 339 + metadata.setdefault("file_path", str(journal_root / "imports" / import_id)) 340 + metadata.setdefault("user_timestamp", import_id) 341 + metadata.setdefault("original_filename", None) 342 + metadata.setdefault("mime_type", None) 343 + metadata.setdefault("imported_via", duplicate.get("imported_via")) 344 + metadata.setdefault("observer_handle", duplicate.get("observer_handle")) 345 + metadata.setdefault("source_hint", None) 346 + metadata.setdefault("client", {}) 347 + metadata.setdefault("facet", None) 348 + metadata.setdefault("setting", None) 349 + metadata.setdefault("timestamp_detection_method", "duplicate") 350 + metadata.setdefault("timestamp_detection_model_called", False) 351 + metadata.setdefault("timestamp_detection_no_match_reason", None) 352 + metadata["client_item_id"] = client_item_id 353 + metadata["source_inference"] = metadata.get("source_inference") or source_inference 354 + if metadata.get("source") not in CANONICAL_IMPORT_SOURCES: 355 + metadata["source"] = source 356 + return metadata 357 + 358 + 359 + def _duplicate_or_replay_response( 360 + journal_root: Path, 361 + *, 362 + client_item_id: str, 363 + source_hash: str, 364 + source: str, 365 + source_inference: str, 366 + ) -> Any | None: 367 + existing = find_staged_by_client_item_id(journal_root, client_item_id) 368 + if existing: 369 + if existing.get("source_hash") == source_hash: 370 + replay_action = ( 371 + "do_not_start" 372 + if existing.get("task_id") or existing.get("processing_completed") 373 + else None 374 + ) 375 + return jsonify( 376 + _build_save_summary( 377 + existing, 378 + status="staged", 379 + replay=True, 380 + duplicate=None, 381 + recommended_action=replay_action, 382 + ) 383 + ) 384 + return error_response( 385 + IMPORT_CLIENT_ID_CONFLICT, 386 + detail=( 387 + "client_item_id already staged for different content; use a new " 388 + "client_item_id or re-fetch the existing item" 389 + ), 390 + ) 391 + 392 + imported = find_manifest_by_hash(journal_root, source_hash) 393 + if imported: 394 + duplicate = { 395 + "import_id": imported.get("import_id"), 396 + "imported_at": imported.get("imported_at"), 397 + "entry_count": imported.get("entry_count"), 398 + "state": "imported", 399 + "imported_via": imported.get("imported_via"), 400 + "observer_handle": imported.get("observer_handle"), 401 + } 402 + metadata = _duplicate_summary_metadata( 403 + journal_root, 404 + client_item_id=client_item_id, 405 + source=source, 406 + source_inference=source_inference, 407 + duplicate=duplicate, 408 + ) 409 + return jsonify( 410 + _build_save_summary( 411 + metadata, 412 + status="duplicate", 413 + replay=False, 414 + duplicate=duplicate, 415 + ) 416 + ) 417 + 418 + staged_duplicate = find_staged_by_source_hash(journal_root, source_hash) 419 + if staged_duplicate: 420 + duplicate = { 421 + "import_id": staged_duplicate.get("timestamp"), 422 + "imported_at": None, 423 + "entry_count": None, 424 + "state": "staged", 425 + } 426 + metadata = _duplicate_summary_metadata( 427 + journal_root, 428 + client_item_id=client_item_id, 429 + source=source, 430 + source_inference=source_inference, 431 + duplicate=duplicate, 432 + existing_metadata=staged_duplicate, 433 + ) 434 + return jsonify( 435 + _build_save_summary( 436 + metadata, 437 + status="duplicate", 438 + replay=False, 439 + duplicate=duplicate, 440 + ) 441 + ) 442 + 443 + return None 444 + 445 + 225 446 @import_bp.route("/api/save", methods=["POST"]) 226 447 def import_save() -> Any: 227 - from datetime import datetime 228 - 229 448 upload = request.files.get("file") 230 449 text = request.form.get("text", "").strip() 450 + client_item_id = request.form.get("client_item_id", "").strip() 231 451 facet = request.form.get("facet", "").strip() or None 232 452 setting = request.form.get("setting", "").strip() or None 453 + source_hint = request.form.get("source_hint", "").strip() or None 454 + imported_via = request.form.get("imported_via", "").strip() or "web_dashboard" 455 + observer_handle = request.form.get("observer_handle", "").strip() or None 233 456 deterministic_only = _form_bool(request.form.get("deterministic_only")) 457 + client = _client_bag(request.form.get("client")) 458 + 459 + if not client_item_id: 460 + return error_response(MISSING_REQUIRED_FIELD, detail="Missing client_item_id") 234 461 235 462 # Generate timestamp for folder name 236 463 timestamp_ms = now_ms() ··· 242 469 filename = "paste.txt" 243 470 else: 244 471 return error_response(INGEST_NO_FILES, detail="No input") 472 + 473 + original_filename = upload.filename if upload else "paste.txt" 474 + mime_type = upload.content_type if upload else "text/plain" 475 + source = canonical_source(filename=original_filename, content_type=mime_type) 476 + source_inference = canonical_source_signal( 477 + filename=original_filename, 478 + content_type=mime_type, 479 + ) 480 + journal_root = Path(state.journal_root) 245 481 246 482 # Detect timestamp from content first (need temporary save for detection) 247 483 ts = None ··· 272 508 temp_path = tmp.name 273 509 274 510 try: 511 + temp_source = Path(temp_path) 512 + source_hash = hash_source(temp_source) 513 + duplicate_or_replay = _duplicate_or_replay_response( 514 + journal_root, 515 + client_item_id=client_item_id, 516 + source_hash=source_hash, 517 + source=source, 518 + source_inference=source_inference, 519 + ) 520 + if duplicate_or_replay is not None: 521 + return duplicate_or_replay 522 + 275 523 try: 276 524 original_name = upload.filename if upload else None 277 525 detection_result = resolve_created_deterministic( ··· 313 561 detection_result = None 314 562 timestamp_detection_model_called = True 315 563 timestamp_detection_no_match_reason = "model_no_match" 316 - finally: 317 - # Clean up temporary file 318 - Path(temp_path).unlink(missing_ok=True) 319 564 320 - # Use detected timestamp or fall back to upload timestamp 321 - folder_timestamp = ( 322 - ts 323 - if ts 324 - else f"{datetime.fromtimestamp(timestamp_ms / 1000).strftime('%Y%m%d_%H%M%S')}" 325 - ) 565 + # Use detected timestamp or fall back to upload timestamp 566 + folder_timestamp = ( 567 + ts 568 + if ts 569 + else datetime.fromtimestamp(timestamp_ms / 1000).strftime("%Y%m%d_%H%M%S") 570 + ) 326 571 327 - # Save the actual file using utility function 328 - if upload: 329 - # Save uploaded file to temp location first, then move to import dir 330 - import tempfile 331 - 332 - with tempfile.NamedTemporaryFile(delete=False) as tmp: 333 - upload.save(tmp.name) 334 - temp_source = Path(tmp.name) 335 - 336 - try: 572 + # Save the actual file using utility function 573 + if upload: 337 574 file_path = save_import_file( 338 - journal_root=Path(state.journal_root), 575 + journal_root=journal_root, 339 576 timestamp=folder_timestamp, 340 577 source_path=temp_source, 341 578 filename=filename, 342 579 ) 343 - finally: 344 - temp_source.unlink(missing_ok=True) 345 - else: 346 - file_path = save_import_text( 347 - journal_root=Path(state.journal_root), 580 + else: 581 + file_path = save_import_text( 582 + journal_root=journal_root, 583 + timestamp=folder_timestamp, 584 + content=text, 585 + filename=filename, 586 + ) 587 + 588 + # Build metadata dict 589 + metadata = { 590 + "original_filename": original_filename, 591 + "upload_timestamp": timestamp_ms, 592 + "upload_datetime": datetime.fromtimestamp(timestamp_ms / 1000).isoformat(), 593 + "detection_result": detection_result, 594 + "detected_timestamp": ts, 595 + "user_timestamp": folder_timestamp, 596 + "timestamp_detection_method": timestamp_detection_method, 597 + "timestamp_detection_model_called": timestamp_detection_model_called, 598 + "timestamp_detection_no_match_reason": timestamp_detection_no_match_reason, 599 + "source_inference": source_inference, 600 + "file_size": file_path.stat().st_size if file_path.exists() else 0, 601 + "mime_type": mime_type, 602 + "facet": facet, 603 + "setting": setting, 604 + "file_path": str(file_path), 605 + "imported_via": imported_via, 606 + "link_id": _link_id_from_identity(), 607 + "observer_handle": observer_handle, 608 + "client_item_id": client_item_id, 609 + "source_hash": source_hash, 610 + "source": source, 611 + "source_hint": source_hint, 612 + "client": client, 613 + } 614 + 615 + # Write metadata using utility function 616 + write_import_metadata( 617 + journal_root=journal_root, 348 618 timestamp=folder_timestamp, 349 - content=text, 350 - filename=filename, 619 + metadata=metadata, 351 620 ) 352 621 353 - # Build metadata dict 354 - metadata = { 355 - "original_filename": upload.filename if upload else "paste.txt", 356 - "upload_timestamp": timestamp_ms, 357 - "upload_datetime": datetime.fromtimestamp(timestamp_ms / 1000).isoformat(), 358 - "detection_result": detection_result, 359 - "detected_timestamp": ts, 360 - "user_timestamp": folder_timestamp, # The timestamp used for the folder 361 - "timestamp_detection_method": timestamp_detection_method, 362 - "timestamp_detection_model_called": timestamp_detection_model_called, 363 - "timestamp_detection_no_match_reason": timestamp_detection_no_match_reason, 364 - "file_size": file_path.stat().st_size if file_path.exists() else 0, 365 - "mime_type": upload.content_type if upload else "text/plain", 366 - "facet": facet, # Include selected facet 367 - "setting": setting, 368 - "file_path": str(file_path), # Store the actual file path 369 - "imported_via": request.form.get("imported_via", "").strip() or "web_dashboard", 370 - "link_id": _link_id_from_identity(), 371 - "observer_handle": request.form.get("observer_handle", "").strip() or None, 372 - } 373 - 374 - # Write metadata using utility function 375 - write_import_metadata( 376 - journal_root=Path(state.journal_root), 377 - timestamp=folder_timestamp, 378 - metadata=metadata, 379 - ) 380 - 381 - # Check for dedup — has this exact file been imported before? 382 - dedup = None 383 - try: 384 - from solstone.think.importers.shared import find_manifest_by_hash, hash_source 385 - 386 - source_hash = hash_source(file_path) 387 - existing = find_manifest_by_hash(Path(state.journal_root), source_hash) 388 - if existing: 389 - dedup = { 390 - "imported_at": existing.get("imported_at", "unknown"), 391 - "entry_count": existing.get("entry_count", 0), 392 - "import_id": existing.get("import_id", ""), 393 - } 394 - except OSError as exc: 395 - logging.warning("Dedup check failed for %s: %s", file_path, exc) 396 - 397 - result: dict[str, Any] = { 398 - "path": str(file_path), 399 - "timestamp": folder_timestamp, 400 - "facet": facet, 401 - "setting": setting, 402 - "timestamp_detection_method": timestamp_detection_method, 403 - "timestamp_detection_model_called": timestamp_detection_model_called, 404 - "timestamp_detection_no_match_reason": timestamp_detection_no_match_reason, 405 - } 406 - if dedup: 407 - result["dedup"] = dedup 408 - 409 - return jsonify(result) 622 + return jsonify( 623 + _build_save_summary( 624 + metadata, 625 + status="staged", 626 + replay=False, 627 + duplicate=None, 628 + ) 629 + ) 630 + finally: 631 + # Clean up temporary file 632 + Path(temp_path).unlink(missing_ok=True) 410 633 411 634 412 635 @import_bp.route("/api/save-path", methods=["POST"]) 413 636 def import_save_path() -> Any: 414 637 """Register a local filesystem path for import (e.g. Obsidian vault).""" 415 - from datetime import datetime 416 - 417 638 data = request.get_json(force=True) 639 + client_item_id = data.get("client_item_id", "").strip() 418 640 local_path = data.get("path", "").strip() 419 641 facet = data.get("facet", "").strip() or None 420 642 setting = data.get("setting", "").strip() or None 643 + source_hint = data.get("source_hint", "").strip() or None 644 + imported_via = data.get("imported_via", "").strip() or "web_dashboard" 645 + observer_handle = data.get("observer_handle", "").strip() or None 646 + client = _client_bag(data.get("client")) 647 + 648 + if not client_item_id: 649 + return error_response(MISSING_REQUIRED_FIELD, detail="Missing client_item_id") 421 650 422 651 if not local_path: 423 652 return error_response(MISSING_REQUIRED_FIELD, detail="Missing path") ··· 431 660 f"{datetime.fromtimestamp(timestamp_ms / 1000).strftime('%Y%m%d_%H%M%S')}" 432 661 ) 433 662 434 - # Create import directory and metadata 435 663 journal_root = Path(state.journal_root) 436 - import_dir = journal_root / "imports" / folder_timestamp 437 - import_dir.mkdir(parents=True, exist_ok=True) 664 + source_hash = hash_source(local) 665 + source = canonical_source(filename=local.name) 666 + source_inference = canonical_source_signal(filename=local.name) 667 + duplicate_or_replay = _duplicate_or_replay_response( 668 + journal_root, 669 + client_item_id=client_item_id, 670 + source_hash=source_hash, 671 + source=source, 672 + source_inference=source_inference, 673 + ) 674 + if duplicate_or_replay is not None: 675 + return duplicate_or_replay 438 676 439 677 metadata = { 440 678 "original_filename": local.name, 441 679 "upload_timestamp": timestamp_ms, 442 680 "upload_datetime": datetime.fromtimestamp(timestamp_ms / 1000).isoformat(), 443 681 "user_timestamp": folder_timestamp, 682 + "timestamp_detection_method": "path_fallback", 683 + "timestamp_detection_model_called": False, 684 + "timestamp_detection_no_match_reason": None, 685 + "source_inference": source_inference, 444 686 "file_path": local_path, 445 687 "facet": facet, 446 688 "setting": setting, 447 689 "is_local_path": True, 448 - "imported_via": data.get("imported_via", "").strip() or "web_dashboard", 690 + "mime_type": None, 691 + "imported_via": imported_via, 449 692 "link_id": _link_id_from_identity(), 450 - "observer_handle": data.get("observer_handle", "").strip() or None, 693 + "observer_handle": observer_handle, 694 + "client_item_id": client_item_id, 695 + "source_hash": source_hash, 696 + "source": source, 697 + "source_hint": source_hint, 698 + "client": client, 451 699 } 452 700 453 701 write_import_metadata( ··· 457 705 ) 458 706 459 707 return jsonify( 460 - { 461 - "path": local_path, 462 - "timestamp": folder_timestamp, 463 - "facet": facet, 464 - "setting": setting, 465 - } 708 + _build_save_summary( 709 + metadata, 710 + status="staged", 711 + replay=False, 712 + duplicate=None, 713 + ) 466 714 ) 467 715 468 716 469 - @import_bp.route("/api/facet", methods=["POST"]) 717 + @import_bp.route("/api/meta", methods=["POST"]) 470 718 def import_update_metadata() -> Any: 471 - """Update stored metadata (facet/setting) for a saved import.""" 719 + """Update stored metadata for a saved import.""" 472 720 data = request.get_json(force=True) 473 721 raw_path = data.get("path", "").strip() 474 722 if not raw_path: 475 723 return error_response(MISSING_REQUIRED_FIELD, detail="Missing import path") 476 724 477 - facet = data.get("facet", "").strip() or None 478 - setting = data.get("setting", "").strip() or None 479 - 480 725 # Extract timestamp from path 481 726 # Path format: .../imports/{timestamp}/{filename} 482 727 file_path = Path(raw_path) 483 728 timestamp = file_path.parent.name 729 + journal_root = Path(state.journal_root) 484 730 485 731 try: 486 - # Use utility function to update metadata 487 - metadata, updated = update_import_metadata_fields( 488 - journal_root=Path(state.journal_root), 732 + metadata = read_import_metadata(journal_root=journal_root, timestamp=timestamp) 733 + except FileNotFoundError: 734 + return error_response(IMPORT_NOT_FOUND, detail="Import metadata not found") 735 + except Exception as exc: 736 + return error_response( 737 + IMPORT_METADATA_FAILED, 738 + detail=f"Failed to read metadata: {exc}", 739 + ) 740 + 741 + if metadata.get("task_id") or metadata.get("processing_completed"): 742 + return error_response( 743 + INVALID_OPERATION_FOR_STATE, 744 + detail="import already started or processed", 745 + ) 746 + 747 + source_hash = metadata.get("source_hash") 748 + if source_hash and find_manifest_by_hash(journal_root, source_hash): 749 + return error_response( 750 + INVALID_OPERATION_FOR_STATE, 751 + detail="content already imported", 752 + ) 753 + 754 + updates: dict[str, Any] = {} 755 + for key in ( 756 + "facet", 757 + "setting", 758 + "original_filename", 759 + "mime_type", 760 + "source_hint", 761 + "observer_handle", 762 + "imported_via", 763 + "client", 764 + ): 765 + if key not in data: 766 + continue 767 + if key in {"facet", "setting", "source_hint", "observer_handle"}: 768 + updates[key] = _clean_optional(data.get(key)) 769 + elif key == "client": 770 + updates[key] = _client_bag(data.get(key)) 771 + else: 772 + updates[key] = data.get(key) 773 + 774 + changed = { 775 + key: value 776 + for key, value in updates.items() 777 + if key not in metadata or metadata.get(key) != value 778 + } 779 + 780 + try: 781 + update_import_metadata_fields( 782 + journal_root=journal_root, 489 783 timestamp=timestamp, 490 - updates={"facet": facet, "setting": setting}, 784 + updates=updates, 491 785 ) 492 786 except FileNotFoundError: 493 787 return error_response(IMPORT_NOT_FOUND, detail="Import metadata not found") ··· 500 794 return jsonify( 501 795 { 502 796 "status": "ok", 503 - "facet": facet, 504 - "setting": setting, 505 - "updated": updated, 797 + "path": raw_path, 798 + "timestamp": timestamp, 799 + "updated": changed, 506 800 } 507 801 ) 508 802 ··· 854 1148 data = request.get_json(force=True) 855 1149 path = data.get("path") 856 1150 ts = data.get("timestamp") 857 - source = data.get("source") 858 1151 force = data.get("force", False) 859 1152 if not path or not ts: 860 1153 return error_response(MISSING_REQUIRED_FIELD, detail="missing params") 861 - 862 - # Generate task ID 863 - task_id = str(now_ms()) 864 1154 865 1155 # Extract original timestamp from path and handle timestamp changes 866 1156 file_path = Path(path) ··· 869 1159 is_local_path = not str(file_path).startswith(str(imports_dir)) 870 1160 original_timestamp = file_path.parent.name if not is_local_path else ts 871 1161 1162 + # Read import metadata before any move. Saved metadata is the authority for 1163 + # facet, setting, and source routing. 1164 + try: 1165 + metadata = read_import_metadata( 1166 + journal_root=journal_root, 1167 + timestamp=original_timestamp, 1168 + ) 1169 + except FileNotFoundError: 1170 + return error_response( 1171 + IMPORT_NOT_FOUND, 1172 + detail=f"Import metadata not found for {original_timestamp}", 1173 + ) 1174 + except Exception as e: 1175 + return error_response( 1176 + IMPORT_METADATA_FAILED, 1177 + detail=f"Failed to read metadata: {str(e)}", 1178 + ) 1179 + 1180 + source_hash = metadata.get("source_hash") 1181 + if source_hash and find_manifest_by_hash(journal_root, source_hash): 1182 + return error_response( 1183 + INVALID_OPERATION_FOR_STATE, 1184 + detail="content already imported; will not start", 1185 + ) 1186 + 1187 + # Generate task ID 1188 + task_id = str(now_ms()) 1189 + 872 1190 # If timestamp changed, move the import directory through the imports/ owner 873 1191 if not is_local_path and original_timestamp != ts: 874 1192 try: ··· 899 1217 # Update file_path in metadata (need to update after reading) 900 1218 # We'll handle this after reading the metadata below 901 1219 902 - # Read import metadata to get facet and setting 903 - try: 904 - metadata = read_import_metadata(journal_root=journal_root, timestamp=ts) 905 - except FileNotFoundError: 906 - return error_response( 907 - IMPORT_NOT_FOUND, 908 - detail=f"Import metadata not found for {ts}", 909 - ) 910 - except Exception as e: 911 - return error_response( 912 - IMPORT_METADATA_FAILED, 913 - detail=f"Failed to read metadata: {str(e)}", 914 - ) 915 - 916 1220 # Update file_path in metadata if timestamp changed 917 1221 if not is_local_path and original_timestamp != ts: 918 1222 try: ··· 921 1225 timestamp=ts, 922 1226 updates={"file_path": path}, 923 1227 ) 1228 + metadata["file_path"] = path 924 1229 except Exception as e: 925 1230 return error_response( 926 1231 IMPORT_METADATA_FAILED, ··· 929 1234 930 1235 facet = metadata.get("facet") 931 1236 setting = metadata.get("setting") 1237 + source_hint = _clean_optional(metadata.get("source_hint")) 932 1238 933 1239 # Build command 934 1240 cmd = ["journal", "importer", path, ts] ··· 936 1242 cmd.extend(["--facet", facet]) 937 1243 if setting: 938 1244 cmd.extend(["--setting", setting]) 939 - if source: 940 - cmd.extend(["--source", source]) 1245 + if source_hint: 1246 + cmd.extend(["--source", source_hint]) 941 1247 if force: 942 1248 cmd.append("--force") 943 1249 ··· 946 1252 update_import_metadata_fields( 947 1253 journal_root=journal_root, 948 1254 timestamp=ts, 949 - updates={"task_id": task_id, "source": source}, 1255 + updates={"task_id": task_id, "source_hint": source_hint}, 950 1256 ) 951 1257 except Exception as e: 952 1258 return error_response(
+102 -20
solstone/apps/import/workspace.html
··· 669 669 </div> 670 670 </div> 671 671 <input type="hidden" id="savedPath" /> 672 + <div id="detectDuplicateNotice"></div> 672 673 <div class="modal-actions"> 673 674 <button id="startBtn" class="import-primary-btn">start import</button> 674 675 </div> ··· 708 709 let currentGuideSource = null; 709 710 let currentGuidedFile = null; 710 711 let currentGuidedSaved = null; 712 + window._quickClientItemId = null; 713 + window._quickSaved = null; 714 + window._guidedClientItemId = null; 711 715 let importEvents = {}; 712 716 let importsCache = []; 713 717 let currentSourceFilter = ''; ··· 787 791 currentGuideSource = null; 788 792 currentGuidedFile = null; 789 793 currentGuidedSaved = null; 790 - window._guidedForceImport = false; 794 + window._guidedClientItemId = null; 791 795 navigateTo('grid'); 792 796 } 793 797 ··· 838 842 let previousDetectFocus = null; 839 843 let detectFocusTrapHandler = null; 840 844 845 + function isTerminalDuplicate(data) { 846 + return data?.status === 'duplicate' || data?.recommended_action === 'do_not_start'; 847 + } 848 + 849 + function duplicateNoticeText(duplicate = {}) { 850 + const state = duplicate.state || 'imported'; 851 + const importId = duplicate.import_id || ''; 852 + if (state === 'staged') { 853 + return `this file was already staged${importId ? ` as ${importId}` : ''}.`; 854 + } 855 + const importedAt = duplicate.imported_at || 'an earlier import'; 856 + const entries = duplicate.entry_count ? ` (${duplicate.entry_count} entries)` : ''; 857 + return `this file was already imported on ${importedAt}${entries}.`; 858 + } 859 + 841 860 function showDetect(res) { 861 + window._quickSaved = res; 842 862 document.getElementById('timestampInput').value = res.timestamp || ''; 843 863 document.getElementById('savedPath').value = res.path; 864 + const duplicateNotice = document.getElementById('detectDuplicateNotice'); 865 + if (duplicateNotice) { 866 + duplicateNotice.innerHTML = ''; 867 + } 868 + startBtn.style.display = ''; 869 + startBtn.disabled = false; 870 + if (isTerminalDuplicate(res)) { 871 + if (duplicateNotice) { 872 + duplicateNotice.innerHTML = ` 873 + <div class="import-dedup-notice"> 874 + <span class="dedup-text">${escapeHtml(duplicateNoticeText(res.duplicate))}</span> 875 + </div> 876 + `; 877 + } 878 + startBtn.disabled = true; 879 + startBtn.style.display = 'none'; 880 + } 844 881 const defaultFacet = res.facet || window.selectedFacet || ''; 845 882 loadFacets(defaultFacet); 846 883 settingInput.value = res.setting || ''; ··· 1406 1443 currentGuideSource = sourceName; 1407 1444 currentGuidedFile = null; 1408 1445 currentGuidedSaved = null; 1446 + window._guidedClientItemId = null; 1409 1447 if (!guideSteps) { 1410 1448 return; 1411 1449 } ··· 1480 1518 } else { 1481 1519 // Auto-detect default path for path_input sources 1482 1520 checkDefaultPath(sourceName); 1521 + const guidedPathInput = document.getElementById('guidedPathInput'); 1522 + if (guidedPathInput) { 1523 + guidedPathInput.addEventListener('input', () => { 1524 + currentGuidedSaved = null; 1525 + window._guidedClientItemId = crypto.randomUUID(); 1526 + }); 1527 + } 1483 1528 } 1484 1529 document.getElementById('guidedStartBtn').addEventListener('click', () => startGuidedImport(source)); 1485 1530 const guidedForm = document.getElementById('guidedImportForm'); ··· 1573 1618 currentGuidedFile = file || null; 1574 1619 fileLabel.textContent = file ? file.name : ''; 1575 1620 currentGuidedSaved = null; 1621 + window._guidedClientItemId = file ? crypto.randomUUID() : null; 1576 1622 }; 1577 1623 1578 1624 dropArea.onclick = () => fileInput.click(); ··· 1619 1665 } 1620 1666 1621 1667 currentFile = null; 1668 + window._quickClientItemId = null; 1669 + window._quickSaved = null; 1622 1670 dropArea.onclick = () => fileInput.click(); 1623 1671 dropArea.addEventListener('keydown', e => { 1624 1672 if (e.key === 'Enter' || e.key === ' ') { ··· 1639 1687 if (file) { 1640 1688 currentFile = file; 1641 1689 fileLabel.textContent = file.name; 1690 + window._quickClientItemId = crypto.randomUUID(); 1691 + window._quickSaved = null; 1642 1692 } 1643 1693 }); 1644 1694 fileInput.onchange = e => { 1645 1695 currentFile = e.target.files[0]; 1646 1696 if (currentFile) { 1647 1697 fileLabel.textContent = currentFile.name; 1698 + window._quickClientItemId = crypto.randomUUID(); 1699 + window._quickSaved = null; 1648 1700 } 1649 1701 }; 1702 + ['input', 'paste'].forEach(eventName => { 1703 + pasteText.addEventListener(eventName, () => { 1704 + window._quickClientItemId = crypto.randomUUID(); 1705 + window._quickSaved = null; 1706 + }); 1707 + }); 1650 1708 1651 1709 importForm.onsubmit = async e => { 1652 1710 e.preventDefault(); ··· 1671 1729 } else { 1672 1730 fd.append('text', pasteText.value); 1673 1731 } 1732 + fd.append( 1733 + 'client_item_id', 1734 + window._quickClientItemId || (window._quickClientItemId = crypto.randomUUID()) 1735 + ); 1674 1736 1675 1737 const facet = quickFacetSelect ? quickFacetSelect.value : ''; 1676 1738 if (facet) { ··· 1703 1765 }; 1704 1766 } 1705 1767 1706 - async function uploadGuidedSourceFile() { 1768 + async function uploadGuidedSourceFile(source) { 1707 1769 if (!currentGuidedFile) { 1708 1770 throw new Error('Please select a file to import.'); 1709 1771 } ··· 1712 1774 const guidedSettingInput = document.getElementById('guidedSettingInput'); 1713 1775 const fd = new FormData(); 1714 1776 fd.append('file', currentGuidedFile); 1777 + fd.append( 1778 + 'client_item_id', 1779 + window._guidedClientItemId || (window._guidedClientItemId = crypto.randomUUID()) 1780 + ); 1781 + if (source && source.name !== 'recording') { 1782 + fd.append('source_hint', source.name); 1783 + } 1715 1784 if (guidedFacetSelect && guidedFacetSelect.value) { 1716 1785 fd.append('facet', guidedFacetSelect.value); 1717 1786 } ··· 1730 1799 1731 1800 currentGuidedSaved = data; 1732 1801 1733 - // Show dedup notice if the file was previously imported 1734 - if (data.dedup) { 1735 - showDedupNotice(data.dedup); 1802 + if (isTerminalDuplicate(data)) { 1803 + showDedupNotice(data.duplicate); 1736 1804 } 1737 1805 1738 1806 return data; 1739 1807 } 1740 1808 1741 - function showDedupNotice(dedup) { 1809 + function showDedupNotice(duplicate) { 1742 1810 const container = document.getElementById('guidedDedupNotice'); 1743 1811 if (!container) return; 1744 1812 container.innerHTML = ` 1745 1813 <div class="import-dedup-notice"> 1746 - <span class="dedup-text">this file was already imported on ${escapeHtml(dedup.imported_at)}${dedup.entry_count ? ` (${dedup.entry_count} entries)` : ''}.</span> 1747 - <button type="button" id="dedupForceBtn">re-import anyway</button> 1814 + <span class="dedup-text">${escapeHtml(duplicateNoticeText(duplicate))}</span> 1748 1815 </div> 1749 1816 `; 1750 - document.getElementById('dedupForceBtn').addEventListener('click', () => { 1751 - window._guidedForceImport = true; 1752 - container.innerHTML = '<div class="import-dedup-notice"><span class="dedup-text">will re-import. click start import to proceed.</span></div>'; 1753 - }); 1754 1817 } 1755 1818 1756 1819 async function saveGuidedPath(source) { ··· 1763 1826 const guidedSettingInput = document.getElementById('guidedSettingInput'); 1764 1827 const facet = guidedFacetSelect ? guidedFacetSelect.value : ''; 1765 1828 const setting = guidedSettingInput ? guidedSettingInput.value.trim() : ''; 1829 + const payload = { 1830 + path: pathInput.value.trim(), 1831 + facet, 1832 + setting, 1833 + client_item_id: window._guidedClientItemId 1834 + || (window._guidedClientItemId = crypto.randomUUID()), 1835 + }; 1836 + if (source && source.name !== 'recording') { 1837 + payload.source_hint = source.name; 1838 + } 1766 1839 1767 1840 const response = await fetch('/app/import/api/save-path', { 1768 1841 method: 'POST', 1769 1842 headers: { 'Content-Type': 'application/json' }, 1770 - body: JSON.stringify({ path: pathInput.value.trim(), facet, setting }), 1843 + body: JSON.stringify(payload), 1771 1844 }); 1772 1845 const data = await response.json(); 1773 1846 if (!response.ok) { ··· 1775 1848 } 1776 1849 1777 1850 currentGuidedSaved = data; 1851 + if (isTerminalDuplicate(data)) { 1852 + showDedupNotice(data.duplicate); 1853 + } 1778 1854 return data; 1779 1855 } 1780 1856 ··· 1792 1868 if (source.input_type === 'path_input') { 1793 1869 saved = currentGuidedSaved || await saveGuidedPath(source); 1794 1870 } else { 1795 - saved = currentGuidedSaved || await uploadGuidedSourceFile(); 1871 + saved = currentGuidedSaved || await uploadGuidedSourceFile(source); 1872 + } 1873 + if (isTerminalDuplicate(saved)) { 1874 + showDedupNotice(saved.duplicate); 1875 + return; 1796 1876 } 1797 1877 const path = saved.path; 1798 1878 const ts = saved.timestamp; 1799 1879 const facet = guidedFacetSelect ? guidedFacetSelect.value : ''; 1800 1880 const setting = guidedSettingInput ? guidedSettingInput.value.trim() : ''; 1801 - const force = !!(window._guidedForceImport); 1802 1881 1803 - const metaResponse = await fetch('/app/import/api/facet', { 1882 + const metaResponse = await fetch('/app/import/api/meta', { 1804 1883 method: 'POST', 1805 1884 headers: { 'Content-Type': 'application/json' }, 1806 1885 body: JSON.stringify({ path, facet, setting }), ··· 1814 1893 const startResponse = await fetch('/app/import/api/start', { 1815 1894 method: 'POST', 1816 1895 headers: { 'Content-Type': 'application/json' }, 1817 - body: JSON.stringify({ path, timestamp: ts, source: source.name, force }), 1896 + body: JSON.stringify({ path, timestamp: ts }), 1818 1897 }); 1819 1898 1820 1899 if (!startResponse.ok) { ··· 1830 1909 source_display: source.display_name, 1831 1910 }; 1832 1911 trackPendingImport(ts); 1833 - window._guidedForceImport = false; 1834 1912 loadImports(); 1835 1913 navigateTo(`progress/${ts}`); 1836 1914 } catch (err) { ··· 1925 2003 if (window.showError) showError('Timestamp is required before starting the import.'); 1926 2004 return; 1927 2005 } 2006 + if (isTerminalDuplicate(window._quickSaved)) { 2007 + showDetect(window._quickSaved); 2008 + return; 2009 + } 1928 2010 1929 2011 const originalText = startBtn.textContent; 1930 2012 startBtn.disabled = true; 1931 2013 startBtn.innerHTML = 'starting<span class="spinner"></span>'; 1932 2014 1933 2015 try { 1934 - const response = await fetch('/app/import/api/facet', { 2016 + const response = await fetch('/app/import/api/meta', { 1935 2017 method: 'POST', 1936 2018 headers: { 'Content-Type': 'application/json' }, 1937 2019 body: JSON.stringify({ path, facet, setting }), ··· 1945 2027 const startResponse = await fetch('/app/import/api/start', { 1946 2028 method: 'POST', 1947 2029 headers: { 'Content-Type': 'application/json' }, 1948 - body: JSON.stringify({ path, timestamp: ts, source: 'quick', force: false }), 2030 + body: JSON.stringify({ path, timestamp: ts }), 1949 2031 }); 1950 2032 1951 2033 if (!startResponse.ok) {
+5
solstone/convey/reasons.py
··· 297 297 "I couldn't start that import because it already exists.", 298 298 409, 299 299 ) 300 + IMPORT_CLIENT_ID_CONFLICT = Reason( 301 + "import_client_id_conflict", 302 + "That client_item_id is already staged for different content.", 303 + 409, 304 + ) 300 305 IMPORT_METADATA_FAILED = Reason( 301 306 "import_metadata_failed", 302 307 "I couldn't update that import metadata.",
+46 -10
solstone/think/import_client.py
··· 9 9 import json 10 10 import logging 11 11 import sys 12 + import uuid 12 13 from pathlib import Path 13 14 from typing import Any 14 15 ··· 57 58 ) 58 59 parser.add_argument( 59 60 "--source", 60 - help="Import source type (apple, plaud, audio, text, or a file importer name)", 61 + help=( 62 + "Import source hint: a file importer name (e.g. obsidian, plaud, ics) " 63 + "or a canonical category (audio, image, document, text)" 64 + ), 61 65 ) 62 66 parser.add_argument( 63 67 "--force", ··· 165 169 return stripped or None 166 170 167 171 168 - def _save_media(client: ConveyClient, args: argparse.Namespace) -> dict[str, Any]: 172 + def _save_media( 173 + client: ConveyClient, 174 + args: argparse.Namespace, 175 + client_item_id: str, 176 + ) -> dict[str, Any]: 169 177 media_path = Path(args.media).expanduser() 170 178 data = { 171 179 key: value 172 180 for key, value in { 181 + "client_item_id": client_item_id, 173 182 "facet": _payload_value(args.facet), 174 183 "setting": _payload_value(args.setting), 184 + "source_hint": _payload_value(args.source), 175 185 }.items() 176 186 if value is not None 177 187 } ··· 216 226 "timestamp": timestamp, 217 227 "force": bool(args.force), 218 228 } 219 - for key, value in { 220 - "facet": _payload_value(args.facet), 221 - "setting": _payload_value(args.setting), 222 - "source": _payload_value(args.source), 223 - }.items(): 224 - if value is not None: 225 - payload[key] = value 226 229 start_response = client.request("POST", f"{IMPORT_API}/start", json=payload) 227 230 if not isinstance(start_response, dict): 228 231 raise ConveyClientError(MALFORMED_RESPONSE) ··· 294 297 print("queued processing") 295 298 296 299 300 + def _print_duplicate(save_response: dict[str, Any], *, json_out: bool) -> None: 301 + if json_out: 302 + print(json.dumps(save_response, sort_keys=True)) 303 + return 304 + 305 + duplicate = save_response.get("duplicate") 306 + if not isinstance(duplicate, dict): 307 + print("sol import: duplicate import; skipping") 308 + return 309 + 310 + state = duplicate.get("state") 311 + import_id = duplicate.get("import_id") or "unknown" 312 + if state == "imported": 313 + imported_at = duplicate.get("imported_at") or "unknown date" 314 + entry_count = duplicate.get("entry_count") 315 + entries = f" ({entry_count} entries)" if entry_count is not None else "" 316 + print(f"sol import: already imported on {imported_at}{entries}; skipping") 317 + return 318 + if state == "staged": 319 + print(f"sol import: already staged as {import_id}; skipping") 320 + return 321 + 322 + print("sol import: duplicate import; skipping") 323 + 324 + 297 325 def _run(args: argparse.Namespace, client: ConveyClient) -> int: 326 + client_item_id = uuid.uuid4().hex 298 327 try: 299 - save_response = _save_media(client, args) 328 + save_response = _save_media(client, args, client_item_id) 300 329 if not isinstance(save_response, dict): 301 330 raise ConveyClientError(MALFORMED_RESPONSE) 302 331 except ConveyUnreachableError: ··· 308 337 except ConveyClientError as err: 309 338 _print_client_error("stage import", err) 310 339 return 1 340 + 341 + if ( 342 + save_response.get("status") == "duplicate" 343 + or save_response.get("recommended_action") == "do_not_start" 344 + ): 345 + _print_duplicate(save_response, json_out=bool(args.json)) 346 + return 0 311 347 312 348 staged_path = str(save_response.get("path") or args.media) 313 349 try:
+15 -15
solstone/think/importers/cli.py
··· 403 403 global _stage_start_time, _stages_run, _status_thread, _status_running 404 404 405 405 args.media = os.path.expanduser(args.media) 406 - if args.source == "quick": 407 - args.source = None 408 406 409 407 _file_importer = None 410 408 import_source = None ··· 434 432 _file_importer = detected 435 433 import_source = detected.name 436 434 437 - # Try file importer detection for unknown file extensions 438 - if _file_importer is None and not args.source: 435 + # Try file importer detection for unknown file extensions. Non-file-importer 436 + # --source values are intentionally ignored, so they should not suppress 437 + # detection for image/structured imports. 438 + if _file_importer is None: 439 439 _ext = os.path.splitext(args.media)[1].lower() 440 440 if _ext not in {".m4a", ".txt", ".md", ".pdf"}: 441 441 from solstone.think.importers.file_importer import detect_file_importer ··· 537 537 day = base_dt.strftime("%Y%m%d") 538 538 539 539 # --- Derive import_source for non-file-importer paths --- 540 + # Generic (non-file-importer) items stream as audio or text by extension. 541 + # A non-importer --source value (e.g. legacy "recording"/"quick") never 542 + # becomes a stream name; it is ignored here so leaked vocabulary cannot 543 + # reach the stream/manifest. File-importer sources are handled earlier. 540 544 if import_source is None: 541 - if args.source: 542 - import_source = args.source 543 - else: 544 - _ext = os.path.splitext(args.media)[1].lower() 545 - if _ext == ".m4a": 546 - import_source = "apple" 547 - elif _ext in {".txt", ".md", ".pdf"}: 548 - import_source = "text" 549 - else: 550 - import_source = "audio" 545 + _ext = os.path.splitext(args.media)[1].lower() 546 + import_source = "text" if _ext in {".txt", ".md", ".pdf"} else "audio" 551 547 552 548 stream = stream_name(import_source=import_source) 553 549 needs_setup = _file_importer is None and not _is_in_imports(args.media) ··· 1347 1343 "--source", 1348 1344 type=str, 1349 1345 default=None, 1350 - help="Import source type (apple, plaud, audio, text, or a file importer name). Auto-detected if omitted.", 1346 + help=( 1347 + "Import source hint: a file importer name (e.g. obsidian, plaud, ics) " 1348 + "or a canonical category (audio, image, document, text). " 1349 + "Auto-detected if omitted." 1350 + ), 1351 1351 ) 1352 1352 parser.add_argument( 1353 1353 "--force",
+43
solstone/think/importers/utils.py
··· 161 161 raise FileNotFoundError(f"Import metadata not found for {timestamp}") 162 162 163 163 164 + def find_staged_by_client_item_id( 165 + journal_root: Path, client_item_id: str 166 + ) -> dict | None: 167 + """Return staged import metadata with matching client_item_id, else None.""" 168 + for timestamp in list_import_timestamps(journal_root): 169 + try: 170 + metadata = read_import_metadata(journal_root, timestamp) 171 + except (FileNotFoundError, json.JSONDecodeError): 172 + continue 173 + if not isinstance(metadata, dict): 174 + continue 175 + if metadata.get("client_item_id") == client_item_id: 176 + result = dict(metadata) 177 + result["timestamp"] = timestamp 178 + return result 179 + 180 + return None 181 + 182 + 183 + def find_staged_by_source_hash( 184 + journal_root: Path, 185 + source_hash: str, 186 + *, 187 + exclude_timestamp: str | None = None, 188 + ) -> dict | None: 189 + """Return staged import metadata with matching source_hash, else None.""" 190 + for timestamp in list_import_timestamps(journal_root): 191 + if timestamp == exclude_timestamp: 192 + continue 193 + try: 194 + metadata = read_import_metadata(journal_root, timestamp) 195 + except (FileNotFoundError, json.JSONDecodeError): 196 + continue 197 + if not isinstance(metadata, dict): 198 + continue 199 + if metadata.get("source_hash") == source_hash: 200 + result = dict(metadata) 201 + result["timestamp"] = timestamp 202 + return result 203 + 204 + return None 205 + 206 + 164 207 def update_import_metadata_fields( 165 208 journal_root: Path, 166 209 timestamp: str,
+54
solstone/think/media.py
··· 3 3 4 4 """Media format registry - single source of truth for extensions, MIME types, and kind.""" 5 5 6 + from pathlib import Path 7 + 6 8 FORMATS = [ 7 9 (".flac", "audio/flac", "audio"), 8 10 (".opus", "audio/opus", "audio"), ··· 27 29 {".png", ".jpg", ".jpeg", ".heic", ".heif", ".gif", ".webp", ".tiff"} 28 30 ) 29 31 PDF_EXTENSIONS: frozenset[str] = frozenset({".pdf"}) 32 + 33 + 34 + def canonical_source( 35 + *, filename: str | None = None, content_type: str | None = None 36 + ) -> str: 37 + """Infer the canonical import source category from content signals. 38 + 39 + Returns one of "audio", "image", "document", "text". This is a protocol 40 + metadata category describing WHAT the content is; it does not drive importer 41 + routing. 42 + """ 43 + suffix = Path(filename).suffix.lower() if filename else "" 44 + if suffix in AUDIO_EXTENSIONS or suffix in VIDEO_EXTENSIONS: 45 + return "audio" 46 + if suffix in IMAGE_EXTENSIONS: 47 + return "image" 48 + if suffix in PDF_EXTENSIONS: 49 + return "document" 50 + 51 + media_type = content_type.lower() if content_type else "" 52 + if media_type.startswith(("audio/", "video/")): 53 + return "audio" 54 + if media_type.startswith("image/"): 55 + return "image" 56 + if media_type == "application/pdf": 57 + return "document" 58 + 59 + return "text" 60 + 61 + 62 + def canonical_source_signal( 63 + *, filename: str | None = None, content_type: str | None = None 64 + ) -> str: 65 + """Return the signal that determined the canonical source.""" 66 + suffix = Path(filename).suffix.lower() if filename else "" 67 + if ( 68 + suffix in AUDIO_EXTENSIONS 69 + or suffix in VIDEO_EXTENSIONS 70 + or suffix in IMAGE_EXTENSIONS 71 + or suffix in PDF_EXTENSIONS 72 + ): 73 + return "extension" 74 + 75 + media_type = content_type.lower() if content_type else "" 76 + if ( 77 + media_type.startswith(("audio/", "video/")) 78 + or media_type.startswith("image/") 79 + or media_type == "application/pdf" 80 + ): 81 + return "content_type" 82 + 83 + return "default"
+99 -21
tests/test_import_client.py
··· 57 57 return self.request_responses[index] 58 58 59 59 60 + def _pop_client_item_id(payload: dict[str, Any]) -> str: 61 + client_item_id = payload.pop("client_item_id") 62 + assert isinstance(client_item_id, str) 63 + assert client_item_id 64 + return client_item_id 65 + 66 + 60 67 def test_mode_disposition_table_covers_d5_modes() -> None: 61 68 assert import_client.MODE_DISPOSITIONS == { 62 69 "positional_media": "http-client", ··· 89 96 code = import_client.main([str(media)], client=client) # type: ignore[arg-type] 90 97 91 98 assert code == 0 92 - assert client.uploads == [ 93 - { 94 - "path": "/app/import/api/save", 95 - "files": { 96 - "file": ("sample.txt", media, "application/octet-stream"), 97 - }, 98 - "data": {}, 99 - } 100 - ] 99 + save_data = dict(client.uploads[0]["data"]) 100 + client_item_id = _pop_client_item_id(save_data) 101 + assert len(client_item_id) == 32 102 + assert len(client.uploads) == 1 103 + assert client.uploads[0]["path"] == "/app/import/api/save" 104 + assert client.uploads[0]["files"] == { 105 + "file": ("sample.txt", media, "application/octet-stream"), 106 + } 107 + assert save_data == {} 101 108 assert client.requests == [ 102 109 { 103 110 "method": "POST", ··· 130 137 131 138 assert code == 0 132 139 assert client.uploads == [] 133 - assert client.requests[0] == { 134 - "method": "POST", 135 - "path": "/app/import/api/save-path", 136 - "params": None, 137 - "json": {"path": str(media_dir)}, 138 - } 140 + save_json = dict(client.requests[0]["json"]) 141 + client_item_id = _pop_client_item_id(save_json) 142 + assert len(client_item_id) == 32 143 + assert client.requests[0]["method"] == "POST" 144 + assert client.requests[0]["path"] == "/app/import/api/save-path" 145 + assert client.requests[0]["params"] is None 146 + assert save_json == {"path": str(media_dir)} 139 147 assert client.requests[1]["path"] == "/app/import/api/start" 140 148 assert client.requests[1]["json"]["timestamp"] == "20260101_130000" 141 149 ··· 175 183 ) 176 184 177 185 assert code == 0 178 - assert client.uploads[0]["data"] == {"facet": "work", "setting": "office"} 186 + save_data = dict(client.uploads[0]["data"]) 187 + _pop_client_item_id(save_data) 188 + assert save_data == { 189 + "facet": "work", 190 + "setting": "office", 191 + "source_hint": "ics", 192 + } 179 193 assert client.requests[0]["json"] == { 180 194 "path": "/journal/imports/20260101_120000/sample.txt", 181 195 "timestamp": "20260101_120000", 182 196 "force": True, 183 - "facet": "work", 184 - "setting": "office", 185 - "source": "ics", 186 197 } 187 198 188 199 ··· 197 208 ) 198 209 199 210 assert code == 0 200 - assert client.uploads[0]["data"] == {"deterministic_only": "true"} 211 + save_data = dict(client.uploads[0]["data"]) 212 + _pop_client_item_id(save_data) 213 + assert save_data == {"deterministic_only": "true"} 201 214 assert "deterministic_only" not in client.requests[0]["json"] 202 215 203 216 ··· 206 219 media.write_text("hello", encoding="utf-8") 207 220 client = FakeClient() 208 221 209 - code = import_client.main([str(media), "--json"], client=client) # type: ignore[arg-type] 222 + code = import_client.main( 223 + [str(media), "--json"], 224 + client=client, # type: ignore[arg-type] 225 + ) 210 226 211 227 assert code == 0 212 228 payload = json.loads(capsys.readouterr().out) ··· 220 236 }, 221 237 "start": {"status": "ok", "task_id": "task-1"}, 222 238 } 239 + 240 + 241 + def test_duplicate_save_skips_start_and_returns_success( 242 + tmp_path: Path, capsys: pytest.CaptureFixture[str] 243 + ) -> None: 244 + media = tmp_path / "sample.txt" 245 + media.write_text("hello", encoding="utf-8") 246 + client = FakeClient( 247 + upload_response={ 248 + "schema_version": 1, 249 + "status": "duplicate", 250 + "recommended_action": "do_not_start", 251 + "path": "/journal/imports/20260101_120000/sample.txt", 252 + "timestamp": "20260101_120000", 253 + "duplicate": { 254 + "import_id": "20260101_120000", 255 + "imported_at": "2026-01-01T12:00:00", 256 + "entry_count": 2, 257 + "state": "imported", 258 + }, 259 + } 260 + ) 261 + 262 + code = import_client.main([str(media)], client=client) # type: ignore[arg-type] 263 + 264 + assert code == 0 265 + assert client.requests == [] 266 + captured = capsys.readouterr() 267 + assert ( 268 + "sol import: already imported on 2026-01-01T12:00:00 (2 entries); skipping" 269 + ) in captured.out 270 + assert captured.err == "" 271 + 272 + 273 + def test_duplicate_json_outputs_save_response_without_start( 274 + tmp_path: Path, capsys: pytest.CaptureFixture[str] 275 + ) -> None: 276 + media = tmp_path / "sample.txt" 277 + media.write_text("hello", encoding="utf-8") 278 + duplicate_response = { 279 + "schema_version": 1, 280 + "status": "duplicate", 281 + "recommended_action": "do_not_start", 282 + "path": "/journal/imports/20260101_120000/sample.txt", 283 + "timestamp": "20260101_120000", 284 + "duplicate": { 285 + "import_id": "20260101_120000", 286 + "imported_at": None, 287 + "entry_count": None, 288 + "state": "staged", 289 + }, 290 + } 291 + client = FakeClient(upload_response=duplicate_response) 292 + 293 + code = import_client.main( 294 + [str(media), "--json"], 295 + client=client, # type: ignore[arg-type] 296 + ) 297 + 298 + assert code == 0 299 + assert client.requests == [] 300 + assert json.loads(capsys.readouterr().out) == duplicate_response 223 301 224 302 225 303 def test_unreachable_is_clean(
+470 -17
tests/test_import_start.py
··· 3 3 4 4 from __future__ import annotations 5 5 6 + import hashlib 7 + import io 6 8 import json 7 - from importlib import import_module 8 9 from pathlib import Path 10 + from types import SimpleNamespace 9 11 10 12 import pytest 11 - from flask import Flask 13 + from flask import Flask, g 12 14 13 15 import solstone.convey.state as convey_state 14 16 import solstone.think.utils as think_utils 15 - from solstone.convey.reasons import IMPORT_CONFLICT, IMPORT_NOT_FOUND 17 + from solstone.convey.reasons import ( 18 + IMPORT_CLIENT_ID_CONFLICT, 19 + IMPORT_CONFLICT, 20 + IMPORT_NOT_FOUND, 21 + INVALID_OPERATION_FOR_STATE, 22 + MISSING_REQUIRED_FIELD, 23 + ) 24 + from solstone.think.importers.utils import ( 25 + read_import_metadata, 26 + update_import_metadata_fields, 27 + write_import_metadata, 28 + ) 16 29 17 - import_routes = import_module("solstone.apps.import.routes") 30 + import_routes = __import__("solstone.apps.import.routes", fromlist=["routes"]) 18 31 19 32 20 33 @pytest.fixture ··· 26 39 return tmp_path 27 40 28 41 42 + @pytest.fixture(autouse=True) 43 + def _stable_timestamp_detection(monkeypatch): 44 + monkeypatch.setattr( 45 + import_routes, 46 + "resolve_created_deterministic", 47 + lambda *args, **kwargs: None, 48 + ) 49 + 50 + def _no_model(*args, **kwargs): 51 + raise AssertionError("model timestamp detection should not run") 52 + 53 + monkeypatch.setattr(import_routes, "detect_created", _no_model) 54 + 55 + 29 56 @pytest.fixture 30 57 def client(journal_env): 31 58 app = Flask(__name__) 59 + 60 + @app.before_request 61 + def _identity() -> None: 62 + g.identity = SimpleNamespace(mode="local", fingerprint=None) 63 + 32 64 app.register_blueprint(import_routes.import_bp) 33 65 return app.test_client() 34 66 35 67 36 - def test_import_start_moves_staging_dir_and_updates_file_path( 68 + def _sha(content: bytes) -> str: 69 + return hashlib.sha256(content).hexdigest() 70 + 71 + 72 + def _import_dirs(journal_root: Path) -> list[Path]: 73 + return sorted( 74 + path for path in (journal_root / "imports").iterdir() if path.is_dir() 75 + ) 76 + 77 + 78 + def _save_upload( 79 + client, 80 + *, 81 + client_item_id: str, 82 + content: bytes = b"sample audio", 83 + filename: str = "sample.m4a", 84 + content_type: str = "audio/mp4", 85 + extra: dict | None = None, 86 + ): 87 + data = { 88 + "client_item_id": client_item_id, 89 + "deterministic_only": "true", 90 + "file": (io.BytesIO(content), filename, content_type), 91 + } 92 + if extra: 93 + data.update(extra) 94 + return client.post( 95 + "/app/import/api/save", 96 + data=data, 97 + content_type="multipart/form-data", 98 + ) 99 + 100 + 101 + def _write_manifest( 102 + journal_root: Path, 103 + *, 104 + import_id: str, 105 + source_hash: str, 106 + entry_count: int = 2, 107 + ) -> None: 108 + import_dir = journal_root / "imports" / import_id 109 + import_dir.mkdir(parents=True) 110 + (import_dir / "manifest.json").write_text( 111 + json.dumps( 112 + { 113 + "import_id": import_id, 114 + "source_type": "audio", 115 + "source_hash": source_hash, 116 + "entry_count": entry_count, 117 + "imported_at": "2026-01-01T12:00:00", 118 + "imported_via": "cli", 119 + } 120 + ), 121 + encoding="utf-8", 122 + ) 123 + 124 + 125 + def _write_staged_import( 126 + journal_root: Path, 127 + timestamp: str, 128 + metadata: dict, 129 + ) -> Path: 130 + import_dir = journal_root / "imports" / timestamp 131 + import_dir.mkdir(parents=True) 132 + media_path = import_dir / metadata.get("original_filename", "sample.m4a") 133 + media_path.write_bytes(b"sample") 134 + metadata = { 135 + "file_path": str(media_path), 136 + "user_timestamp": timestamp, 137 + "source": "audio", 138 + "source_inference": "extension", 139 + "client": {}, 140 + "facet": None, 141 + "setting": None, 142 + "imported_via": "web_dashboard", 143 + "observer_handle": None, 144 + "source_hint": None, 145 + "mime_type": "audio/mp4", 146 + **metadata, 147 + } 148 + write_import_metadata(journal_root, timestamp, metadata) 149 + return media_path 150 + 151 + 152 + def test_import_save_audio_upload_stages_versioned_summary(client, journal_env): 153 + response = _save_upload( 154 + client, 155 + client_item_id="ios-audio-1", 156 + content=b"audio bytes", 157 + extra={"facet": "work", "client": json.dumps({"device": "ios"})}, 158 + ) 159 + 160 + assert response.status_code == 200 161 + body = response.get_json() 162 + assert body["schema_version"] == 1 163 + assert body["status"] == "staged" 164 + assert body["replay"] is False 165 + assert body["source"] == "audio" 166 + assert body["client_item_id"] == "ios-audio-1" 167 + assert body["recommended_action"] == "start" 168 + assert body["facet"] == "work" 169 + assert body["metadata"] == { 170 + "original_filename": "sample.m4a", 171 + "mime_type": "audio/mp4", 172 + "imported_via": "web_dashboard", 173 + "observer_handle": None, 174 + "source_hint": None, 175 + "client": {"device": "ios"}, 176 + } 177 + assert body["diagnostics"]["source_inference"] == "extension" 178 + assert "timestamp_detection_method" not in body 179 + assert "dedup" not in body 180 + 181 + metadata = read_import_metadata(journal_env, body["timestamp"]) 182 + assert metadata["client_item_id"] == "ios-audio-1" 183 + assert metadata["source_hash"] == _sha(b"audio bytes") 184 + assert metadata["source"] == "audio" 185 + assert metadata["client"] == {"device": "ios"} 186 + 187 + 188 + def test_import_save_missing_client_item_id_returns_missing_required(client): 189 + response = client.post( 190 + "/app/import/api/save", 191 + data={ 192 + "deterministic_only": "true", 193 + "file": (io.BytesIO(b"audio"), "sample.m4a", "audio/mp4"), 194 + }, 195 + content_type="multipart/form-data", 196 + ) 197 + 198 + assert response.status_code == MISSING_REQUIRED_FIELD.status 199 + assert response.get_json()["reason_code"] == MISSING_REQUIRED_FIELD.code 200 + 201 + 202 + def test_import_save_replay_same_client_and_content_does_not_stage_again( 203 + client, journal_env 204 + ): 205 + first = _save_upload( 206 + client, 207 + client_item_id="ios-replay", 208 + content=b"same bytes", 209 + ) 210 + before_dirs = _import_dirs(journal_env) 211 + 212 + second = _save_upload( 213 + client, 214 + client_item_id="ios-replay", 215 + content=b"same bytes", 216 + ) 217 + 218 + assert second.status_code == 200 219 + body = second.get_json() 220 + assert body["status"] == "staged" 221 + assert body["replay"] is True 222 + assert body["recommended_action"] == "start" 223 + assert body["path"] == first.get_json()["path"] 224 + assert _import_dirs(journal_env) == before_dirs 225 + 226 + 227 + def test_import_save_replay_started_item_does_not_recommend_start(client, journal_env): 228 + first = _save_upload( 229 + client, 230 + client_item_id="ios-replay-started", 231 + content=b"same started bytes", 232 + ) 233 + timestamp = first.get_json()["timestamp"] 234 + update_import_metadata_fields( 235 + journal_root=journal_env, 236 + timestamp=timestamp, 237 + updates={"task_id": "task-started"}, 238 + ) 239 + before_dirs = _import_dirs(journal_env) 240 + 241 + second = _save_upload( 242 + client, 243 + client_item_id="ios-replay-started", 244 + content=b"same started bytes", 245 + ) 246 + 247 + assert second.status_code == 200 248 + body = second.get_json() 249 + assert body["status"] == "staged" 250 + assert body["replay"] is True 251 + assert body["recommended_action"] == "do_not_start" 252 + assert body["path"] == first.get_json()["path"] 253 + assert _import_dirs(journal_env) == before_dirs 254 + 255 + 256 + def test_import_save_same_client_different_content_conflicts(client): 257 + _save_upload(client, client_item_id="ios-conflict", content=b"one") 258 + 259 + response = _save_upload(client, client_item_id="ios-conflict", content=b"two") 260 + 261 + assert response.status_code == IMPORT_CLIENT_ID_CONFLICT.status 262 + assert response.get_json()["reason_code"] == IMPORT_CLIENT_ID_CONFLICT.code 263 + 264 + 265 + def test_import_save_duplicate_imported_content_is_terminal(client, journal_env): 266 + content = b"already imported" 267 + _write_manifest( 268 + journal_env, 269 + import_id="20260101_120000", 270 + source_hash=_sha(content), 271 + entry_count=3, 272 + ) 273 + before_dirs = _import_dirs(journal_env) 274 + 275 + response = _save_upload( 276 + client, 277 + client_item_id="ios-imported-dup", 278 + content=content, 279 + ) 280 + 281 + assert response.status_code == 200 282 + body = response.get_json() 283 + assert body["status"] == "duplicate" 284 + assert body["replay"] is False 285 + assert body["recommended_action"] == "do_not_start" 286 + assert body["duplicate"] == { 287 + "import_id": "20260101_120000", 288 + "imported_at": "2026-01-01T12:00:00", 289 + "entry_count": 3, 290 + "state": "imported", 291 + } 292 + assert _import_dirs(journal_env) == before_dirs 293 + assert not any((path / "import.json").exists() for path in before_dirs) 294 + 295 + 296 + def test_import_save_duplicate_staged_content_is_terminal(client, journal_env): 297 + content = b"already staged" 298 + _write_staged_import( 299 + journal_env, 300 + "20260101_121500", 301 + { 302 + "original_filename": "existing.m4a", 303 + "client_item_id": "other-client", 304 + "source_hash": _sha(content), 305 + }, 306 + ) 307 + before_dirs = _import_dirs(journal_env) 308 + 309 + response = _save_upload( 310 + client, 311 + client_item_id="ios-staged-dup", 312 + content=content, 313 + ) 314 + 315 + assert response.status_code == 200 316 + body = response.get_json() 317 + assert body["status"] == "duplicate" 318 + assert body["client_item_id"] == "ios-staged-dup" 319 + assert body["recommended_action"] == "do_not_start" 320 + assert body["duplicate"] == { 321 + "import_id": "20260101_121500", 322 + "imported_at": None, 323 + "entry_count": None, 324 + "state": "staged", 325 + } 326 + assert _import_dirs(journal_env) == before_dirs 327 + 328 + 329 + def test_import_meta_updates_facet_and_setting(client, journal_env): 330 + media_path = _write_staged_import( 331 + journal_env, 332 + "20260101_130000", 333 + {"original_filename": "meta.m4a", "client_item_id": "meta-client"}, 334 + ) 335 + 336 + response = client.post( 337 + "/app/import/api/meta", 338 + json={"path": str(media_path), "facet": "work", "setting": "office"}, 339 + ) 340 + 341 + assert response.status_code == 200 342 + assert response.get_json() == { 343 + "status": "ok", 344 + "path": str(media_path), 345 + "timestamp": "20260101_130000", 346 + "updated": {"facet": "work", "setting": "office"}, 347 + } 348 + metadata = read_import_metadata(journal_env, "20260101_130000") 349 + assert metadata["facet"] == "work" 350 + assert metadata["setting"] == "office" 351 + 352 + 353 + def test_import_meta_missing_path_returns_missing_required(client): 354 + response = client.post("/app/import/api/meta", json={}) 355 + 356 + assert response.status_code == MISSING_REQUIRED_FIELD.status 357 + assert response.get_json()["reason_code"] == MISSING_REQUIRED_FIELD.code 358 + 359 + 360 + def test_import_facet_route_removed_no_alias(client): 361 + rules = list(client.application.url_map.iter_rules()) 362 + 363 + assert not any(rule.rule == "/app/import/api/facet" for rule in rules) 364 + assert any( 365 + rule.rule == "/app/import/api/meta" and "POST" in rule.methods for rule in rules 366 + ) 367 + 368 + 369 + def test_import_meta_missing_item_returns_import_not_found(client, journal_env): 370 + missing_path = journal_env / "imports" / "20260101_130001" / "sample.m4a" 371 + 372 + response = client.post( 373 + "/app/import/api/meta", 374 + json={"path": str(missing_path), "facet": "work"}, 375 + ) 376 + 377 + assert response.status_code == IMPORT_NOT_FOUND.status 378 + assert response.get_json()["reason_code"] == IMPORT_NOT_FOUND.code 379 + 380 + 381 + def test_import_meta_started_item_returns_invalid_operation(client, journal_env): 382 + media_path = _write_staged_import( 383 + journal_env, 384 + "20260101_130002", 385 + { 386 + "original_filename": "started.m4a", 387 + "client_item_id": "started-client", 388 + "task_id": "task-1", 389 + }, 390 + ) 391 + 392 + response = client.post( 393 + "/app/import/api/meta", 394 + json={"path": str(media_path), "facet": "work"}, 395 + ) 396 + 397 + assert response.status_code == INVALID_OPERATION_FOR_STATE.status 398 + assert response.get_json()["reason_code"] == INVALID_OPERATION_FOR_STATE.code 399 + 400 + 401 + def test_import_start_moves_dir_uses_saved_metadata_and_omits_generic_source( 37 402 client, journal_env, monkeypatch 38 403 ): 39 404 emitted: list[dict[str, object]] = [] ··· 46 411 ) 47 412 old_ts = "20260101_120000" 48 413 new_ts = "20260101_121500" 49 - import_dir = journal_env / "imports" / old_ts 50 - import_dir.mkdir() 51 - media_path = import_dir / "sample.m4a" 52 - media_path.write_bytes(b"sample") 53 - (import_dir / "import.json").write_text( 54 - json.dumps({"facet": "work"}), encoding="utf-8" 414 + media_path = _write_staged_import( 415 + journal_env, 416 + old_ts, 417 + { 418 + "original_filename": "sample.m4a", 419 + "client_item_id": "start-client", 420 + "facet": "work", 421 + "setting": "office", 422 + "source_hint": None, 423 + }, 55 424 ) 56 425 57 426 response = client.post( ··· 59 428 json={ 60 429 "path": str(media_path), 61 430 "timestamp": new_ts, 62 - "source": "audio", 431 + "source": "recording", 432 + "facet": "ignored", 63 433 "force": True, 64 434 }, 65 435 ) 66 436 67 437 assert response.status_code == 200 68 - assert response.get_json()["task_id"] 69 438 new_dir = journal_env / "imports" / new_ts 70 439 assert new_dir.exists() 71 - assert not import_dir.exists() 72 - metadata = json.loads((new_dir / "import.json").read_text(encoding="utf-8")) 440 + assert not (journal_env / "imports" / old_ts).exists() 441 + metadata = read_import_metadata(journal_env, new_ts) 73 442 assert metadata["file_path"] == str(new_dir / media_path.name) 74 443 assert emitted == [ 75 444 { ··· 83 452 new_ts, 84 453 "--facet", 85 454 "work", 86 - "--source", 87 - "audio", 455 + "--setting", 456 + "office", 88 457 "--force", 89 458 ], 90 459 } 91 460 ] 92 461 93 462 463 + def test_import_start_forwards_only_saved_source_hint(client, journal_env, monkeypatch): 464 + emitted: list[dict[str, object]] = [] 465 + monkeypatch.setattr( 466 + import_routes, 467 + "emit", 468 + lambda tract, event, **payload: emitted.append( 469 + {"tract": tract, "event": event, **payload} 470 + ), 471 + ) 472 + ts = "20260101_122000" 473 + media_path = _write_staged_import( 474 + journal_env, 475 + ts, 476 + { 477 + "original_filename": "vault", 478 + "client_item_id": "source-hint-client", 479 + "source_hint": "obsidian", 480 + }, 481 + ) 482 + 483 + response = client.post( 484 + "/app/import/api/start", 485 + json={"path": str(media_path), "timestamp": ts, "source": "quick"}, 486 + ) 487 + 488 + assert response.status_code == 200 489 + assert emitted[0]["cmd"] == [ 490 + "journal", 491 + "importer", 492 + str(media_path), 493 + ts, 494 + "--source", 495 + "obsidian", 496 + ] 497 + 498 + 499 + def test_import_start_refuses_terminal_duplicate_even_with_force( 500 + client, journal_env, monkeypatch 501 + ): 502 + emitted: list[dict[str, object]] = [] 503 + monkeypatch.setattr( 504 + import_routes, 505 + "emit", 506 + lambda tract, event, **payload: emitted.append( 507 + {"tract": tract, "event": event, **payload} 508 + ), 509 + ) 510 + content = b"terminal" 511 + source_hash = _sha(content) 512 + ts = "20260101_123000" 513 + media_path = _write_staged_import( 514 + journal_env, 515 + ts, 516 + { 517 + "original_filename": "terminal.m4a", 518 + "client_item_id": "terminal-client", 519 + "source_hash": source_hash, 520 + }, 521 + ) 522 + _write_manifest( 523 + journal_env, 524 + import_id="20260101_124500", 525 + source_hash=source_hash, 526 + ) 527 + 528 + response = client.post( 529 + "/app/import/api/start", 530 + json={"path": str(media_path), "timestamp": ts, "force": True}, 531 + ) 532 + 533 + assert response.status_code == INVALID_OPERATION_FOR_STATE.status 534 + assert response.get_json()["reason_code"] == INVALID_OPERATION_FOR_STATE.code 535 + assert emitted == [] 536 + 537 + 94 538 def test_import_start_missing_source_returns_import_not_found(client, journal_env): 95 539 old_ts = "20260101_120000" 96 540 new_ts = "20260101_121500" ··· 115 559 new_dir.mkdir() 116 560 media_path = old_dir / "sample.m4a" 117 561 media_path.write_bytes(b"sample") 562 + write_import_metadata( 563 + journal_env, 564 + old_ts, 565 + { 566 + "file_path": str(media_path), 567 + "user_timestamp": old_ts, 568 + "client_item_id": "conflict-client", 569 + }, 570 + ) 118 571 119 572 response = client.post( 120 573 "/app/import/api/start",
+19 -12
tests/test_importer.py
··· 8 8 import os 9 9 import subprocess 10 10 import time 11 + import uuid 11 12 import zipfile 12 13 from io import BytesIO 13 14 from pathlib import Path ··· 127 128 def _post_import_save(client, data: dict): 128 129 payload = { 129 130 "file": (BytesIO(b"hello"), "note.txt"), 131 + "client_item_id": uuid.uuid4().hex, 130 132 **data, 131 133 } 132 134 return client.post( ··· 319 321 assert response.status_code == 200 320 322 body = response.get_json() 321 323 assert body["timestamp"] == "20240115_103000" 322 - assert body["timestamp_detection_method"] == "deterministic" 323 - assert body["timestamp_detection_model_called"] is False 324 - assert body["timestamp_detection_no_match_reason"] is None 324 + assert body["diagnostics"]["timestamp_detection_method"] == "deterministic" 325 + assert body["diagnostics"]["timestamp_detection_model_called"] is False 326 + assert body["diagnostics"]["timestamp_detection_no_match_reason"] is None 325 327 metadata = _read_import_metadata(tmp_path, body["timestamp"]) 326 328 assert metadata["detection_result"] == deterministic_result 327 329 assert metadata["detected_timestamp"] == "20240115_103000" ··· 352 354 "%Y%m%d_%H%M%S" 353 355 ) 354 356 assert body["timestamp"] == expected_timestamp 355 - assert body["timestamp_detection_method"] == "upload_fallback" 356 - assert body["timestamp_detection_model_called"] is False 357 - assert body["timestamp_detection_no_match_reason"] == "no_deterministic_match" 357 + assert body["diagnostics"]["timestamp_detection_method"] == "upload_fallback" 358 + assert body["diagnostics"]["timestamp_detection_model_called"] is False 359 + assert ( 360 + body["diagnostics"]["timestamp_detection_no_match_reason"] 361 + == "no_deterministic_match" 362 + ) 358 363 metadata = _read_import_metadata(tmp_path, body["timestamp"]) 359 364 assert metadata["detected_timestamp"] is None 360 365 assert metadata["user_timestamp"] == body["timestamp"] ··· 379 384 assert response.status_code == 200 380 385 body = response.get_json() 381 386 assert body["timestamp"] == "20240115_103000" 382 - assert body["timestamp_detection_method"] == "model" 383 - assert body["timestamp_detection_model_called"] is True 384 - assert body["timestamp_detection_no_match_reason"] is None 387 + assert body["diagnostics"]["timestamp_detection_method"] == "model" 388 + assert body["diagnostics"]["timestamp_detection_model_called"] is True 389 + assert body["diagnostics"]["timestamp_detection_no_match_reason"] is None 385 390 metadata = _read_import_metadata(tmp_path, body["timestamp"]) 386 391 assert metadata["detection_result"] == model_result 387 392 assert metadata["timestamp_detection_method"] == "model" ··· 401 406 402 407 assert response.status_code == 200 403 408 body = response.get_json() 404 - assert body["timestamp_detection_method"] == "upload_fallback" 405 - assert body["timestamp_detection_model_called"] is True 406 - assert body["timestamp_detection_no_match_reason"] == "model_no_match" 409 + assert body["diagnostics"]["timestamp_detection_method"] == "upload_fallback" 410 + assert body["diagnostics"]["timestamp_detection_model_called"] is True 411 + assert ( 412 + body["diagnostics"]["timestamp_detection_no_match_reason"] == "model_no_match" 413 + ) 407 414 metadata = _read_import_metadata(tmp_path, body["timestamp"]) 408 415 assert metadata["detected_timestamp"] is None 409 416 assert metadata["timestamp_detection_method"] == "upload_fallback"
+66
tests/test_media.py
··· 1 + # SPDX-License-Identifier: AGPL-3.0-only 2 + # Copyright (c) 2026 sol pbc 3 + 4 + """Tests for media source inference helpers.""" 5 + 6 + import pytest 7 + 8 + from solstone.think.media import canonical_source, canonical_source_signal 9 + 10 + 11 + @pytest.mark.parametrize( 12 + ("filename", "expected"), 13 + [ 14 + ("recording.m4a", "audio"), 15 + ("recording.mp3", "audio"), 16 + ("recording.wav", "audio"), 17 + ("clip.mp4", "audio"), 18 + ("clip.mov", "audio"), 19 + ("clip.webm", "audio"), 20 + ("photo.png", "image"), 21 + ("photo.jpg", "image"), 22 + ("photo.heic", "image"), 23 + ("doc.pdf", "document"), 24 + ("note.txt", "text"), 25 + ("note.md", "text"), 26 + ("", "text"), 27 + ], 28 + ) 29 + def test_canonical_source_from_filename(filename, expected): 30 + assert canonical_source(filename=filename) == expected 31 + 32 + 33 + @pytest.mark.parametrize( 34 + ("content_type", "expected"), 35 + [ 36 + ("audio/mp4", "audio"), 37 + ("image/png", "image"), 38 + ("application/pdf", "document"), 39 + ("text/plain", "text"), 40 + ], 41 + ) 42 + def test_canonical_source_from_content_type(content_type, expected): 43 + assert canonical_source(content_type=content_type) == expected 44 + 45 + 46 + def test_canonical_source_prefers_extension_over_content_type(): 47 + assert canonical_source(filename="doc.pdf", content_type="text/plain") == "document" 48 + 49 + 50 + @pytest.mark.parametrize( 51 + ("filename", "content_type", "expected"), 52 + [ 53 + ("recording.m4a", None, "extension"), 54 + ("photo.png", "text/plain", "extension"), 55 + (None, "audio/mp4", "content_type"), 56 + (None, "image/png", "content_type"), 57 + (None, "application/pdf", "content_type"), 58 + ("note.txt", "text/plain", "default"), 59 + (None, "text/plain", "default"), 60 + ], 61 + ) 62 + def test_canonical_source_signal(filename, content_type, expected): 63 + assert ( 64 + canonical_source_signal(filename=filename, content_type=content_type) 65 + == expected 66 + )
+5 -1
tests/test_openapi_contract.py
··· 48 48 "/api/voice/session", 49 49 "/api/voice/status", 50 50 "/app/home/api/pulse", 51 + "/app/import/api/meta", 51 52 "/app/import/api/save", 53 + "/app/import/api/save-path", 52 54 "/app/import/api/start", 53 55 "/app/network/api/status", 54 56 "/app/network/local-endpoints", ··· 79 81 ("GET", "/api/voice/observer-actions", "voice.observerActions"), 80 82 ("GET", "/api/voice/status", "voice.status"), 81 83 ("GET", "/app/home/api/pulse", "home.pulse"), 84 + ("POST", "/app/import/api/meta", "import.meta"), 82 85 ("POST", "/app/import/api/save", "import.save"), 86 + ("POST", "/app/import/api/save-path", "import.savePath"), 83 87 ("POST", "/app/import/api/start", "import.start"), 84 88 ("DELETE", "/app/observer/source/{stream}", "observer.deleteSource"), 85 89 } ··· 338 342 assert "/api/config/convey" not in document["paths"] 339 343 assert "/api/system/status" not in document["paths"] 340 344 assert set(document["paths"]) == CONTRACTED_PATHS 341 - assert len(document["paths"]) == 28 345 + assert len(document["paths"]) == 30 342 346 343 347 344 348 def test_home_pulse_named_fields_present(contract_app):