solpbc.org / solstone
personal memory agent
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

Renew SPB credentials during long backups

author
Jer Miller date (Jul 13, 2026, 7:05 AM -0600) commit a56ccad5 parent fceb0028
+414 -77
+1 -2
solstone/apps/backup/routes.py
··· 393 393 except HostedCredsUnavailable as exc: 394 394 return OpOutcome("error", exc.reason_code) 395 395 396 - destination = operated_destination(binding, creds) 397 396 save_hosted_binding(binding) 398 - restore_result = restore_journal_operated(destination, recovery_key) 397 + restore_result = restore_journal_operated(binding, creds, recovery_key) 399 398 return OpOutcome( 400 399 status=restore_result.status, 401 400 reason_code=restore_result.reason_code,
+2 -2
solstone/apps/backup/tests/test_maintenance_routines.py
··· 19 19 assert "backup:run" in routines 20 20 assert "backup:prune" in routines 21 21 assert routines["backup:run"].every == "hourly" 22 - assert routines["backup:run"].max_runtime == "7h" 22 + assert routines["backup:run"].max_runtime == "49h" 23 23 assert routines["backup:prune"].every == "daily" 24 24 assert routines["backup:prune"].max_runtime == "3h" 25 25 assert expected_schedule_entry("backup:run", routines["backup:run"]) == { 26 26 "cmd": ["journal", "maintenance", "run", "backup:run"], 27 27 "every": "hourly", 28 28 "enabled": True, 29 - "max_runtime": "7h", 29 + "max_runtime": "49h", 30 30 } 31 31 assert expected_schedule_entry("backup:prune", routines["backup:prune"]) == { 32 32 "cmd": ["journal", "maintenance", "run", "backup:prune"],
+2 -4
solstone/apps/backup/tests/test_routes.py
··· 63 63 secret_access_key="SAK", 64 64 session_token="SESS", 65 65 endpoint="https://r2.example", 66 + expires_at="2026-07-13T12:00:00Z", 66 67 ) 67 68 68 69 ··· 549 550 assert response.get_json()["operation"]["portal_url"] == CONSENT_URL 550 551 assert final["reason_code"] is None 551 552 save_hosted_binding.assert_called_once_with(binding) 552 - restore_journal_operated.assert_called_once() 553 - destination = restore_journal_operated.call_args.args[0] 554 - assert destination.credentials["session_token"] == "SESS" 555 - assert restore_journal_operated.call_args.args[1] == "A" * 64 553 + restore_journal_operated.assert_called_once_with(binding, _creds(), "A" * 64) 556 554 557 555 558 556 def test_restore_hosted_needs_subscription_returns_terminal_phase(
+96 -44
solstone/think/backup/engine.py
··· 7 7 8 8 import logging 9 9 import time 10 - from collections.abc import Mapping 10 + from collections.abc import Iterator, Mapping 11 + from contextlib import contextmanager 11 12 from dataclasses import dataclass 12 13 from pathlib import Path 13 14 ··· 16 17 assemble_backend_env, 17 18 ) 18 19 from solstone.think.backup.hosted import ( 20 + HostedBinding, 21 + HostedCredentials, 19 22 HostedCredsUnavailable, 20 23 fetch_hosted_credentials, 21 24 load_hosted_binding, 22 25 operated_destination, 23 26 ) 27 + from solstone.think.backup.hosted_provider import hosted_restic_session 24 28 from solstone.think.backup.install import ensure_restic 25 29 from solstone.think.backup.runner import ( 26 30 reason_for_returncode, ··· 67 71 PRUNE_MAX_REPACK_SIZE = "1G" 68 72 UNLOCK_TIMEOUT_SECONDS = 5 * 60 69 73 BACKUP_TIMEOUT_SECONDS = 6 * 60 * 60 74 + INITIAL_BACKUP_TIMEOUT_SECONDS = 48 * 60 * 60 70 75 PRUNE_TIMEOUT_SECONDS = 2 * 60 * 60 71 - BACKUP_MAX_RUNTIME = "7h" 76 + BACKUP_MAX_RUNTIME = "49h" 72 77 PRUNE_MAX_RUNTIME = "3h" 73 78 BACKUP_RUN_CMD = ["journal", "maintenance", "run", "backup:run"] 74 79 ··· 91 96 destination: Destination 92 97 keys: BackupKeys 93 98 restic_path: Path 99 + binding: HostedBinding | None = None 100 + hosted_credentials: HostedCredentials | None = None 94 101 95 102 96 103 class _ResticUnavailable(RuntimeError): ··· 113 120 creds = fetch_hosted_credentials(binding, scope=scope) 114 121 destination = operated_destination(binding, creds) 115 122 else: 123 + binding = None 124 + creds = None 116 125 destination = get_destination() 117 126 if destination is None: 118 127 return None ··· 122 131 except Exception as exc: 123 132 raise _ResticUnavailable from exc 124 133 125 - return _Runtime(destination=destination, keys=keys, restic_path=restic_path) 134 + return _Runtime( 135 + destination=destination, 136 + keys=keys, 137 + restic_path=restic_path, 138 + binding=binding, 139 + hosted_credentials=creds, 140 + ) 126 141 127 142 128 143 def _backup_args() -> list[str]: ··· 149 164 return None 150 165 151 166 167 + @contextmanager 168 + def _runtime_backend( 169 + runtime: _Runtime, 170 + *, 171 + scope: str, 172 + operation: str, 173 + ) -> Iterator[tuple[Destination, Mapping[str, str | None]] | None]: 174 + if runtime.binding is not None and runtime.hosted_credentials is not None: 175 + with hosted_restic_session( 176 + runtime.binding, 177 + scope=scope, 178 + initial_credentials=runtime.hosted_credentials, 179 + ) as session: 180 + yield session.destination, session.backend_env 181 + return 182 + 183 + backend_env = _assemble_backend_env(runtime.destination, operation=operation) 184 + if backend_env is None: 185 + yield None 186 + return 187 + yield runtime.destination, backend_env 188 + 189 + 190 + def _backup_timeout() -> int: 191 + last_backup = get_backup_config()["last_backup"] 192 + snapshot_id = last_backup.get("snapshot_id") 193 + if ( 194 + last_backup.get("status") != "ok" 195 + or not isinstance(snapshot_id, str) 196 + or not snapshot_id 197 + ): 198 + return INITIAL_BACKUP_TIMEOUT_SECONDS 199 + return BACKUP_TIMEOUT_SECONDS 200 + 201 + 152 202 def _recover_stale_lock( 153 203 runtime: _Runtime, 204 + destination: Destination, 154 205 backend_env: Mapping[str, str | None], 155 206 ) -> None: 156 207 result = run_restic( 157 208 ["unlock"], 158 - repository=runtime.destination.repository, 209 + repository=destination.repository, 159 210 password=runtime.keys.daily_key, 160 211 restic_path=runtime.restic_path, 161 212 backend_env=backend_env, ··· 219 270 if runtime is None: 220 271 return BackupResult(status="skipped", snapshot_id=None, error_reason=None) 221 272 222 - backend_env = _assemble_backend_env(runtime.destination, operation="run") 223 - if backend_env is None: 224 - return _record_backup_error(reason="failed") 225 - 226 - _recover_stale_lock(runtime, backend_env) 227 - result = run_restic( 228 - _backup_args(), 229 - repository=runtime.destination.repository, 230 - password=runtime.keys.daily_key, 231 - restic_path=runtime.restic_path, 232 - backend_env=backend_env, 233 - json=True, 234 - timeout=BACKUP_TIMEOUT_SECONDS, 235 - ) 273 + with _runtime_backend(runtime, scope="backup", operation="run") as backend: 274 + if backend is None: 275 + return _record_backup_error(reason="failed") 276 + destination, backend_env = backend 277 + _recover_stale_lock(runtime, destination, backend_env) 278 + result = run_restic( 279 + _backup_args(), 280 + repository=destination.repository, 281 + password=runtime.keys.daily_key, 282 + restic_path=runtime.restic_path, 283 + backend_env=backend_env, 284 + json=True, 285 + timeout=_backup_timeout(), 286 + ) 236 287 summary = select_summary(result.json) 237 288 snapshot_id = None 238 289 if summary is not None: ··· 288 339 if runtime is None: 289 340 return PruneResult(status="skipped", error_reason=None) 290 341 291 - backend_env = _assemble_backend_env(runtime.destination, operation="prune") 292 - if backend_env is None: 293 - return _record_prune_error(reason="failed") 294 - 295 - _recover_stale_lock(runtime, backend_env) 296 - retention = get_backup_config()["retention"] 297 - result = run_restic( 298 - [ 299 - "forget", 300 - "--keep-hourly", 301 - str(retention.get("hourly", 24)), 302 - "--keep-daily", 303 - str(retention.get("daily", 7)), 304 - "--keep-weekly", 305 - str(retention.get("weekly", 4)), 306 - "--keep-monthly", 307 - str(retention.get("monthly", 12)), 308 - "--prune", 309 - ], 310 - repository=runtime.destination.repository, 311 - password=runtime.keys.daily_key, 312 - restic_path=runtime.restic_path, 313 - backend_env=backend_env, 314 - timeout=PRUNE_TIMEOUT_SECONDS, 315 - max_repack_size=PRUNE_MAX_REPACK_SIZE, 316 - ) 342 + with _runtime_backend(runtime, scope="maintenance", operation="prune") as backend: 343 + if backend is None: 344 + return _record_prune_error(reason="failed") 345 + destination, backend_env = backend 346 + _recover_stale_lock(runtime, destination, backend_env) 347 + retention = get_backup_config()["retention"] 348 + result = run_restic( 349 + [ 350 + "forget", 351 + "--keep-hourly", 352 + str(retention.get("hourly", 24)), 353 + "--keep-daily", 354 + str(retention.get("daily", 7)), 355 + "--keep-weekly", 356 + str(retention.get("weekly", 4)), 357 + "--keep-monthly", 358 + str(retention.get("monthly", 12)), 359 + "--prune", 360 + ], 361 + repository=destination.repository, 362 + password=runtime.keys.daily_key, 363 + restic_path=runtime.restic_path, 364 + backend_env=backend_env, 365 + timeout=PRUNE_TIMEOUT_SECONDS, 366 + max_repack_size=PRUNE_MAX_REPACK_SIZE, 367 + ) 317 368 318 369 if result.returncode == 0: 319 370 record_prune_result( ··· 343 394 __all__ = [ 344 395 "BACKUP_MAX_RUNTIME", 345 396 "BACKUP_TIMEOUT_SECONDS", 397 + "INITIAL_BACKUP_TIMEOUT_SECONDS", 346 398 "BackupResult", 347 399 "PRUNE_MAX_REPACK_SIZE", 348 400 "PRUNE_MAX_RUNTIME",
+4
solstone/think/backup/hosted.py
··· 52 52 secret_access_key: str 53 53 session_token: str 54 54 endpoint: str 55 + expires_at: str 55 56 56 57 def __repr__(self) -> str: 57 58 return ( ··· 226 227 secret_access_key = _non_blank_string(payload, "secret_access_key") 227 228 session_token = _non_blank_string(payload, "session_token") 228 229 endpoint = _non_blank_string(payload, "endpoint") 230 + expires_at = _non_blank_string(payload, "expires_at") 229 231 if ( 230 232 access_key_id is None 231 233 or secret_access_key is None 232 234 or session_token is None 233 235 or endpoint is None 236 + or expires_at is None 234 237 ): 235 238 logger.warning("hosted credential broker response was incomplete") 236 239 raise HostedCredsUnavailable("broker_error") ··· 240 243 secret_access_key=secret_access_key, 241 244 session_token=session_token, 242 245 endpoint=endpoint, 246 + expires_at=expires_at, 243 247 ) 244 248 245 249
+143
solstone/think/backup/hosted_provider.py
··· 1 + # SPDX-License-Identifier: AGPL-3.0-only 2 + # Copyright (c) 2026 sol pbc 3 + 4 + """Authenticated loopback credential provider for long operated restic runs.""" 5 + 6 + from __future__ import annotations 7 + 8 + import json 9 + import secrets 10 + import threading 11 + from collections.abc import Iterator, Mapping 12 + from contextlib import contextmanager 13 + from dataclasses import dataclass 14 + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer 15 + 16 + from solstone.think.backup.destination import Destination 17 + from solstone.think.backup.hosted import ( 18 + HostedBinding, 19 + HostedCredentials, 20 + HostedCredsUnavailable, 21 + fetch_hosted_credentials, 22 + operated_repository, 23 + ) 24 + 25 + _CREDENTIAL_PATH = "/credentials" 26 + 27 + 28 + @dataclass(frozen=True) 29 + class HostedResticSession: 30 + destination: Destination 31 + backend_env: Mapping[str, str] 32 + 33 + 34 + class _CredentialState: 35 + def __init__( 36 + self, 37 + binding: HostedBinding, 38 + scope: str, 39 + initial_credentials: HostedCredentials, 40 + ) -> None: 41 + self.binding = binding 42 + self.scope = scope 43 + self._initial_credentials: HostedCredentials | None = initial_credentials 44 + self._lock = threading.Lock() 45 + 46 + def next_credentials(self) -> HostedCredentials: 47 + with self._lock: 48 + if self._initial_credentials is not None: 49 + credentials = self._initial_credentials 50 + self._initial_credentials = None 51 + return credentials 52 + return fetch_hosted_credentials(self.binding, scope=self.scope) 53 + 54 + 55 + class _CredentialServer(ThreadingHTTPServer): 56 + daemon_threads = True 57 + 58 + def __init__( 59 + self, 60 + state: _CredentialState, 61 + authorization_token: str, 62 + ) -> None: 63 + self.state = state 64 + self.authorization_token = authorization_token 65 + super().__init__(("127.0.0.1", 0), _CredentialHandler) 66 + 67 + 68 + class _CredentialHandler(BaseHTTPRequestHandler): 69 + server: _CredentialServer 70 + 71 + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler contract 72 + if self.path != _CREDENTIAL_PATH: 73 + self.send_error(404) 74 + return 75 + if self.headers.get("Authorization") != self.server.authorization_token: 76 + self.send_error(401) 77 + return 78 + 79 + try: 80 + credentials = self.server.state.next_credentials() 81 + except HostedCredsUnavailable: 82 + self.send_error(503) 83 + return 84 + 85 + body = json.dumps( 86 + { 87 + "AccessKeyId": credentials.access_key_id, 88 + "SecretAccessKey": credentials.secret_access_key, 89 + "Token": credentials.session_token, 90 + "Expiration": credentials.expires_at, 91 + }, 92 + separators=(",", ":"), 93 + ).encode("utf-8") 94 + self.send_response(200) 95 + self.send_header("Content-Type", "application/json") 96 + self.send_header("Content-Length", str(len(body))) 97 + self.end_headers() 98 + self.wfile.write(body) 99 + 100 + def log_message(self, _format: str, *_args: object) -> None: 101 + return 102 + 103 + 104 + @contextmanager 105 + def hosted_restic_session( 106 + binding: HostedBinding, 107 + *, 108 + scope: str, 109 + initial_credentials: HostedCredentials | None = None, 110 + ) -> Iterator[HostedResticSession]: 111 + credentials = initial_credentials or fetch_hosted_credentials(binding, scope=scope) 112 + authorization_token = secrets.token_urlsafe(32) 113 + state = _CredentialState(binding, scope, credentials) 114 + server = _CredentialServer(state, authorization_token) 115 + thread = threading.Thread( 116 + target=server.serve_forever, 117 + kwargs={"poll_interval": 0.05}, 118 + name="spb-credential-provider", 119 + daemon=True, 120 + ) 121 + thread.start() 122 + host, port = server.server_address 123 + try: 124 + yield HostedResticSession( 125 + destination=Destination( 126 + repository=operated_repository(binding, credentials), 127 + backend="s3", 128 + credentials={}, 129 + ), 130 + backend_env={ 131 + "AWS_CONTAINER_CREDENTIALS_FULL_URI": ( 132 + f"http://{host}:{port}{_CREDENTIAL_PATH}" 133 + ), 134 + "AWS_CONTAINER_AUTHORIZATION_TOKEN": authorization_token, 135 + }, 136 + ) 137 + finally: 138 + server.shutdown() 139 + server.server_close() 140 + thread.join() 141 + 142 + 143 + __all__ = ["HostedResticSession", "hosted_restic_session"]
+25 -9
solstone/think/backup/restore.py
··· 6 6 from __future__ import annotations 7 7 8 8 import logging 9 - from collections.abc import Callable 9 + from collections.abc import Callable, Mapping 10 10 from dataclasses import dataclass 11 11 from typing import Any 12 12 13 13 from solstone.think.backup.destination import Destination, assemble_backend_env 14 + from solstone.think.backup.hosted import HostedBinding, HostedCredentials 15 + from solstone.think.backup.hosted_provider import hosted_restic_session 14 16 from solstone.think.backup.install import ensure_restic 15 17 from solstone.think.backup.keys import parse_recovery_key 16 18 from solstone.think.backup.runner import ( ··· 31 33 logger = logging.getLogger("solstone.backup.restore") 32 34 33 35 RESTORE_LIST_TIMEOUT_SECONDS = 5 * 60 34 - RESTORE_TIMEOUT_SECONDS = 6 * 60 * 60 35 - RESTORE_CHECK_TIMEOUT_SECONDS = 60 * 60 36 + RESTORE_TIMEOUT_SECONDS = 48 * 60 * 60 37 + RESTORE_CHECK_TIMEOUT_SECONDS = 6 * 60 * 60 36 38 37 39 38 40 @dataclass(frozen=True) ··· 88 90 destination: Destination, 89 91 entered_recovery_key: str, 90 92 persist: Callable[[str], None], 93 + *, 94 + backend_env: Mapping[str, str | None] | None = None, 91 95 ) -> RestoreResult: 92 96 try: 93 97 canonical = parse_recovery_key(entered_recovery_key) 94 98 except ValueError: 95 99 return _restore_error("invalid_key") 96 100 97 - try: 98 - backend_env = assemble_backend_env(destination) 99 - except (KeyError, ValueError): 100 - return _restore_error("failed") 101 + if backend_env is None: 102 + try: 103 + backend_env = assemble_backend_env(destination) 104 + except (KeyError, ValueError): 105 + return _restore_error("failed") 101 106 102 107 try: 103 108 restic_path = ensure_restic() ··· 198 203 199 204 200 205 def restore_journal_operated( 201 - destination: Destination, 206 + binding: HostedBinding, 207 + initial_credentials: HostedCredentials, 202 208 entered_recovery_key: str, 203 209 ) -> RestoreResult: 204 210 def persist(canonical: str) -> None: ··· 206 212 set_recovery_key(canonical) 207 213 set_recovery_key_confirmed(True) 208 214 209 - return _run_restore(destination, entered_recovery_key, persist) 215 + with hosted_restic_session( 216 + binding, 217 + scope="backup", 218 + initial_credentials=initial_credentials, 219 + ) as session: 220 + return _run_restore( 221 + session.destination, 222 + entered_recovery_key, 223 + persist, 224 + backend_env=session.backend_env, 225 + ) 210 226 211 227 212 228 __all__ = [
+38 -7
tests/test_backup_engine.py
··· 262 262 "AWS_SECRET_ACCESS_KEY": "secret-key", 263 263 }, 264 264 "json": True, 265 - "timeout": engine.BACKUP_TIMEOUT_SECONDS, 265 + "timeout": engine.INITIAL_BACKUP_TIMEOUT_SECONDS, 266 266 }, 267 267 ) 268 268 record_backup_result.assert_called_once_with( ··· 661 661 secret_access_key="SAK", 662 662 session_token="SESS", 663 663 endpoint="https://acct.r2.cloudflarestorage.com", 664 + expires_at="2026-07-13T12:00:00Z", 664 665 ) 665 666 666 667 def fake_run_restic(args: list[str], **kwargs: Any) -> ResticResult: ··· 682 683 assert result.status == "ok" 683 684 backup_call = next(call for call in calls if call[0][0] == "backup") 684 685 backup_kwargs = backup_call[1] 685 - assert backup_kwargs["backend_env"] == { 686 - "AWS_ACCESS_KEY_ID": "AKID", 687 - "AWS_SECRET_ACCESS_KEY": "SAK", 688 - "AWS_SESSION_TOKEN": "SESS", 689 - } 686 + backend_env = backup_kwargs["backend_env"] 687 + assert backend_env["AWS_CONTAINER_CREDENTIALS_FULL_URI"].startswith( 688 + "http://127.0.0.1:" 689 + ) 690 + assert backend_env["AWS_CONTAINER_AUTHORIZATION_TOKEN"] 691 + for secret in ("AKID", "SAK", "SESS"): 692 + assert secret not in json.dumps(backend_env) 690 693 assert ( 691 694 backup_kwargs["repository"] 692 695 == "s3:https://acct.r2.cloudflarestorage.com/bkt/users/acct/inst" ··· 736 739 secret_access_key="SAK", 737 740 session_token="SESS", 738 741 endpoint="https://acct.r2.cloudflarestorage.com", 742 + expires_at="2026-07-13T12:00:00Z", 739 743 ) 740 744 741 745 def fake_run_restic(args: list[str], **kwargs: Any) -> ResticResult: ··· 751 755 assert result.status == "ok" 752 756 forget_call = next(call for call in calls if call[0][0] == "forget") 753 757 assert captured["scope"] == "maintenance" 754 - assert forget_call[1]["backend_env"]["AWS_SESSION_TOKEN"] == "SESS" 758 + assert forget_call[1]["backend_env"][ 759 + "AWS_CONTAINER_CREDENTIALS_FULL_URI" 760 + ].startswith("http://127.0.0.1:") 761 + 762 + 763 + def test_backup_timeout_is_long_only_until_first_snapshot( 764 + tmp_path: Path, 765 + monkeypatch: pytest.MonkeyPatch, 766 + ) -> None: 767 + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) 768 + config = _valid_backup_config() 769 + _write_config(tmp_path, config) 770 + 771 + assert engine._backup_timeout() == engine.INITIAL_BACKUP_TIMEOUT_SECONDS 772 + 773 + config["backup"]["last_backup"] = { 774 + "status": "error", 775 + "snapshot_id": "partial-snap", 776 + } 777 + _write_config(tmp_path, config) 778 + 779 + assert engine._backup_timeout() == engine.INITIAL_BACKUP_TIMEOUT_SECONDS 780 + 781 + config["backup"]["last_backup"] = {"status": "ok", "snapshot_id": "snap-1"} 782 + _write_config(tmp_path, config) 783 + 784 + assert engine._backup_timeout() == engine.BACKUP_TIMEOUT_SECONDS 755 785 756 786 757 787 def _assert_operated_backup_degrades_on_hosted_credential_error( ··· 918 948 secret_access_key="SAK-SECRET", 919 949 session_token="SESS-SECRET", 920 950 endpoint="https://acct.r2.cloudflarestorage.com", 951 + expires_at="2026-07-13T12:00:00Z", 921 952 ) 922 953 923 954 def fake_run_restic(args: list[str], **kwargs: Any) -> ResticResult:
+60
tests/test_backup_hosted.py
··· 11 11 import urllib.request 12 12 from pathlib import Path 13 13 from typing import Any 14 + from unittest.mock import Mock 14 15 15 16 import pytest 16 17 ··· 25 26 operated_repository, 26 27 save_hosted_binding, 27 28 ) 29 + from solstone.think.backup.hosted_provider import hosted_restic_session 28 30 29 31 30 32 class _FakeResponse: ··· 64 66 secret_access_key="SAK", 65 67 session_token="SESS", 66 68 endpoint="https://acct.r2.cloudflarestorage.com/", 69 + expires_at="2026-07-13T12:00:00Z", 67 70 ) 68 71 69 72 ··· 158 161 "secret_access_key": "SAK", 159 162 "session_token": "SESS", 160 163 "endpoint": "https://acct.r2.cloudflarestorage.com", 164 + "expires_at": "2026-07-13T12:00:00Z", 161 165 } 162 166 ).encode("utf-8") 163 167 ) ··· 174 178 secret_access_key="SAK", 175 179 session_token="SESS", 176 180 endpoint="https://acct.r2.cloudflarestorage.com", 181 + expires_at="2026-07-13T12:00:00Z", 177 182 ) 178 183 179 184 ··· 275 280 secret_access_key="SAK-SECRET", 276 281 session_token="SESS-SECRET", 277 282 endpoint="https://acct.r2.cloudflarestorage.com", 283 + expires_at="2026-07-13T12:00:00Z", 278 284 ) 279 285 rendered = repr(creds) 280 286 for secret in ("AKID-SECRET", "SAK-SECRET", "SESS-SECRET"): ··· 296 302 fetch_hosted_credentials(_binding(broker_token="the-token"), scope="backup") 297 303 298 304 assert "the-token" not in caplog.text 305 + 306 + 307 + def test_hosted_restic_session_serves_initial_then_renewed_credentials( 308 + monkeypatch: pytest.MonkeyPatch, 309 + ) -> None: 310 + initial = _credentials() 311 + renewed = HostedCredentials( 312 + access_key_id="AKID-2", 313 + secret_access_key="SAK-2", 314 + session_token="SESS-2", 315 + endpoint="https://acct.r2.cloudflarestorage.com/", 316 + expires_at="2026-07-13T13:00:00Z", 317 + ) 318 + fetch = Mock(return_value=renewed) 319 + monkeypatch.setattr( 320 + "solstone.think.backup.hosted_provider.fetch_hosted_credentials", 321 + fetch, 322 + ) 323 + 324 + with hosted_restic_session( 325 + _binding(prefix="users/acct/inst/"), 326 + scope="backup", 327 + initial_credentials=initial, 328 + ) as session: 329 + uri = session.backend_env["AWS_CONTAINER_CREDENTIALS_FULL_URI"] 330 + token = session.backend_env["AWS_CONTAINER_AUTHORIZATION_TOKEN"] 331 + assert uri.startswith("http://127.0.0.1:") 332 + assert session.destination.credentials == {} 333 + 334 + with pytest.raises(urllib.error.HTTPError) as exc_info: 335 + urllib.request.urlopen(uri, timeout=1) 336 + assert exc_info.value.code == 401 337 + 338 + first_request = urllib.request.Request( 339 + uri, 340 + headers={"Authorization": token}, 341 + ) 342 + with urllib.request.urlopen(first_request, timeout=1) as response: 343 + first = json.loads(response.read()) 344 + assert first == { 345 + "AccessKeyId": "AKID", 346 + "SecretAccessKey": "SAK", 347 + "Token": "SESS", 348 + "Expiration": "2026-07-13T12:00:00Z", 349 + } 350 + fetch.assert_not_called() 351 + 352 + with urllib.request.urlopen(first_request, timeout=1) as response: 353 + second = json.loads(response.read()) 354 + assert second["AccessKeyId"] == "AKID-2" 355 + assert second["Expiration"] == "2026-07-13T13:00:00Z" 356 + fetch.assert_called_once_with( 357 + _binding(prefix="users/acct/inst/"), scope="backup" 358 + )
+40 -8
tests/test_backup_restore.py
··· 12 12 13 13 from solstone.think.backup import restore 14 14 from solstone.think.backup.destination import Destination 15 + from solstone.think.backup.hosted import HostedBinding, HostedCredentials 15 16 from solstone.think.backup.runner import ResticResult 16 17 17 18 ··· 49 50 "secret_access_key": "SAK-OPERATED", 50 51 "session_token": "SESSION-OPERATED", 51 52 }, 53 + ) 54 + 55 + 56 + def _operated_binding() -> HostedBinding: 57 + return HostedBinding( 58 + broker_endpoint="https://broker.example", 59 + account_id="acct", 60 + instance_id="inst", 61 + bucket="journal-backups", 62 + prefix="users/acct/inst/", 63 + broker_token="BTOKEN-OPERATED", 64 + ) 65 + 66 + 67 + def _operated_credentials() -> HostedCredentials: 68 + return HostedCredentials( 69 + access_key_id="AKID-OPERATED", 70 + secret_access_key="SAK-OPERATED", 71 + session_token="SESSION-OPERATED", 72 + endpoint="https://r2.example", 73 + expires_at="2026-07-13T12:00:00Z", 52 74 ) 53 75 54 76 ··· 479 501 ) 480 502 monkeypatch.setattr(restore, "scan_journal", fake_scan_journal) 481 503 482 - result = restore.restore_journal_operated(destination, entered) 504 + result = restore.restore_journal_operated( 505 + _operated_binding(), 506 + _operated_credentials(), 507 + entered, 508 + ) 483 509 484 510 assert result == restore.RestoreResult( 485 511 status="ok", ··· 497 523 "set_recovery_key_confirmed", 498 524 "scan_journal", 499 525 ] 500 - assert calls[0][1]["backend_env"] == { 501 - "AWS_ACCESS_KEY_ID": "AKID-OPERATED", 502 - "AWS_SECRET_ACCESS_KEY": "SAK-OPERATED", 503 - "AWS_SESSION_TOKEN": "SESSION-OPERATED", 504 - } 526 + assert calls[0][1]["backend_env"]["AWS_CONTAINER_CREDENTIALS_FULL_URI"].startswith( 527 + "http://127.0.0.1:" 528 + ) 505 529 config = _read_config(tmp_path) 506 530 serialized = json.dumps(config) 507 531 assert config["backup"]["mode"] == "operated" ··· 540 564 lambda destination: pytest.fail("must not persist destination"), 541 565 ) 542 566 543 - result = restore.restore_journal_operated(_operated_destination(), "too-short") 567 + result = restore.restore_journal_operated( 568 + _operated_binding(), 569 + _operated_credentials(), 570 + "too-short", 571 + ) 544 572 545 573 assert result.reason_code == "invalid_key" 546 574 assert _read_config(tmp_path) == original_config ··· 571 599 lambda key: pytest.fail("must not persist key"), 572 600 ) 573 601 574 - result = restore.restore_journal_operated(_operated_destination(), "A" * 64) 602 + result = restore.restore_journal_operated( 603 + _operated_binding(), 604 + _operated_credentials(), 605 + "A" * 64, 606 + ) 575 607 576 608 assert result.reason_code == "auth_failed" 577 609 assert _read_config(tmp_path) == original_config
+2
tests/test_backup_teardown.py
··· 261 261 secret_access_key="SAK", 262 262 session_token="SESS", 263 263 endpoint="https://acct.r2.cloudflarestorage.com", 264 + expires_at="2026-07-13T12:00:00Z", 264 265 ) 265 266 266 267 def fail_run_restic(*_args: Any, **_kwargs: Any) -> ResticResult: ··· 370 371 secret_access_key="SAK", 371 372 session_token="SESS", 372 373 endpoint="https://acct.r2.cloudflarestorage.com", 374 + expires_at="2026-07-13T12:00:00Z", 373 375 ) 374 376 ), 375 377 )
+1 -1
tests/test_supervisor.py
··· 1180 1180 "segment": 4500, 1181 1181 "indexer": 7200, 1182 1182 "importer": 3600, 1183 - backup_partition: 25200, 1183 + backup_partition: 49 * 60 * 60, 1184 1184 } 1185 1185 for name, seconds in expected.items(): 1186 1186 assert queue._effective_cap(name) == seconds