+130 -3

solstone/think/brain_cli.py

View file Reviewed

··· 34 34 BrainEvidenceComponent, 35 35 BrainProbeOutcome, 36 36 BrainStateConflictError, 37 37 + BrainStateExpectedFingerprintStaleError, 37 38 BrainStateInspection, 38 39 BrainStateRecord, 40 40 + abandon_brain_prerequisite_renewal, 39 41 abandon_brain_refresh, 42 42 + begin_brain_prerequisite_renewal, 40 43 begin_brain_refresh, 41 44 brain_state_path, 42 45 build_active_brain_fingerprint, 46 46 + finish_brain_prerequisite_renewal, 43 47 finish_brain_refresh, 44 48 inspect_brain_state, 45 49 probe_brain_refresh_lease_held, 50 50 + read_active_brain_fingerprint_sha256, 46 51 runtime_phase_reason, 47 52 ) 48 53 from solstone.think.providers.runtime_health import ( ··· 331 336 return _current_bundled_runtime_fingerprint(config) == expected 332 337 333 338 339 339 + def _expected_active_fingerprint_matches(expected: str) -> bool: 340 340 + try: 341 341 + return read_active_brain_fingerprint_sha256() == expected 342 342 + except Exception: 343 343 + return False 344 344 + 345 345 + 346 346 + def _expected_refresh_fingerprint_matches( 347 347 + args: argparse.Namespace, expected: str 348 348 + ) -> bool: 349 349 + if getattr(args, "expected_active_fingerprint", False): 350 350 + return _expected_active_fingerprint_matches(expected) 351 351 + return _expected_fingerprint_matches(expected) 352 352 + 353 353 + 334 354 def _runtime_diagnostic( 335 355 reason_code: str, 336 356 *, ··· 422 442 try: 423 443 recheck_confidential_attestation() 424 444 except AttestationStaleError: 445 445 + # Compatibility for verifier implementations that report stale directly. 425 446 return _failed_component(now, "attestation_expired"), "attestation_expired" 426 447 except AttestationFailedError: 427 448 return _failed_component(now, "attestation_rejected"), "attestation_rejected" ··· 620 641 return brain_exit_code(refresh_outcome="stale_expected_fingerprint") 621 642 622 643 644 644 + def _refresh_args_for_active_fallback(args: argparse.Namespace) -> argparse.Namespace: 645 645 + return argparse.Namespace( 646 646 + json=args.json, 647 647 + expected_fingerprint=args.expected_fingerprint, 648 648 + expected_active_fingerprint=True, 649 649 + expect_active_fingerprint_absent=False, 650 650 + ) 651 651 + 652 652 + 653 653 + def _run_renew_prerequisites(args: argparse.Namespace) -> int: 654 654 + now = _now() 655 655 + expected = args.expected_fingerprint 656 656 + if expected and not _expected_active_fingerprint_matches(expected): 657 657 + return _render_stale_expected(args) 658 658 + 659 659 + begin = begin_brain_prerequisite_renewal( 660 660 + now, 661 661 + expected_fingerprint_sha256=expected, 662 662 + run_id=uuid.uuid4().hex, 663 663 + ) 664 664 + if begin["status"] == "busy": 665 665 + inspection = inspect_brain_state(now) 666 666 + view = _view_from_inspection(inspection, now) 667 667 + busy = _transient_view( 668 668 + "busy", 669 669 + active_lane=view.active_lane, 670 670 + active_provider=view.active_provider, 671 671 + active_model=view.active_model, 672 672 + fingerprint_sha256=view.fingerprint_sha256, 673 673 + ) 674 674 + render(busy, json_output=args.json) 675 675 + return brain_exit_code(refresh_outcome="busy") 676 676 + if begin["status"] == "unsafe": 677 677 + return _run_refresh(_refresh_args_for_active_fallback(args)) 678 678 + 679 679 + permit = begin["permit"] 680 680 + try: 681 681 + lane_prerequisites, _reason = _spp_prerequisite(now) 682 682 + except Exception: 683 683 + LOG.exception("brain prerequisite renewal probe failed") 684 684 + try: 685 685 + record = abandon_brain_prerequisite_renewal( 686 686 + permit, 687 687 + "probe_internal_error", 688 688 + now, 689 689 + ) 690 690 + except BrainStateConflictError: 691 691 + lost = _transient_view("lost_fence") 692 692 + render(lost, json_output=args.json) 693 693 + return brain_exit_code(refresh_outcome="lost_fence") 694 694 + else: 695 695 + try: 696 696 + record = finish_brain_prerequisite_renewal( 697 697 + permit, 698 698 + lane_prerequisites, 699 699 + now, 700 700 + ) 701 701 + except BrainStateConflictError: 702 702 + lost = _transient_view("lost_fence") 703 703 + render(lost, json_output=args.json) 704 704 + return brain_exit_code(refresh_outcome="lost_fence") 705 705 + 706 706 + inspection = inspect_brain_state(now) 707 707 + view = _view_from_inspection(inspection, now) 708 708 + render(view, json_output=args.json) 709 709 + return brain_exit_code(aggregate_state=record["aggregate_state"]) 710 710 + 711 711 + 623 712 def _run_refresh(args: argparse.Namespace) -> int: 624 713 now = _now() 625 714 expected = args.expected_fingerprint 626 626 - if expected and not _expected_fingerprint_matches(expected): 715 715 + expected_active = getattr(args, "expected_active_fingerprint", False) 716 716 + expected_absent = getattr(args, "expect_active_fingerprint_absent", False) 717 717 + if expected and expected_absent: 718 718 + return _render_stale_expected(args) 719 719 + if expected and not _expected_refresh_fingerprint_matches(args, expected): 627 720 return _render_stale_expected(args) 628 721 629 722 inspection = inspect_brain_state(now) ··· 631 724 if view.reason_code == "configuration_invalid": 632 725 render(view, json_output=args.json) 633 726 return brain_exit_code(aggregate_state=view.aggregate_state) 634 634 - if expected and view.aggregate_state == "ready": 727 727 + if ( 728 728 + expected 729 729 + and not expected_active 730 730 + and not expected_absent 731 731 + and view.aggregate_state == "ready" 732 732 + ): 635 733 render(view, json_output=args.json) 636 734 return brain_exit_code(aggregate_state=view.aggregate_state) 637 735 638 638 - permit = begin_brain_refresh(now, run_id=uuid.uuid4().hex) 736 736 + try: 737 737 + permit = begin_brain_refresh( 738 738 + now, 739 739 + run_id=uuid.uuid4().hex, 740 740 + expected_active_fingerprint_sha256=expected if expected_active else None, 741 741 + expect_active_fingerprint_absent=expected_absent, 742 742 + ) 743 743 + except BrainStateExpectedFingerprintStaleError: 744 744 + return _render_stale_expected(args) 639 745 if permit is None: 640 746 busy = False 641 747 try: ··· 719 825 "--expected-fingerprint", 720 826 help="Only refresh if the bundled runtime fingerprint still matches", 721 827 ) 828 828 + refresh_parser.add_argument( 829 829 + "--expected-active-fingerprint", 830 830 + action="store_true", 831 831 + help="Interpret --expected-fingerprint as the active brain fingerprint", 832 832 + ) 833 833 + refresh_parser.add_argument( 834 834 + "--expect-active-fingerprint-absent", 835 835 + action="store_true", 836 836 + help="Only refresh if no active brain fingerprint exists yet", 837 837 + ) 838 838 + renew_parser = subparsers.add_parser( 839 839 + "renew-prerequisites", 840 840 + help=argparse.SUPPRESS, 841 841 + ) 842 842 + renew_parser.add_argument("--json", action="store_true", help=argparse.SUPPRESS) 843 843 + renew_parser.add_argument( 844 844 + "--expected-fingerprint", 845 845 + help=argparse.SUPPRESS, 846 846 + ) 722 847 return parser 723 848 724 849 ··· 727 852 return _run_status(args) 728 853 if args.subcommand == "refresh": 729 854 return _run_refresh(args) 855 855 + if args.subcommand == "renew-prerequisites": 856 856 + return _run_renew_prerequisites(args) 730 857 parser.print_help() 731 858 return 2 732 859

+575 -3

solstone/think/cortex.py

View file Reviewed

··· 26 26 import sys 27 27 import threading 28 28 import time 29 29 + import uuid 30 30 + from collections.abc import Callable 29 31 from datetime import datetime, timezone 30 32 from pathlib import Path 31 33 from typing import Any, Dict, Optional 32 34 33 35 from solstone.think.callosum import CallosumConnection 34 36 from solstone.think.models import calc_agent_cost 35 35 - from solstone.think.providers.brain_state import inspect_brain_state 37 37 + from solstone.think.providers.brain_state import ( 38 38 + inspect_brain_state, 39 39 + read_active_brain_fingerprint_sha256, 40 40 + ) 36 41 from solstone.think.runner import _atomic_symlink 42 42 + from solstone.think.services.spp_attest.cadence import TPM_HEARTBEAT_INTERVAL 37 43 from solstone.think.talent import get_output_path 38 44 from solstone.think.talents import TALENT_EXECUTION_MODULE 39 45 from solstone.think.utils import get_journal, get_rev, now_ms ··· 100 106 return 101 107 102 108 109 109 + SPP_RENEWAL_ATTEMPT_BOUND_S = 120.0 110 110 + SPP_REFRESH_OBSERVATION_BOUND_S = 300.0 111 111 + SPP_RENEWAL_RETRY_DELAYS_S = (5.0, 10.0, 20.0, 40.0, 60.0) 112 112 + SPP_RENEWAL_PROACTIVE_MARGIN_S = ( 113 113 + SPP_RENEWAL_ATTEMPT_BOUND_S + SPP_RENEWAL_RETRY_DELAYS_S[0] 114 114 + ) 115 115 + SPP_RENEWAL_ACK_TIMEOUT_S = 15.0 116 116 + SPP_RENEWAL_MAX_WAIT_S = 60.0 117 117 + assert SPP_RENEWAL_PROACTIVE_MARGIN_S < TPM_HEARTBEAT_INTERVAL.total_seconds() / 2 118 118 + 119 119 + 120 120 + class SppRenewalController: 121 121 + """Unattended SPP prerequisite renewal controller for Cortex.""" 122 122 + 123 123 + def __init__( 124 124 + self, 125 125 + *, 126 126 + callosum: CallosumConnection, 127 127 + stop_event: threading.Event, 128 128 + logger: logging.Logger, 129 129 + clock: Callable[[], datetime], 130 130 + wait: Callable[[float], bool], 131 131 + journal_path: Path, 132 132 + ) -> None: 133 133 + self.callosum = callosum 134 134 + self.stop_event = stop_event 135 135 + self.logger = logger 136 136 + self.clock = clock 137 137 + self.wait = wait 138 138 + self.journal_path = journal_path 139 139 + self._pending_ref: str | None = None 140 140 + self._pending_action: str | None = None 141 141 + self._pending_fingerprint: str | None = None 142 142 + self._pending_expect_fingerprint_absent = False 143 143 + self._pending_observed_at: datetime | None = None 144 144 + self._pending_expires_at: datetime | None = None 145 145 + self._ack_deadline: datetime | None = None 146 146 + self._running_ref: str | None = None 147 147 + self._running_action: str | None = None 148 148 + self._running_fingerprint: str | None = None 149 149 + self._running_expect_fingerprint_absent = False 150 150 + self._running_observed_at: datetime | None = None 151 151 + self._running_expires_at: datetime | None = None 152 152 + self._running_deadline: datetime | None = None 153 153 + self._successor_after_ref: str | None = None 154 154 + self._successor_deadline: datetime | None = None 155 155 + self._retry_index = 0 156 156 + self._retry_after: datetime | None = None 157 157 + self._last_mode: str | None = None 158 158 + 159 159 + def run(self) -> None: 160 160 + while not self.stop_event.is_set(): 161 161 + try: 162 162 + delay = self.step() 163 163 + except Exception as exc: 164 164 + now = self._now() 165 165 + self._handle_step_exception(exc, now) 166 166 + delay = self._seconds_until( 167 167 + self._retry_after, now, default=SPP_RENEWAL_RETRY_DELAYS_S[0] 168 168 + ) 169 169 + self.wait(max(0.0, min(delay, SPP_RENEWAL_MAX_WAIT_S))) 170 170 + 171 171 + def step(self) -> float: 172 172 + now = self._now() 173 173 + try: 174 174 + return self._step(now) 175 175 + except Exception as exc: 176 176 + self._handle_step_exception(exc, now) 177 177 + return self._seconds_until( 178 178 + self._retry_after, now, default=SPP_RENEWAL_RETRY_DELAYS_S[0] 179 179 + ) 180 180 + 181 181 + def _step(self, now: datetime) -> float: 182 182 + if self._spp_disabled(now): 183 183 + self._clear_demand() 184 184 + if self._last_mode != "disabled": 185 185 + self._log("disabled", reason="non_spp_lane") 186 186 + self._last_mode = "disabled" 187 187 + return 30.0 188 188 + if self._pending_ref is not None: 189 189 + if self._ack_deadline is not None and now >= self._ack_deadline: 190 190 + self._log("failed", reason="start_ack_timeout", ref=self._pending_ref) 191 191 + self._clear_pending() 192 192 + self._schedule_retry(now) 193 193 + return self._seconds_until(self._ack_deadline, now, default=5.0) 194 194 + if self._running_ref is not None: 195 195 + if self._running_deadline is not None and now >= self._running_deadline: 196 196 + self._log( 197 197 + "stale", reason="running_observation_timeout", ref=self._running_ref 198 198 + ) 199 199 + self._clear_running() 200 200 + self._schedule_retry(now) 201 201 + return self._seconds_until( 202 202 + self._retry_after, now, default=SPP_RENEWAL_RETRY_DELAYS_S[0] 203 203 + ) 204 204 + return self._seconds_until(self._running_deadline, now, default=5.0) 205 205 + if self._successor_after_ref is not None: 206 206 + if self._successor_deadline is not None and now < self._successor_deadline: 207 207 + return self._seconds_until( 208 208 + self._successor_deadline, 209 209 + now, 210 210 + default=5.0, 211 211 + ) 212 212 + self._log( 213 213 + "stale", 214 214 + reason="successor_observation_timeout", 215 215 + active_ref=self._successor_after_ref, 216 216 + ) 217 217 + self._clear_successor() 218 218 + if self._retry_after is not None: 219 219 + if now < self._retry_after: 220 220 + return self._seconds_until(self._retry_after, now, default=5.0) 221 221 + self._retry_after = None 222 222 + 223 223 + plan = self._plan(now) 224 224 + if plan["action"] == "disabled": 225 225 + self._clear_demand() 226 226 + if self._last_mode != "disabled": 227 227 + self._log("disabled", reason=plan.get("reason")) 228 228 + self._last_mode = "disabled" 229 229 + return 30.0 230 230 + self._last_mode = plan["action"] 231 231 + if plan["action"] == "checking": 232 232 + return 5.0 233 233 + if plan["action"] == "wait": 234 234 + return float(plan["delay"]) 235 235 + if plan["action"] in {"renew", "refresh"}: 236 236 + if self._send_request(plan): 237 237 + return SPP_RENEWAL_ACK_TIMEOUT_S 238 238 + return self._seconds_until( 239 239 + self._retry_after, 240 240 + now, 241 241 + default=SPP_RENEWAL_RETRY_DELAYS_S[0], 242 242 + ) 243 243 + return 30.0 244 244 + 245 245 + def _spp_disabled(self, now: datetime) -> bool: 246 246 + inspection = inspect_brain_state(now, journal_path=self.journal_path) 247 247 + return inspection["projection"]["active_lane"] != "spp" 248 248 + 249 249 + def handle_supervisor_message(self, message: dict[str, Any]) -> None: 250 250 + if message.get("tract") != "supervisor": 251 251 + return 252 252 + event = message.get("event") 253 253 + ref = message.get("ref") 254 254 + now = self._now() 255 255 + if event == "started" and ref == self._pending_ref: 256 256 + self._running_ref = self._pending_ref 257 257 + self._running_action = self._pending_action 258 258 + self._running_fingerprint = self._pending_fingerprint 259 259 + self._running_expect_fingerprint_absent = ( 260 260 + self._pending_expect_fingerprint_absent 261 261 + ) 262 262 + self._running_observed_at = self._pending_observed_at 263 263 + self._running_expires_at = self._pending_expires_at 264 264 + self._running_deadline = datetime.fromtimestamp( 265 265 + now.timestamp() + self._observation_bound(self._pending_action), 266 266 + tz=timezone.utc, 267 267 + ) 268 268 + self._log("in_flight", ref=self._running_ref, action=self._running_action) 269 269 + self._clear_pending(keep_retry=True) 270 270 + return 271 271 + if event == "skipped" and ref == self._pending_ref: 272 272 + active_ref = message.get("active_ref") 273 273 + self._log( 274 274 + "in_flight", 275 275 + reason=str(message.get("reason") or "skipped"), 276 276 + ref=ref, 277 277 + active_ref=str(active_ref) if active_ref else None, 278 278 + ) 279 279 + self._successor_after_ref = str(active_ref) if active_ref else None 280 280 + self._successor_deadline = ( 281 281 + datetime.fromtimestamp( 282 282 + now.timestamp() + SPP_REFRESH_OBSERVATION_BOUND_S, 283 283 + tz=timezone.utc, 284 284 + ) 285 285 + if self._successor_after_ref is not None 286 286 + else None 287 287 + ) 288 288 + self._clear_pending(keep_retry=True) 289 289 + if self._successor_after_ref is None: 290 290 + self._schedule_retry(now) 291 291 + return 292 292 + if event == "stopped": 293 293 + if ref == self._running_ref: 294 294 + self._verify_running_result(self._exit_code(message), now) 295 295 + return 296 296 + if ref == self._successor_after_ref: 297 297 + self._clear_successor() 298 298 + 299 299 + def _plan(self, now: datetime) -> dict[str, Any]: 300 300 + inspection = inspect_brain_state(now, journal_path=self.journal_path) 301 301 + projection = inspection["projection"] 302 302 + if projection["active_lane"] != "spp": 303 303 + return {"action": "disabled", "reason": "non_spp_lane"} 304 304 + if projection["aggregate_state"] == "checking": 305 305 + return {"action": "checking"} 306 306 + 307 307 + fingerprint = read_active_brain_fingerprint_sha256( 308 308 + journal_path=self.journal_path 309 309 + ) 310 310 + if fingerprint is None: 311 311 + return { 312 312 + "action": "refresh", 313 313 + "fingerprint": None, 314 314 + "expect_fingerprint_absent": True, 315 315 + } 316 316 + record = inspection["record"] 317 317 + component = None 318 318 + if record is not None: 319 319 + component = record["evidence"].get("lane_prerequisites") 320 320 + observed_at = None 321 321 + expires_at = None 322 322 + if isinstance(component, dict): 323 323 + observed_at = self._parse_time(component.get("observed_at")) 324 324 + expires_at = self._parse_time(component.get("expires_at")) 325 325 + if ( 326 326 + projection["aggregate_state"] != "ready" 327 327 + or not isinstance(record, dict) 328 328 + or record.get("fingerprint_sha256") != fingerprint 329 329 + or not isinstance(component, dict) 330 330 + or component.get("status") != "ok" 331 331 + ): 332 332 + return { 333 333 + "action": "refresh", 334 334 + "fingerprint": fingerprint, 335 335 + "observed_at": observed_at, 336 336 + "expires_at": expires_at, 337 337 + } 338 338 + 339 339 + if observed_at is None or expires_at is None: 340 340 + return { 341 341 + "action": "refresh", 342 342 + "fingerprint": fingerprint, 343 343 + "observed_at": observed_at, 344 344 + "expires_at": expires_at, 345 345 + } 346 346 + if now >= expires_at: 347 347 + return { 348 348 + "action": "refresh", 349 349 + "fingerprint": fingerprint, 350 350 + "observed_at": observed_at, 351 351 + "expires_at": expires_at, 352 352 + } 353 353 + renew_at = expires_at.timestamp() - SPP_RENEWAL_PROACTIVE_MARGIN_S 354 354 + delay = renew_at - now.timestamp() 355 355 + if delay > 0: 356 356 + self._log("scheduled", delay_s=round(delay, 3)) 357 357 + return {"action": "wait", "delay": delay} 358 358 + return { 359 359 + "action": "renew", 360 360 + "fingerprint": fingerprint, 361 361 + "observed_at": observed_at, 362 362 + "expires_at": expires_at, 363 363 + } 364 364 + 365 365 + def _send_request(self, plan: dict[str, Any]) -> bool: 366 366 + action = str(plan["action"]) 367 367 + fingerprint = plan.get("fingerprint") 368 368 + ref = f"spp-renewal-{uuid.uuid4().hex}" 369 369 + if action == "renew": 370 370 + cmd = [ 371 371 + "journal", 372 372 + "brain", 373 373 + "renew-prerequisites", 374 374 + "--json", 375 375 + "--expected-fingerprint", 376 376 + str(fingerprint), 377 377 + ] 378 378 + else: 379 379 + expect_absent = bool(plan.get("expect_fingerprint_absent")) 380 380 + if fingerprint is None and not expect_absent: 381 381 + self._log("failed", reason="fingerprint_unavailable", action=action) 382 382 + self._schedule_retry(self._now()) 383 383 + return False 384 384 + cmd = ["journal", "brain", "refresh", "--json"] 385 385 + if expect_absent: 386 386 + cmd.append("--expect-active-fingerprint-absent") 387 387 + else: 388 388 + cmd.extend( 389 389 + [ 390 390 + "--expected-fingerprint", 391 391 + str(fingerprint), 392 392 + "--expected-active-fingerprint", 393 393 + ] 394 394 + ) 395 395 + now = self._now() 396 396 + try: 397 397 + self.callosum.emit( 398 398 + "supervisor", 399 399 + "request", 400 400 + cmd=cmd, 401 401 + ref=ref, 402 402 + scheduler_name="spp-renewal", 403 403 + ) 404 404 + except Exception as exc: 405 405 + self._log("failed", reason=type(exc).__name__, action=action) 406 406 + self._schedule_retry(now) 407 407 + return False 408 408 + self._pending_ref = ref 409 409 + self._pending_action = action 410 410 + self._pending_fingerprint = str(fingerprint) if fingerprint else None 411 411 + self._pending_expect_fingerprint_absent = bool( 412 412 + plan.get("expect_fingerprint_absent") 413 413 + ) 414 414 + self._pending_observed_at = plan.get("observed_at") 415 415 + self._pending_expires_at = plan.get("expires_at") 416 416 + self._ack_deadline = datetime.fromtimestamp( 417 417 + now.timestamp() + SPP_RENEWAL_ACK_TIMEOUT_S, tz=timezone.utc 418 418 + ) 419 419 + self._log("in_flight", ref=ref, action=action) 420 420 + return True 421 421 + 422 422 + def _verify_running_result(self, exit_code: int, now: datetime) -> None: 423 423 + action = self._running_action 424 424 + fingerprint = self._running_fingerprint 425 425 + expect_absent = self._running_expect_fingerprint_absent 426 426 + previous_observed = self._running_observed_at 427 427 + previous_expires = self._running_expires_at 428 428 + ref = self._running_ref 429 429 + self._clear_running() 430 430 + if action == "renew" and self._persisted_spp_prerequisite_verified( 431 431 + fingerprint, 432 432 + previous_observed, 433 433 + previous_expires, 434 434 + now, 435 435 + require_ready=False, 436 436 + ): 437 437 + self._retry_index = 0 438 438 + self._retry_after = None 439 439 + self._log("verified", ref=ref) 440 440 + return 441 441 + if ( 442 442 + action == "refresh" 443 443 + and exit_code == 0 444 444 + and ( 445 445 + self._persisted_spp_prerequisite_verified( 446 446 + fingerprint, 447 447 + previous_observed, 448 448 + previous_expires, 449 449 + now, 450 450 + require_ready=True, 451 451 + ) 452 452 + if not expect_absent 453 453 + else self._persisted_spp_absence_bootstrap_verified( 454 454 + previous_observed, 455 455 + previous_expires, 456 456 + now, 457 457 + ) 458 458 + ) 459 459 + ): 460 460 + self._retry_index = 0 461 461 + self._retry_after = None 462 462 + self._log("verified", ref=ref, action="refresh") 463 463 + return 464 464 + self._log("failed", ref=ref, action=action, exit_code=exit_code) 465 465 + self._schedule_retry(now) 466 466 + 467 467 + def _persisted_spp_prerequisite_verified( 468 468 + self, 469 469 + fingerprint: str | None, 470 470 + previous_observed: datetime | None, 471 471 + previous_expires: datetime | None, 472 472 + now: datetime, 473 473 + *, 474 474 + require_ready: bool, 475 475 + ) -> bool: 476 476 + if fingerprint is None: 477 477 + return False 478 478 + try: 479 479 + active_fingerprint = read_active_brain_fingerprint_sha256( 480 480 + journal_path=self.journal_path 481 481 + ) 482 482 + inspection = inspect_brain_state(now, journal_path=self.journal_path) 483 483 + except Exception: 484 484 + return False 485 485 + if active_fingerprint != fingerprint: 486 486 + return False 487 487 + projection = inspection["projection"] 488 488 + if projection["active_lane"] != "spp": 489 489 + return False 490 490 + if require_ready and projection["aggregate_state"] != "ready": 491 491 + return False 492 492 + record = inspection["record"] 493 493 + if record is None or record["active_lane"] != "spp": 494 494 + return False 495 495 + if record["fingerprint_sha256"] != fingerprint: 496 496 + return False 497 497 + component = record["evidence"].get("lane_prerequisites") 498 498 + if not isinstance(component, dict) or component.get("status") != "ok": 499 499 + return False 500 500 + observed_at = self._parse_time(component.get("observed_at")) 501 501 + expires_at = self._parse_time(component.get("expires_at")) 502 502 + return ( 503 503 + observed_at is not None 504 504 + and expires_at is not None 505 505 + and (previous_observed is None or observed_at > previous_observed) 506 506 + and (previous_expires is None or expires_at > previous_expires) 507 507 + ) 508 508 + 509 509 + def _persisted_spp_absence_bootstrap_verified( 510 510 + self, 511 511 + previous_observed: datetime | None, 512 512 + previous_expires: datetime | None, 513 513 + now: datetime, 514 514 + ) -> bool: 515 515 + try: 516 516 + active_fingerprint = read_active_brain_fingerprint_sha256( 517 517 + journal_path=self.journal_path 518 518 + ) 519 519 + inspection = inspect_brain_state(now, journal_path=self.journal_path) 520 520 + except Exception: 521 521 + return False 522 522 + if active_fingerprint is None: 523 523 + return False 524 524 + projection = inspection["projection"] 525 525 + if ( 526 526 + projection["active_lane"] != "spp" 527 527 + or projection["aggregate_state"] != "ready" 528 528 + ): 529 529 + return False 530 530 + record = inspection["record"] 531 531 + if record is None or record["active_lane"] != "spp": 532 532 + return False 533 533 + if record["fingerprint_sha256"] != active_fingerprint: 534 534 + return False 535 535 + component = record["evidence"].get("lane_prerequisites") 536 536 + if not isinstance(component, dict) or component.get("status") != "ok": 537 537 + return False 538 538 + observed_at = self._parse_time(component.get("observed_at")) 539 539 + expires_at = self._parse_time(component.get("expires_at")) 540 540 + return ( 541 541 + observed_at is not None 542 542 + and expires_at is not None 543 543 + and (previous_observed is None or observed_at > previous_observed) 544 544 + and (previous_expires is None or expires_at > previous_expires) 545 545 + ) 546 546 + 547 547 + def _schedule_retry(self, now: datetime) -> None: 548 548 + delay = SPP_RENEWAL_RETRY_DELAYS_S[ 549 549 + min(self._retry_index, len(SPP_RENEWAL_RETRY_DELAYS_S) - 1) 550 550 + ] 551 551 + self._retry_index += 1 552 552 + self._retry_after = datetime.fromtimestamp( 553 553 + now.timestamp() + delay, tz=timezone.utc 554 554 + ) 555 555 + self._log("retrying", delay_s=delay) 556 556 + 557 557 + def _handle_step_exception(self, exc: Exception, now: datetime) -> None: 558 558 + self._log("failed", reason=type(exc).__name__) 559 559 + self._clear_pending(keep_retry=True) 560 560 + self._clear_running() 561 561 + self._clear_successor() 562 562 + self._schedule_retry(now) 563 563 + 564 564 + def _observation_bound(self, action: str | None) -> float: 565 565 + if action == "refresh": 566 566 + return SPP_REFRESH_OBSERVATION_BOUND_S 567 567 + return SPP_RENEWAL_ATTEMPT_BOUND_S 568 568 + 569 569 + def _exit_code(self, message: dict[str, Any]) -> int: 570 570 + value = message.get("exit_code") 571 571 + if value is None: 572 572 + return -1 573 573 + try: 574 574 + return int(value) 575 575 + except (TypeError, ValueError): 576 576 + return -1 577 577 + 578 578 + def _clear_pending(self, *, keep_retry: bool = False) -> None: 579 579 + self._pending_ref = None 580 580 + self._pending_action = None 581 581 + self._pending_fingerprint = None 582 582 + self._pending_expect_fingerprint_absent = False 583 583 + self._pending_observed_at = None 584 584 + self._pending_expires_at = None 585 585 + self._ack_deadline = None 586 586 + if not keep_retry: 587 587 + self._retry_after = None 588 588 + 589 589 + def _clear_running(self) -> None: 590 590 + self._running_ref = None 591 591 + self._running_action = None 592 592 + self._running_fingerprint = None 593 593 + self._running_expect_fingerprint_absent = False 594 594 + self._running_observed_at = None 595 595 + self._running_expires_at = None 596 596 + self._running_deadline = None 597 597 + 598 598 + def _clear_successor(self) -> None: 599 599 + self._successor_after_ref = None 600 600 + self._successor_deadline = None 601 601 + 602 602 + def _clear_demand(self) -> None: 603 603 + self._clear_pending() 604 604 + self._clear_running() 605 605 + self._clear_successor() 606 606 + self._retry_index = 0 607 607 + self._retry_after = None 608 608 + 609 609 + def _now(self) -> datetime: 610 610 + return self.clock().astimezone(timezone.utc) 611 611 + 612 612 + def _seconds_until( 613 613 + self, deadline: datetime | None, now: datetime, *, default: float 614 614 + ) -> float: 615 615 + if deadline is None: 616 616 + return default 617 617 + return max(0.0, deadline.timestamp() - now.timestamp()) 618 618 + 619 619 + def _parse_time(self, value: object) -> datetime | None: 620 620 + if not isinstance(value, str): 621 621 + return None 622 622 + try: 623 623 + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone( 624 624 + timezone.utc 625 625 + ) 626 626 + except ValueError: 627 627 + return None 628 628 + 629 629 + def _log(self, event: str, **fields: object) -> None: 630 630 + safe = " ".join( 631 631 + f"{key}={value}" 632 632 + for key, value in sorted(fields.items()) 633 633 + if value is not None 634 634 + ) 635 635 + suffix = f" {safe}" if safe else "" 636 636 + self.logger.info("event=spp_renewal_%s%s", event, suffix) 637 637 + 638 638 + 103 639 class CortexService: 104 640 """Callosum-based talent process manager.""" 105 641 106 106 - def __init__(self, journal_path: Optional[str] = None): 642 642 + def __init__( 643 643 + self, 644 644 + journal_path: Optional[str] = None, 645 645 + *, 646 646 + clock: Callable[[], datetime] | None = None, 647 647 + wait: Callable[[float], bool] | None = None, 648 648 + ): 107 649 self.journal_path = Path(journal_path or get_journal()) 108 650 self.talents_dir = self.journal_path / "talents" 109 651 self.talents_dir.mkdir(parents=True, exist_ok=True) ··· 117 659 self.spawn_queue: queue.Queue = queue.Queue() 118 660 self._pending_spawns: int = 0 119 661 self._spawn_worker: threading.Thread | None = None 662 662 + self._clock = clock or (lambda: datetime.now(timezone.utc)) 663 663 + self._wait = wait or self.stop_event.wait 664 664 + self._spp_renewal_controller: SppRenewalController | None = None 665 665 + self._spp_renewal_worker: threading.Thread | None = None 120 666 121 667 # Callosum connection for receiving requests and broadcasting events 122 668 self.callosum = CallosumConnection(defaults={"rev": get_rev()}) ··· 264 810 daemon=True, 265 811 ) 266 812 self._spawn_worker.start() 813 813 + self._start_spp_renewal_controller() 267 814 268 815 self.logger.info("Cortex service started, listening for talent requests") 269 816 ··· 285 832 286 833 def _should_request_brain_refresh(self) -> bool: 287 834 try: 288 288 - inspection = inspect_brain_state(datetime.now(timezone.utc)) 835 835 + inspection = inspect_brain_state( 836 836 + self._clock(), journal_path=self.journal_path 837 837 + ) 289 838 except Exception: 290 839 return True 291 840 projection = inspection["projection"] 841 841 + if projection["active_lane"] == "spp": 842 842 + return False 292 843 if projection["aggregate_state"] in {"checking", "ready"}: 293 844 return False 294 845 return not projection["runtime_transition_in_progress"] 295 846 847 847 + def _start_spp_renewal_controller(self) -> None: 848 848 + self._spp_renewal_controller = SppRenewalController( 849 849 + callosum=self.callosum, 850 850 + stop_event=self.stop_event, 851 851 + logger=self.logger, 852 852 + clock=self._clock, 853 853 + wait=self._wait, 854 854 + journal_path=self.journal_path, 855 855 + ) 856 856 + self._spp_renewal_worker = threading.Thread( 857 857 + target=self._spp_renewal_controller.run, 858 858 + name="cortex-spp-renewal", 859 859 + daemon=True, 860 860 + ) 861 861 + self._spp_renewal_worker.start() 862 862 + 296 863 def _handle_callosum_message(self, message: Dict[str, Any]) -> None: 297 864 """Handle incoming Callosum messages (callback).""" 865 865 + if self._spp_renewal_controller is not None: 866 866 + self._spp_renewal_controller.handle_supervisor_message(message) 298 867 # Filter for cortex tract and request event 299 868 if message.get("tract") != "cortex" or message.get("event") != "request": 300 869 return ··· 906 1475 907 1476 if self.callosum: 908 1477 self.callosum.stop() 1478 1478 + 1479 1479 + if self._spp_renewal_worker is not None: 1480 1480 + self._spp_renewal_worker.join(timeout=2.0) 909 1481 910 1482 # Let the spawn worker finish its current item and exit (~2s bound; the 911 1483 # worker's 0.5s get-timeout guarantees it observes stop_event promptly).

+3 -2

solstone/think/models.py

View file Reviewed

··· 292 292 293 293 294 294 # Attestation failures are non-retryable. AttestationFailedError is raised by 295 295 - # solstone.think.services.spp_attest.composite.verify_composite; AttestationStaleError 296 296 - # is reserved for the follow-on verifier when a session's cadence windows lapse. 295 295 + # solstone.think.services.spp_attest.composite.verify_composite. AttestationStaleError 296 296 + # remains part of the verifier contract; the process-local egress transport rotates 297 297 + # stale live sessions inline before forwarding bytes. 297 298 _CONFIDENTIAL_ATTESTATION_VERIFIER: Callable[[dict[str, Any]], None] | None = None 298 299 299 300

+353 -11

solstone/think/providers/brain_state.py

View file Reviewed

··· 523 523 error: str | None 524 524 525 525 526 526 + BrainPrerequisiteRenewalStatus = Literal["started", "busy", "unsafe"] 527 527 + 528 528 + 529 529 + class BrainPrerequisiteRenewalBeginResult(TypedDict): 530 530 + status: BrainPrerequisiteRenewalStatus 531 531 + permit: NotRequired["BrainRefreshPermit"] 532 532 + reason: NotRequired[str] 533 533 + 534 534 + 526 535 class BrainStateValidationError(ValueError): 527 536 """Raised when a persisted brain state record violates the closed schema.""" 528 537 ··· 534 543 535 544 class BrainStateConflictError(RuntimeError): 536 545 """Raised when a stale refresh permit attempts to finalize.""" 546 546 + 547 547 + 548 548 + class BrainStateExpectedFingerprintStaleError(BrainStateConflictError): 549 549 + """Raised when a fenced refresh observes a different active fingerprint state.""" 537 550 538 551 539 552 @dataclass ··· 1885 1898 now: datetime, 1886 1899 *, 1887 1900 run_id: str | None = None, 1901 1901 + expected_active_fingerprint_sha256: str | None = None, 1902 1902 + expect_active_fingerprint_absent: bool = False, 1888 1903 journal_path: str | Path | None = None, 1889 1904 ) -> BrainRefreshPermit | None: 1890 1905 now = _utc(now) 1906 1906 + if expected_active_fingerprint_sha256 is not None: 1907 1907 + try: 1908 1908 + _validate_hex( 1909 1909 + expected_active_fingerprint_sha256, 1910 1910 + "expected_active_fingerprint_sha256", 1911 1911 + ) 1912 1912 + except BrainStateValidationError as exc: 1913 1913 + raise BrainStateExpectedFingerprintStaleError(str(exc)) from exc 1914 1914 + if ( 1915 1915 + expected_active_fingerprint_sha256 is not None 1916 1916 + and expect_active_fingerprint_absent 1917 1917 + ): 1918 1918 + raise ValueError( 1919 1919 + "expected active fingerprint and expected absence are mutually exclusive" 1920 1920 + ) 1921 1921 + expected_contract = ( 1922 1922 + expected_active_fingerprint_sha256 is not None 1923 1923 + or expect_active_fingerprint_absent 1924 1924 + ) 1891 1925 try: 1892 1926 config = read_journal_config(journal_path) 1893 1927 except (CorruptConfigError, OSError): 1894 1928 return None 1895 1929 lane, provider, model = _derive_lane(config) 1896 1930 path = brain_state_path(journal_path=journal_path) 1897 1897 - if lane is None: 1931 1931 + if lane is None and not expected_contract: 1898 1932 return None 1899 1899 - if lane == "none": 1933 1933 + if lane == "none" and not expected_contract: 1900 1934 _begin_nonrefresh_record( 1901 1935 now, 1902 1936 path=path, ··· 1908 1942 if lease is None: 1909 1943 return None 1910 1944 try: 1911 1911 - try: 1912 1912 - key = _load_or_generate_fingerprint_key(journal_path=journal_path) 1913 1913 - except Exception: 1914 1914 - lease.release() 1915 1915 - return None 1916 1916 - fingerprint = build_active_brain_fingerprint(config, hmac_key=key) 1917 1917 - if fingerprint["active_lane"] is None or not fingerprint["ok"]: 1918 1918 - lease.release() 1919 1919 - return None 1920 1945 run_id = run_id or uuid.uuid4().hex 1921 1946 expires_at = now + CHECKING_TTL 1922 1947 with hold_lock(path, mode=BRAIN_FILE_MODE): 1948 1948 + try: 1949 1949 + config = read_journal_config(journal_path) 1950 1950 + except (CorruptConfigError, OSError): 1951 1951 + lease.release() 1952 1952 + return None 1953 1953 + lane, provider, model = _derive_lane(config) 1954 1954 + if lane is None: 1955 1955 + if expected_contract: 1956 1956 + raise BrainStateExpectedFingerprintStaleError( 1957 1957 + "active brain fingerprint is unavailable" 1958 1958 + ) 1959 1959 + lease.release() 1960 1960 + return None 1961 1961 + if lane == "none": 1962 1962 + if expected_contract: 1963 1963 + raise BrainStateExpectedFingerprintStaleError( 1964 1964 + "active brain fingerprint is unavailable" 1965 1965 + ) 1966 1966 + _begin_nonrefresh_record( 1967 1967 + now, 1968 1968 + path=path, 1969 1969 + active_provider=provider, 1970 1970 + active_model=model, 1971 1971 + ) 1972 1972 + lease.release() 1973 1973 + return None 1974 1974 + try: 1975 1975 + key = _load_existing_fingerprint_key(journal_path=journal_path) 1976 1976 + if expect_active_fingerprint_absent: 1977 1977 + if key is not None: 1978 1978 + raise BrainStateExpectedFingerprintStaleError( 1979 1979 + "active brain fingerprint is present" 1980 1980 + ) 1981 1981 + key = _load_or_generate_fingerprint_key(journal_path=journal_path) 1982 1982 + elif expected_active_fingerprint_sha256 is not None: 1983 1983 + if key is None: 1984 1984 + raise BrainStateExpectedFingerprintStaleError( 1985 1985 + "active brain fingerprint is absent" 1986 1986 + ) 1987 1987 + else: 1988 1988 + key = _load_or_generate_fingerprint_key(journal_path=journal_path) 1989 1989 + except BrainStateExpectedFingerprintStaleError: 1990 1990 + raise 1991 1991 + except Exception: 1992 1992 + lease.release() 1993 1993 + return None 1994 1994 + assert key is not None 1995 1995 + fingerprint = build_active_brain_fingerprint(config, hmac_key=key) 1996 1996 + if ( 1997 1997 + expected_active_fingerprint_sha256 is not None 1998 1998 + and fingerprint["fingerprint_sha256"] 1999 1999 + != expected_active_fingerprint_sha256 2000 2000 + ): 2001 2001 + raise BrainStateExpectedFingerprintStaleError( 2002 2002 + "active brain fingerprint changed" 2003 2003 + ) 2004 2004 + if fingerprint["active_lane"] is None or not fingerprint["ok"]: 2005 2005 + if expected_contract: 2006 2006 + raise BrainStateExpectedFingerprintStaleError( 2007 2007 + "active brain fingerprint is unavailable" 2008 2008 + ) 2009 2009 + lease.release() 2010 2010 + return None 1923 2011 current = _read_record_unlocked(path) 1924 2012 revision = _next_revision(current) 1925 2013 marker_seen = _runtime_failure_marker_id(current) ··· 1999 2087 return fingerprint["fingerprint_sha256"] 2000 2088 2001 2089 2090 2090 + def _component_ok_unexpired( 2091 2091 + component: BrainEvidenceComponent | None, 2092 2092 + component_name: str, 2093 2093 + now: datetime, 2094 2094 + ) -> bool: 2095 2095 + if component is None or component["status"] != "ok": 2096 2096 + return False 2097 2097 + try: 2098 2098 + expires_at = _parse_timestamp( 2099 2099 + component.get("expires_at"), f"evidence.{component_name}.expires_at" 2100 2100 + ) 2101 2101 + except BrainStateValidationError: 2102 2102 + return False 2103 2103 + return now < expires_at 2104 2104 + 2105 2105 + 2106 2106 + def _safe_prerequisite_renewal_evidence( 2107 2107 + current: BrainStateRecord, 2108 2108 + now: datetime, 2109 2109 + ) -> BrainEvidenceRecord | None: 2110 2110 + if current["active_lane"] != "spp": 2111 2111 + return None 2112 2112 + if _record_timestamp_invalid(current, now): 2113 2113 + return None 2114 2114 + evidence = current["evidence"] 2115 2115 + preserved: dict[str, BrainEvidenceComponent | None] = dict(evidence) 2116 2116 + for component_name in ("configuration", "generate", "cogitate"): 2117 2117 + if not _component_ok_unexpired(preserved[component_name], component_name, now): 2118 2118 + return None 2119 2119 + return cast(BrainEvidenceRecord, preserved) 2120 2120 + 2121 2121 + 2122 2122 + def _prerequisite_renewal_begin_result( 2123 2123 + status: BrainPrerequisiteRenewalStatus, 2124 2124 + *, 2125 2125 + permit: BrainRefreshPermit | None = None, 2126 2126 + reason: str | None = None, 2127 2127 + ) -> BrainPrerequisiteRenewalBeginResult: 2128 2128 + result: BrainPrerequisiteRenewalBeginResult = {"status": status} 2129 2129 + if permit is not None: 2130 2130 + result["permit"] = permit 2131 2131 + if reason is not None: 2132 2132 + result["reason"] = reason 2133 2133 + return result 2134 2134 + 2135 2135 + 2002 2136 def _record_from_evidence( 2003 2137 *, 2004 2138 evidence: BrainEvidenceRecord, ··· 2082 2216 raise BrainStateConflictError("brain runtime failure marker changed") 2083 2217 2084 2218 2219 2219 + def begin_brain_prerequisite_renewal( 2220 2220 + now: datetime, 2221 2221 + *, 2222 2222 + expected_fingerprint_sha256: str | None = None, 2223 2223 + run_id: str | None = None, 2224 2224 + journal_path: str | Path | None = None, 2225 2225 + ) -> BrainPrerequisiteRenewalBeginResult: 2226 2226 + """Begin a fenced SPP prerequisite-only renewal. 2227 2227 + 2228 2228 + This is deliberately narrower than ``begin_brain_refresh``: it preserves 2229 2229 + same-fingerprint model evidence and refuses to start unless replacing only 2230 2230 + ``lane_prerequisites`` can be made safe. 2231 2231 + """ 2232 2232 + 2233 2233 + try: 2234 2234 + now = _utc(now) 2235 2235 + except ValueError as exc: 2236 2236 + return _prerequisite_renewal_begin_result("unsafe", reason=str(exc)) 2237 2237 + if expected_fingerprint_sha256 is not None: 2238 2238 + try: 2239 2239 + _validate_hex(expected_fingerprint_sha256, "expected_fingerprint_sha256") 2240 2240 + except BrainStateValidationError: 2241 2241 + return _prerequisite_renewal_begin_result( 2242 2242 + "unsafe", reason="fingerprint_mismatch" 2243 2243 + ) 2244 2244 + path = brain_state_path(journal_path=journal_path) 2245 2245 + lease = acquire_file_lease(brain_refresh_lease_path(journal_path=journal_path)) 2246 2246 + if lease is None: 2247 2247 + return _prerequisite_renewal_begin_result("busy", reason="lease_held") 2248 2248 + try: 2249 2249 + try: 2250 2250 + _config, key, fingerprint = _load_fingerprint_for_write( 2251 2251 + journal_path=journal_path 2252 2252 + ) 2253 2253 + except (CorruptConfigError, OSError, BrainStateValidationError) as exc: 2254 2254 + lease.release() 2255 2255 + return _prerequisite_renewal_begin_result("unsafe", reason=str(exc)) 2256 2256 + if key is None or fingerprint is None or not fingerprint["ok"]: 2257 2257 + lease.release() 2258 2258 + return _prerequisite_renewal_begin_result( 2259 2259 + "unsafe", reason="fingerprint_not_available" 2260 2260 + ) 2261 2261 + if fingerprint["active_lane"] != "spp": 2262 2262 + lease.release() 2263 2263 + return _prerequisite_renewal_begin_result("unsafe", reason="non_spp_lane") 2264 2264 + fingerprint_sha = fingerprint["fingerprint_sha256"] 2265 2265 + if fingerprint_sha is None or ( 2266 2266 + expected_fingerprint_sha256 is not None 2267 2267 + and fingerprint_sha != expected_fingerprint_sha256 2268 2268 + ): 2269 2269 + lease.release() 2270 2270 + return _prerequisite_renewal_begin_result( 2271 2271 + "unsafe", reason="fingerprint_mismatch" 2272 2272 + ) 2273 2273 + 2274 2274 + run_id = run_id or uuid.uuid4().hex 2275 2275 + expires_at = now + CHECKING_TTL 2276 2276 + with hold_lock(path, mode=BRAIN_FILE_MODE): 2277 2277 + try: 2278 2278 + current = _read_record_unlocked(path) 2279 2279 + except ( 2280 2280 + OSError, 2281 2281 + BrainStateValidationError, 2282 2282 + MalformedDataError, 2283 2283 + json.JSONDecodeError, 2284 2284 + ): 2285 2285 + lease.release() 2286 2286 + return _prerequisite_renewal_begin_result( 2287 2287 + "unsafe", reason="brain_record_unavailable" 2288 2288 + ) 2289 2289 + if current is None or current["fingerprint_sha256"] != fingerprint_sha: 2290 2290 + lease.release() 2291 2291 + return _prerequisite_renewal_begin_result( 2292 2292 + "unsafe", reason="brain_record_missing" 2293 2293 + ) 2294 2294 + evidence = _safe_prerequisite_renewal_evidence(current, now) 2295 2295 + if evidence is None: 2296 2296 + lease.release() 2297 2297 + return _prerequisite_renewal_begin_result( 2298 2298 + "unsafe", reason="unsafe_evidence" 2299 2299 + ) 2300 2300 + revision = _next_revision(current) 2301 2301 + marker_seen = _runtime_failure_marker_id(current) 2302 2302 + checking: BrainCheckingRecord = { 2303 2303 + "run_id": run_id, 2304 2304 + "started_at": _iso(now), 2305 2305 + "expires_at": _iso(expires_at), 2306 2306 + "fingerprint_sha256": fingerprint_sha, 2307 2307 + "checking_revision": revision, 2308 2308 + "runtime_failure_marker_seen": marker_seen, 2309 2309 + } 2310 2310 + record = _record( 2311 2311 + revision=revision, 2312 2312 + aggregate_state="checking", 2313 2313 + reason_code="brain_check_in_progress", 2314 2314 + active_lane="spp", 2315 2315 + active_provider=fingerprint["active_provider"], 2316 2316 + active_model=fingerprint["active_model"], 2317 2317 + fingerprint_sha256=fingerprint_sha, 2318 2318 + checking=checking, 2319 2319 + evidence=evidence, 2320 2320 + runtime_failure_marker=current["runtime_failure_marker"], 2321 2321 + diagnostic={}, 2322 2322 + now=now, 2323 2323 + ) 2324 2324 + _write_record(path, record) 2325 2325 + return _prerequisite_renewal_begin_result( 2326 2326 + "started", 2327 2327 + permit=BrainRefreshPermit( 2328 2328 + run_id=run_id, 2329 2329 + started_at=now, 2330 2330 + expires_at=expires_at, 2331 2331 + fingerprint_sha256=fingerprint_sha, 2332 2332 + checking_revision=revision, 2333 2333 + runtime_failure_marker_seen=marker_seen, 2334 2334 + lease=lease, 2335 2335 + ), 2336 2336 + ) 2337 2337 + except BaseException: 2338 2338 + lease.release() 2339 2339 + raise 2340 2340 + 2341 2341 + 2342 2342 + def _validate_lane_prerequisite_component( 2343 2343 + component: Mapping[str, Any], 2344 2344 + ) -> BrainEvidenceComponent: 2345 2345 + validated = _validate_component( 2346 2346 + component, 2347 2347 + "lane_prerequisites", 2348 2348 + component_name="lane_prerequisites", 2349 2349 + ) 2350 2350 + if validated is None or validated["status"] == "not_attempted": 2351 2351 + raise BrainStateValidationError( 2352 2352 + "lane_prerequisites.status", 2353 2353 + "prerequisite renewal requires ok or declared failure", 2354 2354 + ) 2355 2355 + return validated 2356 2356 + 2357 2357 + 2358 2358 + def finish_brain_prerequisite_renewal( 2359 2359 + permit: BrainRefreshPermit, 2360 2360 + lane_prerequisites: Mapping[str, Any], 2361 2361 + now: datetime, 2362 2362 + *, 2363 2363 + journal_path: str | Path | None = None, 2364 2364 + ) -> BrainStateRecord: 2365 2365 + now = _utc(now) 2366 2366 + path = brain_state_path(journal_path=journal_path) 2367 2367 + try: 2368 2368 + component = _validate_lane_prerequisite_component(lane_prerequisites) 2369 2369 + _config, key, fingerprint = _load_fingerprint_for_write( 2370 2370 + journal_path=journal_path 2371 2371 + ) 2372 2372 + if key is None or fingerprint is None: 2373 2373 + raise BrainStateConflictError("brain fingerprint key is unavailable") 2374 2374 + with hold_lock(path, mode=BRAIN_FILE_MODE): 2375 2375 + current = _read_record_unlocked(path) 2376 2376 + _assert_finish_allowed(permit, current, now) 2377 2377 + if fingerprint["fingerprint_sha256"] != permit.fingerprint_sha256: 2378 2378 + raise BrainStateConflictError("brain fingerprint changed") 2379 2379 + if fingerprint["active_lane"] != "spp": 2380 2380 + raise BrainStateConflictError("brain lane changed") 2381 2381 + assert current is not None 2382 2382 + evidence = _safe_prerequisite_renewal_evidence(current, now) 2383 2383 + if evidence is None: 2384 2384 + raise BrainStateConflictError("brain prerequisite evidence is unsafe") 2385 2385 + evidence["lane_prerequisites"] = component 2386 2386 + record = _record_from_evidence( 2387 2387 + evidence=evidence, 2388 2388 + fingerprint=fingerprint, 2389 2389 + revision=_next_revision(current), 2390 2390 + now=now, 2391 2391 + checking=None, 2392 2392 + runtime_failure_marker=None, 2393 2393 + ) 2394 2394 + return _write_record(path, record) 2395 2395 + finally: 2396 2396 + permit.release() 2397 2397 + 2398 2398 + 2399 2399 + def abandon_brain_prerequisite_renewal( 2400 2400 + permit: BrainRefreshPermit, 2401 2401 + reason_code: BrainReasonCode, 2402 2402 + now: datetime, 2403 2403 + *, 2404 2404 + diagnostic: Mapping[str, BrainDiagnosticValue] | None = None, 2405 2405 + journal_path: str | Path | None = None, 2406 2406 + ) -> BrainStateRecord: 2407 2407 + component = _component( 2408 2408 + _component_status_for_reason(reason_code), 2409 2409 + _utc(now), 2410 2410 + reason_code=reason_code, 2411 2411 + diagnostic=diagnostic, 2412 2412 + ) 2413 2413 + return finish_brain_prerequisite_renewal( 2414 2414 + permit, 2415 2415 + component, 2416 2416 + now, 2417 2417 + journal_path=journal_path, 2418 2418 + ) 2419 2419 + 2420 2420 + 2085 2421 def finish_brain_refresh( 2086 2422 permit: BrainRefreshPermit, 2087 2423 outcome: BrainProbeOutcome, ··· 2320 2656 "BrainLaneId", 2321 2657 "BrainProbeOutcome", 2322 2658 "BrainProjection", 2659 2659 + "BrainPrerequisiteRenewalBeginResult", 2660 2660 + "BrainPrerequisiteRenewalStatus", 2323 2661 "BrainReasonCode", 2324 2662 "BrainRefreshPermit", 2325 2663 "BrainRuntimeFailureComponent", 2326 2664 "BrainRuntimeFailureMarker", 2327 2665 "BrainRuntimeFailureResult", 2666 2666 + "BrainStateExpectedFingerprintStaleError", 2328 2667 "BrainStateConflictError", 2329 2668 "BrainStateInspection", 2330 2669 "BrainStateRecord", 2331 2670 "BrainStateValidationError", 2332 2671 "abandon_brain_refresh", 2672 2672 + "abandon_brain_prerequisite_renewal", 2333 2673 "brain_fingerprint_key_path", 2334 2674 "brain_refresh_lease_path", 2335 2675 "brain_state_path", 2336 2676 "begin_brain_refresh", 2677 2677 + "begin_brain_prerequisite_renewal", 2337 2678 "build_active_brain_fingerprint", 2338 2679 "derive_active_brain_lane", 2339 2680 "finish_brain_refresh", 2681 2681 + "finish_brain_prerequisite_renewal", 2340 2682 "inspect_brain_state", 2341 2683 "project_brain_state", 2342 2684 "read_active_brain_fingerprint_sha256",

+9

solstone/think/services/spp_attest/nvgpu/appraise.py

View file Reviewed

··· 28 28 from solstone.think.services.spp_attest.tlv import GpuEnvelope 29 29 30 30 log = logging.getLogger(__name__) 31 31 + NVATTEST_TIMEOUT_S = 60.0 31 32 32 33 33 34 def appraise_gpu_leg( ··· 85 86 capture_output=True, 86 87 text=True, 87 88 check=False, 89 89 + timeout=NVATTEST_TIMEOUT_S, 88 90 ) 91 91 + except subprocess.TimeoutExpired as exc: 92 92 + _log_gpu_appraisal_failure( 93 93 + "gpu_appraisal_failed", 94 94 + exception_class=type(exc).__name__, 95 95 + stderr=exc.stderr, 96 96 + ) 97 97 + raise GpuAppraisalError("gpu_appraisal_failed") from exc 89 98 except OSError as exc: 90 99 _log_gpu_appraisal_failure( 91 100 "nvattest_unavailable",

+4 -7

solstone/think/services/spp_transport.py

View file Reviewed

··· 18 18 19 19 from OpenSSL import SSL 20 20 21 21 - from solstone.think.models import AttestationFailedError, AttestationStaleError 21 21 + from solstone.think.models import AttestationFailedError 22 22 from solstone.think.providers.nvattest_install import ( 23 23 ensure_nvattest_installed, 24 24 nvattest_cache_ready, ··· 260 260 ) 261 261 262 262 263 263 - def _reuse_or_raise_stale_locked(now: datetime) -> bool: 263 263 + def _reuse_or_teardown_stale_locked(now: datetime) -> bool: 264 264 state = spp.get_attestation_state() 265 265 if ( 266 266 state.session is not None ··· 274 274 and _transport_live_locked() 275 275 ): 276 276 _teardown_locked() 277 277 - raise AttestationStaleError( 278 278 - "the confidential attestation cadence lapsed (attestation_stale)" 279 279 - ) 280 277 return False 281 278 282 279 ··· 286 283 now = datetime.now(timezone.utc) 287 284 with _LOCK: 288 285 _CONFIDENTIAL_BLOCK = dict(block) 289 289 - if _reuse_or_raise_stale_locked(now): 286 286 + if _reuse_or_teardown_stale_locked(now): 290 287 return 291 288 292 289 nvattest_dir = _ensure_nvattest_for_attestation(block) 293 290 with _LOCK: 294 291 _CONFIDENTIAL_BLOCK = dict(block) 295 295 - if _reuse_or_raise_stale_locked(now): 292 292 + if _reuse_or_teardown_stale_locked(now): 296 293 return 297 294 _establish_and_record_locked(block, now, nvattest_dir=nvattest_dir) 298 295

+19

tests/services/test_spp_attest_cadence.py

View file Reviewed

··· 88 88 ) 89 89 90 90 91 91 + def test_attestation_session_verified_one_microsecond_before_each_boundary() -> None: 92 92 + assert ( 93 93 + _session(tpm_heartbeat_at=NOW - TPM_HEARTBEAT_INTERVAL).status( 94 94 + NOW - timedelta(microseconds=1) 95 95 + ) 96 96 + == "verified" 97 97 + ) 98 98 + assert ( 99 99 + _session(gpu_reattest_at=NOW - GPU_REATTEST_INTERVAL).status( 100 100 + NOW - timedelta(microseconds=1) 101 101 + ) 102 102 + == "verified" 103 103 + ) 104 104 + assert ( 105 105 + _session(started_at=NOW - SESSION_CAP).status(NOW - timedelta(microseconds=1)) 106 106 + == "verified" 107 107 + ) 108 108 + 109 109 + 91 110 def test_attestation_session_stale_when_tpm_heartbeat_lapses() -> None: 92 111 session = _session(tpm_heartbeat_at=NOW - TPM_HEARTBEAT_INTERVAL) 93 112

+39

tests/services/test_spp_attest_nvgpu.py

View file Reviewed

··· 396 396 assert "collector detail" not in message 397 397 398 398 399 399 + def test_nvattest_timeout_is_bounded_redacted_and_cleans_evidence( 400 400 + tmp_path: Path, 401 401 + monkeypatch: pytest.MonkeyPatch, 402 402 + caplog: pytest.LogCaptureFixture, 403 403 + ) -> None: 404 404 + nvattest_dir = _fake_nvattest_dir(tmp_path) 405 405 + observed: dict[str, Any] = {} 406 406 + marker = "timeout-secret-nonce_from_ar" 407 407 + 408 408 + def timeout_run(argv, **kwargs): 409 409 + observed["timeout"] = kwargs["timeout"] 410 410 + evidence_path = Path(argv[argv.index("--gpu-evidence-file") + 1]) 411 411 + observed["evidence_path"] = evidence_path 412 412 + assert evidence_path.exists() 413 413 + raise subprocess.TimeoutExpired(argv, kwargs["timeout"], stderr=marker) 414 414 + 415 415 + monkeypatch.setattr(appraise_module.subprocess, "run", timeout_run) 416 416 + caplog.set_level(logging.WARNING, logger=appraise_module.log.name) 417 417 + 418 418 + with pytest.raises(GpuAppraisalError) as exc_info: 419 419 + appraise_module.appraise_gpu_leg( 420 420 + _envelope(), 421 421 + _owner_nonce(), 422 422 + nvattest_dir=nvattest_dir, 423 423 + ) 424 424 + 425 425 + assert exc_info.value.reason == "gpu_appraisal_failed" 426 426 + assert observed["timeout"] == appraise_module.NVATTEST_TIMEOUT_S 427 427 + assert not observed["evidence_path"].exists() 428 428 + messages = [ 429 429 + record.getMessage() 430 430 + for record in caplog.records 431 431 + if "event=nvattest_gpu_appraisal_failed" in record.getMessage() 432 432 + ] 433 433 + assert len(messages) == 1 434 434 + assert "exception=TimeoutExpired" in messages[0] 435 435 + assert marker not in messages[0] 436 436 + 437 437 + 399 438 def test_bool_false_returncode_rejects( 400 439 tmp_path: Path, 401 440 monkeypatch: pytest.MonkeyPatch,

+117 -10

tests/services/test_spp_transport.py

View file Reviewed

··· 4 4 from __future__ import annotations 5 5 6 6 import json 7 7 + import threading 7 8 import time 8 9 from datetime import datetime, timedelta, timezone 9 10 from pathlib import Path ··· 13 14 import pytest 14 15 15 16 from solstone.think.journal_io.locking import hold_lock 16 16 - from solstone.think.models import AttestationFailedError, AttestationStaleError 17 17 + from solstone.think.models import AttestationFailedError 17 18 from solstone.think.providers import nvattest_install 18 19 from solstone.think.services import spp, spp_transport 19 20 from solstone.think.services.spp_attest.cadence import AttestationSession ··· 124 125 ) 125 126 126 127 127 127 - def test_verify_confidential_attestation_reuses_then_raises_stale_once_then_reestablishes( 128 128 + def test_verify_confidential_attestation_reuses_then_rotates_stale_session_inline( 128 129 tmp_path: Path, 129 130 monkeypatch: pytest.MonkeyPatch, 130 131 ) -> None: ··· 145 146 assert spp.get_attestation_state().session is not None 146 147 147 148 spp.record_attestation_verified(_stale_session(verdict)) 148 148 - with pytest.raises(AttestationStaleError): 149 149 - spp_transport.verify_confidential_attestation(block) 150 150 - 151 151 - state = spp.get_attestation_state() 152 152 - assert state.session is not None 153 153 - assert state.session.status(datetime.now(timezone.utc)) == "stale" 154 154 - assert establish.call_count == 1 155 155 - 156 149 spp_transport.verify_confidential_attestation(block) 157 150 158 151 assert establish.call_count == 2 ··· 161 154 spp.get_attestation_state().session.status(datetime.now(timezone.utc)) 162 155 == "verified" 163 156 ) 157 157 + spp_transport.verify_confidential_attestation(block) 158 158 + assert establish.call_count == 2 159 159 + 160 160 + 161 161 + def test_concurrent_stale_rotation_establishes_one_replacement( 162 162 + tmp_path: Path, 163 163 + monkeypatch: pytest.MonkeyPatch, 164 164 + ) -> None: 165 165 + block = _write_confidential_config(tmp_path, monkeypatch) 166 166 + _patch_listener(monkeypatch) 167 167 + old_verdict = object() 168 168 + old_channel = _FakeChannel(old_verdict) 169 169 + spp.record_attestation_verified(_stale_session(old_verdict)) 170 170 + with spp_transport._LOCK: 171 171 + spp_transport._LISTENER = _FakeListener() 172 172 + spp_transport._LISTENER_THREAD = _AliveThread() 173 173 + spp_transport._FORWARDER_BASE_URL = "http://127.0.0.1:1111" 174 174 + spp_transport._POOL[:] = [old_channel] 175 175 + 176 176 + ensure_barrier = threading.Barrier(2) 177 177 + ensure_calls = 0 178 178 + 179 179 + def delayed_ensure(_block): 180 180 + nonlocal ensure_calls 181 181 + ensure_calls += 1 182 182 + ensure_barrier.wait(timeout=2) 183 183 + return Path("/tmp/solstone-nvattest-test") 184 184 + 185 185 + establish_started = threading.Event() 186 186 + release_establish = threading.Event() 187 187 + 188 188 + def delayed_establish(*_args, **kwargs): 189 189 + establish_started.set() 190 190 + assert release_establish.wait(timeout=2) 191 191 + return _FakeChannel(object(), epoch=kwargs["epoch"]) 192 192 + 193 193 + establish = Mock(side_effect=delayed_establish) 194 194 + monkeypatch.setattr( 195 195 + spp_transport, 196 196 + "_ensure_nvattest_for_attestation", 197 197 + delayed_ensure, 198 198 + ) 199 199 + monkeypatch.setattr(spp_transport, "establish_attested_channel", establish) 200 200 + 201 201 + errors: list[BaseException] = [] 202 202 + 203 203 + def verify() -> None: 204 204 + try: 205 205 + spp_transport.verify_confidential_attestation(block) 206 206 + except BaseException as exc: # pragma: no cover - surfaced below 207 207 + errors.append(exc) 208 208 + 209 209 + first = threading.Thread(target=verify) 210 210 + second = threading.Thread(target=verify) 211 211 + first.start() 212 212 + second.start() 213 213 + assert establish_started.wait(timeout=2) 214 214 + assert first.is_alive() 215 215 + assert second.is_alive() 216 216 + release_establish.set() 217 217 + first.join(timeout=2) 218 218 + second.join(timeout=2) 219 219 + 220 220 + assert errors == [] 221 221 + assert not first.is_alive() 222 222 + assert not second.is_alive() 223 223 + assert ensure_calls == 2 224 224 + assert establish.call_count == 1 225 225 + assert old_channel.closed is True 226 226 + assert spp.get_attestation_state().session is not None 164 227 165 228 166 229 def test_confidential_egress_base_url_returns_forwarder_not_configured_endpoint( ··· 236 299 237 300 with pytest.raises(AttestationFailedError): 238 301 spp_transport.confidential_forwarder_base_url() 302 302 + 303 303 + 304 304 + def test_stale_rotation_failure_returns_no_forwarder_and_later_recovers( 305 305 + tmp_path: Path, 306 306 + monkeypatch: pytest.MonkeyPatch, 307 307 + ) -> None: 308 308 + block = _write_confidential_config(tmp_path, monkeypatch) 309 309 + _patch_listener(monkeypatch) 310 310 + old_verdict = object() 311 311 + spp.record_attestation_verified(_stale_session(old_verdict)) 312 312 + old_channel = _FakeChannel(old_verdict) 313 313 + spp_transport._POOL[:] = [old_channel] 314 314 + establish = Mock( 315 315 + side_effect=[ 316 316 + RatlsChannelError("gateway_unreachable"), 317 317 + _FakeChannel(object()), 318 318 + ] 319 319 + ) 320 320 + monkeypatch.setattr(spp_transport, "establish_attested_channel", establish) 321 321 + pump = Mock(return_value="local_closed") 322 322 + monkeypatch.setattr(spp_transport, "_pump", pump) 323 323 + 324 324 + with pytest.raises(AttestationFailedError): 325 325 + spp_transport.confidential_egress_base_url(block["endpoint_url"]) 326 326 + assert old_channel.closed is True 327 327 + assert spp_transport._FORWARDER_BASE_URL is None 328 328 + assert spp_transport._LISTENER is None 329 329 + assert spp_transport._POOL == [] 330 330 + assert spp_transport._ACTIVE == set() 331 331 + assert spp.get_attestation_state().session is None 332 332 + assert spp.get_attestation_state().failure is not None 333 333 + local = _FakeLocal() 334 334 + spp_transport._handle_loopback_connection(local) 335 335 + assert local.closed is True 336 336 + pump.assert_not_called() 337 337 + 338 338 + assert ( 339 339 + spp_transport.confidential_egress_base_url(block["endpoint_url"]) 340 340 + == "http://127.0.0.1:4567" 341 341 + ) 342 342 + assert establish.call_count == 2 343 343 + assert spp.get_attestation_state().session is not None 344 344 + assert spp_transport._POOL 345 345 + assert all(channel is not old_channel for channel in spp_transport._POOL) 239 346 240 347 241 348 def test_confidential_probe_status_reads_state_without_attestation(

+229 -2

tests/test_brain_cli.py

View file Reviewed

··· 8 8 import json 9 9 import logging 10 10 import sys 11 11 - from datetime import datetime, timezone 11 11 + from datetime import datetime, timedelta, timezone 12 12 from pathlib import Path 13 13 from types import SimpleNamespace 14 14 from typing import Any ··· 145 145 146 146 147 147 def _args(**kwargs: Any) -> argparse.Namespace: 148 148 - defaults = {"json": False, "expected_fingerprint": None} 148 148 + defaults = { 149 149 + "json": False, 150 150 + "expected_fingerprint": None, 151 151 + "expected_active_fingerprint": False, 152 152 + "expect_active_fingerprint_absent": False, 153 153 + } 149 154 defaults.update(kwargs) 150 155 return argparse.Namespace(**defaults) 151 156 ··· 641 646 assert record["evidence"]["cogitate"]["reason_code"] == "local_runtime_not_ready" 642 647 643 648 649 649 + def test_renew_prerequisites_updates_spp_only_without_model_probes( 650 650 + brain_journal: Path, 651 651 + monkeypatch: pytest.MonkeyPatch, 652 652 + capsys: pytest.CaptureFixture[str], 653 653 + ) -> None: 654 654 + config = _endpoint_config(confidential=True) 655 655 + prior_now = NOW - timedelta(minutes=1) 656 656 + _write_ready_record(brain_journal, config, now=prior_now) 657 657 + before = json.loads(brain_state_path(journal_path=brain_journal).read_text()) 658 658 + expected = brain_state_module.read_active_brain_fingerprint_sha256( 659 659 + journal_path=brain_journal 660 660 + ) 661 661 + assert expected is not None 662 662 + _forbid_provider_calls(monkeypatch) 663 663 + monkeypatch.setattr( 664 664 + brain_cli, 665 665 + "_spp_prerequisite", 666 666 + lambda now: ( 667 667 + brain_cli._ok_component(now, expires_at=now + timedelta(minutes=10)), 668 668 + None, 669 669 + ), 670 670 + ) 671 671 + 672 672 + code = brain_cli._run_renew_prerequisites( 673 673 + _args(json=True, expected_fingerprint=expected) 674 674 + ) 675 675 + payload = json.loads(capsys.readouterr().out) 676 676 + record = json.loads(brain_state_path(journal_path=brain_journal).read_text()) 677 677 + 678 678 + assert code == 0 679 679 + assert payload["aggregate_state"] == "ready" 680 680 + assert record["evidence"]["configuration"] == before["evidence"]["configuration"] 681 681 + assert record["evidence"]["generate"] == before["evidence"]["generate"] 682 682 + assert record["evidence"]["cogitate"] == before["evidence"]["cogitate"] 683 683 + assert record["evidence"]["lane_prerequisites"]["observed_at"] == NOW.isoformat() 684 684 + 685 685 + 686 686 + def test_renew_prerequisites_busy_does_not_fallback( 687 687 + brain_journal: Path, 688 688 + monkeypatch: pytest.MonkeyPatch, 689 689 + capsys: pytest.CaptureFixture[str], 690 690 + ) -> None: 691 691 + _write_ready_record(brain_journal, _endpoint_config(confidential=True)) 692 692 + lease = acquire_file_lease(brain_refresh_lease_path(journal_path=brain_journal)) 693 693 + assert lease is not None 694 694 + monkeypatch.setattr( 695 695 + brain_cli, 696 696 + "_run_refresh", 697 697 + lambda _args: pytest.fail("busy must not fallback to full refresh"), 698 698 + ) 699 699 + _forbid_provider_calls(monkeypatch) 700 700 + try: 701 701 + code = brain_cli._run_renew_prerequisites(_args(json=True)) 702 702 + finally: 703 703 + lease.release() 704 704 + payload = json.loads(capsys.readouterr().out) 705 705 + 706 706 + assert code == 3 707 707 + assert payload["reason_code"] == "busy" 708 708 + 709 709 + 710 710 + def test_renew_prerequisites_unsafe_dispatches_full_refresh( 711 711 + brain_journal: Path, 712 712 + monkeypatch: pytest.MonkeyPatch, 713 713 + ) -> None: 714 714 + _write_config(brain_journal, _cloud_config()) 715 715 + calls: list[argparse.Namespace] = [] 716 716 + 717 717 + def fake_refresh(args: argparse.Namespace) -> int: 718 718 + calls.append(args) 719 719 + return 7 720 720 + 721 721 + monkeypatch.setattr(brain_cli, "_run_refresh", fake_refresh) 722 722 + 723 723 + assert brain_cli._run_renew_prerequisites(_args(json=True)) == 7 724 724 + assert len(calls) == 1 725 725 + assert calls[0].expected_active_fingerprint is True 726 726 + 727 727 + 728 728 + def test_renew_prerequisites_expected_mismatch_writes_nothing( 729 729 + brain_journal: Path, 730 730 + monkeypatch: pytest.MonkeyPatch, 731 731 + capsys: pytest.CaptureFixture[str], 732 732 + ) -> None: 733 733 + _write_ready_record(brain_journal, _endpoint_config(confidential=True)) 734 734 + before = _health_snapshot(brain_journal) 735 735 + _forbid_provider_calls(monkeypatch) 736 736 + 737 737 + code = brain_cli._run_renew_prerequisites( 738 738 + _args(json=True, expected_fingerprint="f" * 64) 739 739 + ) 740 740 + payload = json.loads(capsys.readouterr().out) 741 741 + 742 742 + assert code == 3 743 743 + assert payload["reason_code"] == "stale_expected_fingerprint" 744 744 + assert _health_snapshot(brain_journal) == before 745 745 + 746 746 + 644 747 def test_refresh_cloud_missing_key_skips_inference( 645 748 brain_journal: Path, 646 749 monkeypatch: pytest.MonkeyPatch, ··· 963 1066 964 1067 assert code == 3 965 1068 assert payload["reason_code"] == "stale_expected_fingerprint" 1069 1069 + 1070 1070 + 1071 1071 + def test_refresh_expected_active_fingerprint_mismatch_writes_nothing( 1072 1072 + brain_journal: Path, 1073 1073 + monkeypatch: pytest.MonkeyPatch, 1074 1074 + capsys: pytest.CaptureFixture[str], 1075 1075 + ) -> None: 1076 1076 + original = _endpoint_config(confidential=True) 1077 1077 + changed = _endpoint_config(confidential=True) 1078 1078 + changed["providers"]["local"]["served_model_id"] = "new-served-model" 1079 1079 + changed["services"]["confidential"]["served_model_id"] = "new-served-model" 1080 1080 + _write_ready_record(brain_journal, original) 1081 1081 + expected = brain_state_module.read_active_brain_fingerprint_sha256( 1082 1082 + journal_path=brain_journal 1083 1083 + ) 1084 1084 + assert expected is not None 1085 1085 + before = _health_snapshot(brain_journal) 1086 1086 + _write_config(brain_journal, changed) 1087 1087 + _forbid_provider_calls(monkeypatch) 1088 1088 + 1089 1089 + code = brain_cli._run_refresh( 1090 1090 + _args( 1091 1091 + json=True, 1092 1092 + expected_fingerprint=expected, 1093 1093 + expected_active_fingerprint=True, 1094 1094 + ) 1095 1095 + ) 1096 1096 + payload = json.loads(capsys.readouterr().out) 1097 1097 + 1098 1098 + assert code == 3 1099 1099 + assert payload["reason_code"] == "stale_expected_fingerprint" 1100 1100 + assert _health_snapshot(brain_journal) == before 1101 1101 + 1102 1102 + 1103 1103 + def test_refresh_expected_active_fingerprint_race_before_begin_writes_nothing( 1104 1104 + brain_journal: Path, 1105 1105 + monkeypatch: pytest.MonkeyPatch, 1106 1106 + capsys: pytest.CaptureFixture[str], 1107 1107 + ) -> None: 1108 1108 + original = _endpoint_config(confidential=True) 1109 1109 + changed = _endpoint_config(confidential=True) 1110 1110 + changed["providers"]["local"]["served_model_id"] = "race-served-model" 1111 1111 + changed["services"]["confidential"]["served_model_id"] = "race-served-model" 1112 1112 + _write_ready_record(brain_journal, original) 1113 1113 + expected = brain_state_module.read_active_brain_fingerprint_sha256( 1114 1114 + journal_path=brain_journal 1115 1115 + ) 1116 1116 + assert expected is not None 1117 1117 + before = _health_snapshot(brain_journal) 1118 1118 + real_match = brain_cli._expected_refresh_fingerprint_matches 1119 1119 + 1120 1120 + def match_then_switch(args: argparse.Namespace, value: str) -> bool: 1121 1121 + assert real_match(args, value) 1122 1122 + _write_config(brain_journal, changed) 1123 1123 + return True 1124 1124 + 1125 1125 + monkeypatch.setattr( 1126 1126 + brain_cli, 1127 1127 + "_expected_refresh_fingerprint_matches", 1128 1128 + match_then_switch, 1129 1129 + ) 1130 1130 + _forbid_provider_calls(monkeypatch) 1131 1131 + 1132 1132 + code = brain_cli._run_refresh( 1133 1133 + _args( 1134 1134 + json=True, 1135 1135 + expected_fingerprint=expected, 1136 1136 + expected_active_fingerprint=True, 1137 1137 + ) 1138 1138 + ) 1139 1139 + payload = json.loads(capsys.readouterr().out) 1140 1140 + 1141 1141 + assert code == 3 1142 1142 + assert payload["reason_code"] == "stale_expected_fingerprint" 1143 1143 + assert _health_snapshot(brain_journal) == before 1144 1144 + 1145 1145 + 1146 1146 + def test_refresh_expected_absent_fingerprint_bootstraps_spp( 1147 1147 + brain_journal: Path, 1148 1148 + monkeypatch: pytest.MonkeyPatch, 1149 1149 + capsys: pytest.CaptureFixture[str], 1150 1150 + ) -> None: 1151 1151 + _write_config(brain_journal, _endpoint_config(confidential=True)) 1152 1152 + assert not brain_fingerprint_key_path(journal_path=brain_journal).exists() 1153 1153 + monkeypatch.setattr( 1154 1154 + brain_cli, 1155 1155 + "_spp_prerequisite", 1156 1156 + lambda now: ( 1157 1157 + brain_cli._ok_component(now, expires_at=now + timedelta(minutes=10)), 1158 1158 + None, 1159 1159 + ), 1160 1160 + ) 1161 1161 + generate_calls, cogitate_calls = _patch_generate_and_cogitate(monkeypatch) 1162 1162 + 1163 1163 + code = brain_cli._run_refresh( 1164 1164 + _args(json=True, expect_active_fingerprint_absent=True) 1165 1165 + ) 1166 1166 + payload = json.loads(capsys.readouterr().out) 1167 1167 + 1168 1168 + assert code == 0 1169 1169 + assert payload["aggregate_state"] == "ready" 1170 1170 + assert payload["lane"] == "spp" 1171 1171 + assert brain_fingerprint_key_path(journal_path=brain_journal).exists() 1172 1172 + assert len(generate_calls) == 1 1173 1173 + assert len(cogitate_calls) == 1 1174 1174 + 1175 1175 + 1176 1176 + def test_refresh_expected_absent_fingerprint_stale_when_key_appears( 1177 1177 + brain_journal: Path, 1178 1178 + monkeypatch: pytest.MonkeyPatch, 1179 1179 + capsys: pytest.CaptureFixture[str], 1180 1180 + ) -> None: 1181 1181 + _write_ready_record(brain_journal, _endpoint_config(confidential=True)) 1182 1182 + before = _health_snapshot(brain_journal) 1183 1183 + _forbid_provider_calls(monkeypatch) 1184 1184 + 1185 1185 + code = brain_cli._run_refresh( 1186 1186 + _args(json=True, expect_active_fingerprint_absent=True) 1187 1187 + ) 1188 1188 + payload = json.loads(capsys.readouterr().out) 1189 1189 + 1190 1190 + assert code == 3 1191 1191 + assert payload["reason_code"] == "stale_expected_fingerprint" 1192 1192 + assert _health_snapshot(brain_journal) == before 966 1193 967 1194 968 1195 def test_expected_fingerprint_match_proceeds_and_ready_short_circuits(

+390

tests/test_brain_state.py

View file Reviewed

··· 27 27 DEFAULT_READY_EVIDENCE_TTL, 28 28 BrainProbeOutcome, 29 29 BrainStateConflictError, 30 30 + BrainStateExpectedFingerprintStaleError, 30 31 BrainStateValidationError, 32 32 + abandon_brain_prerequisite_renewal, 31 33 abandon_brain_refresh, 34 34 + begin_brain_prerequisite_renewal, 32 35 begin_brain_refresh, 33 36 brain_fingerprint_key_path, 34 37 brain_state_path, 35 38 build_active_brain_fingerprint, 39 39 + finish_brain_prerequisite_renewal, 36 40 finish_brain_refresh, 37 41 inspect_brain_state, 38 42 project_brain_state, ··· 1050 1054 assert result["rejected_reason"] == "fingerprint_mismatch" 1051 1055 assert state_path.read_bytes() == prior_bytes 1052 1056 assert _read_raw_record(tmp_path)["revision"] == prior_revision 1057 1057 + 1058 1058 + 1059 1059 + def test_begin_refresh_expected_active_fingerprint_stales_after_switch( 1060 1060 + tmp_path: Path, 1061 1061 + ) -> None: 1062 1062 + original = _spp_config(account_id="acct-a") 1063 1063 + _write_ready_record(tmp_path, original) 1064 1064 + expected = _current_fingerprint(tmp_path, original) 1065 1065 + before_record = brain_state_path(journal_path=tmp_path).read_bytes() 1066 1066 + before_key = brain_fingerprint_key_path(journal_path=tmp_path).read_bytes() 1067 1067 + _write_config(tmp_path, _spp_config(account_id="acct-b")) 1068 1068 + 1069 1069 + with pytest.raises(BrainStateExpectedFingerprintStaleError): 1070 1070 + begin_brain_refresh( 1071 1071 + NOW + timedelta(seconds=1), 1072 1072 + expected_active_fingerprint_sha256=expected, 1073 1073 + journal_path=tmp_path, 1074 1074 + ) 1075 1075 + 1076 1076 + assert brain_state_path(journal_path=tmp_path).read_bytes() == before_record 1077 1077 + assert brain_fingerprint_key_path(journal_path=tmp_path).read_bytes() == before_key 1078 1078 + 1079 1079 + 1080 1080 + def test_begin_refresh_expected_absent_fingerprint_bootstraps_key( 1081 1081 + tmp_path: Path, 1082 1082 + ) -> None: 1083 1083 + _write_config(tmp_path, _spp_config()) 1084 1084 + assert not brain_fingerprint_key_path(journal_path=tmp_path).exists() 1085 1085 + 1086 1086 + permit = begin_brain_refresh( 1087 1087 + NOW, 1088 1088 + expect_active_fingerprint_absent=True, 1089 1089 + journal_path=tmp_path, 1090 1090 + ) 1091 1091 + 1092 1092 + assert permit is not None 1093 1093 + assert brain_fingerprint_key_path(journal_path=tmp_path).exists() 1094 1094 + record = finish_brain_refresh( 1095 1095 + permit, 1096 1096 + _ready_outcome(NOW + timedelta(seconds=1)), 1097 1097 + NOW + timedelta(seconds=1), 1098 1098 + journal_path=tmp_path, 1099 1099 + ) 1100 1100 + assert record["aggregate_state"] == "ready" 1101 1101 + assert record["active_lane"] == "spp" 1102 1102 + 1103 1103 + 1104 1104 + def test_begin_refresh_expected_absent_fingerprint_stales_when_key_exists( 1105 1105 + tmp_path: Path, 1106 1106 + ) -> None: 1107 1107 + _write_ready_record(tmp_path, _spp_config()) 1108 1108 + before_record = brain_state_path(journal_path=tmp_path).read_bytes() 1109 1109 + before_key = brain_fingerprint_key_path(journal_path=tmp_path).read_bytes() 1110 1110 + 1111 1111 + with pytest.raises(BrainStateExpectedFingerprintStaleError): 1112 1112 + begin_brain_refresh( 1113 1113 + NOW + timedelta(seconds=1), 1114 1114 + expect_active_fingerprint_absent=True, 1115 1115 + journal_path=tmp_path, 1116 1116 + ) 1117 1117 + 1118 1118 + assert brain_state_path(journal_path=tmp_path).read_bytes() == before_record 1119 1119 + assert brain_fingerprint_key_path(journal_path=tmp_path).read_bytes() == before_key 1120 1120 + 1121 1121 + 1122 1122 + def test_prerequisite_renewal_preserves_same_fingerprint_model_evidence( 1123 1123 + tmp_path: Path, 1124 1124 + ) -> None: 1125 1125 + config = _spp_config() 1126 1126 + _write_ready_record(tmp_path, config) 1127 1127 + before = _read_raw_record(tmp_path) 1128 1128 + expected = _current_fingerprint(tmp_path, config) 1129 1129 + begin = begin_brain_prerequisite_renewal( 1130 1130 + NOW + timedelta(minutes=1), 1131 1131 + expected_fingerprint_sha256=expected, 1132 1132 + journal_path=tmp_path, 1133 1133 + ) 1134 1134 + 1135 1135 + assert begin["status"] == "started" 1136 1136 + checking = _read_raw_record(tmp_path) 1137 1137 + assert checking["aggregate_state"] == "checking" 1138 1138 + assert checking["evidence"]["generate"] == before["evidence"]["generate"] 1139 1139 + assert checking["evidence"]["cogitate"] == before["evidence"]["cogitate"] 1140 1140 + 1141 1141 + finish_now = NOW + timedelta(minutes=2) 1142 1142 + record = finish_brain_prerequisite_renewal( 1143 1143 + begin["permit"], 1144 1144 + _component(finish_now, expires_at=finish_now + timedelta(minutes=10)), 1145 1145 + finish_now, 1146 1146 + journal_path=tmp_path, 1147 1147 + ) 1148 1148 + 1149 1149 + assert record["aggregate_state"] == "ready" 1150 1150 + assert record["revision"] == before["revision"] + 2 1151 1151 + assert record["evidence"]["configuration"] == before["evidence"]["configuration"] 1152 1152 + assert record["evidence"]["generate"] == before["evidence"]["generate"] 1153 1153 + assert record["evidence"]["cogitate"] == before["evidence"]["cogitate"] 1154 1154 + assert ( 1155 1155 + record["evidence"]["lane_prerequisites"]["observed_at"] 1156 1156 + == finish_now.isoformat() 1157 1157 + ) 1158 1158 + 1159 1159 + 1160 1160 + def test_prerequisite_renewal_refuses_expired_model_evidence(tmp_path: Path) -> None: 1161 1161 + config = _spp_config() 1162 1162 + _write_ready_record(tmp_path, config) 1163 1163 + raw = _read_raw_record(tmp_path) 1164 1164 + raw["evidence"]["generate"]["expires_at"] = (NOW - timedelta(seconds=1)).isoformat() 1165 1165 + atomic_replace(brain_state_path(journal_path=tmp_path), json.dumps(raw), mode=0o600) 1166 1166 + expected = _current_fingerprint(tmp_path, config) 1167 1167 + 1168 1168 + result = begin_brain_prerequisite_renewal( 1169 1169 + NOW, 1170 1170 + expected_fingerprint_sha256=expected, 1171 1171 + journal_path=tmp_path, 1172 1172 + ) 1173 1173 + 1174 1174 + assert result["status"] == "unsafe" 1175 1175 + assert _read_raw_record(tmp_path)["revision"] == raw["revision"] 1176 1176 + 1177 1177 + 1178 1178 + def test_prerequisite_renewal_reports_busy_when_refresh_lease_is_held( 1179 1179 + tmp_path: Path, 1180 1180 + ) -> None: 1181 1181 + config = _spp_config() 1182 1182 + _write_ready_record(tmp_path, config) 1183 1183 + holder = begin_brain_refresh(NOW + timedelta(seconds=1), journal_path=tmp_path) 1184 1184 + assert holder is not None 1185 1185 + try: 1186 1186 + result = begin_brain_prerequisite_renewal( 1187 1187 + NOW + timedelta(seconds=2), 1188 1188 + expected_fingerprint_sha256=_current_fingerprint(tmp_path, config), 1189 1189 + journal_path=tmp_path, 1190 1190 + ) 1191 1191 + finally: 1192 1192 + holder.release() 1193 1193 + 1194 1194 + assert result["status"] == "busy" 1195 1195 + 1196 1196 + 1197 1197 + def test_prerequisite_renewal_refuses_non_spp_lane(tmp_path: Path) -> None: 1198 1198 + config = _cloud_config() 1199 1199 + _write_ready_record(tmp_path, config) 1200 1200 + before = brain_state_path(journal_path=tmp_path).read_bytes() 1201 1201 + 1202 1202 + result = begin_brain_prerequisite_renewal( 1203 1203 + NOW + timedelta(seconds=1), 1204 1204 + expected_fingerprint_sha256=_current_fingerprint(tmp_path, config), 1205 1205 + journal_path=tmp_path, 1206 1206 + ) 1207 1207 + 1208 1208 + assert result["status"] == "unsafe" 1209 1209 + assert brain_state_path(journal_path=tmp_path).read_bytes() == before 1210 1210 + 1211 1211 + 1212 1212 + @pytest.mark.parametrize( 1213 1213 + "mutation", 1214 1214 + ( 1215 1215 + lambda raw: raw["evidence"].__setitem__("generate", None), 1216 1216 + lambda raw: raw["evidence"]["generate"].__setitem__("status", "failed"), 1217 1217 + lambda raw: raw["evidence"]["generate"].__setitem__("expires_at", "not-time"), 1218 1218 + ), 1219 1219 + ) 1220 1220 + def test_prerequisite_renewal_refuses_missing_malformed_or_non_ok_model_evidence( 1221 1221 + tmp_path: Path, 1222 1222 + mutation, 1223 1223 + ) -> None: 1224 1224 + config = _spp_config() 1225 1225 + _write_ready_record(tmp_path, config) 1226 1226 + raw = _read_raw_record(tmp_path) 1227 1227 + mutation(raw) 1228 1228 + atomic_replace(brain_state_path(journal_path=tmp_path), json.dumps(raw), mode=0o600) 1229 1229 + before = brain_state_path(journal_path=tmp_path).read_bytes() 1230 1230 + 1231 1231 + result = begin_brain_prerequisite_renewal( 1232 1232 + NOW + timedelta(seconds=1), 1233 1233 + expected_fingerprint_sha256=_current_fingerprint(tmp_path, config), 1234 1234 + journal_path=tmp_path, 1235 1235 + ) 1236 1236 + 1237 1237 + assert result["status"] == "unsafe" 1238 1238 + assert brain_state_path(journal_path=tmp_path).read_bytes() == before 1239 1239 + 1240 1240 + 1241 1241 + def test_prerequisite_renewal_expected_fingerprint_mismatch_is_no_write( 1242 1242 + tmp_path: Path, 1243 1243 + ) -> None: 1244 1244 + config = _spp_config() 1245 1245 + _write_ready_record(tmp_path, config) 1246 1246 + before = brain_state_path(journal_path=tmp_path).read_bytes() 1247 1247 + 1248 1248 + result = begin_brain_prerequisite_renewal( 1249 1249 + NOW + timedelta(seconds=1), 1250 1250 + expected_fingerprint_sha256="0" * 64, 1251 1251 + journal_path=tmp_path, 1252 1252 + ) 1253 1253 + 1254 1254 + assert result["status"] == "unsafe" 1255 1255 + assert result["reason"] == "fingerprint_mismatch" 1256 1256 + assert brain_state_path(journal_path=tmp_path).read_bytes() == before 1257 1257 + 1258 1258 + 1259 1259 + def test_prerequisite_renewal_recovers_orphaned_checking_after_lease_released( 1260 1260 + tmp_path: Path, 1261 1261 + ) -> None: 1262 1262 + config = _spp_config() 1263 1263 + _write_ready_record(tmp_path, config) 1264 1264 + expected = _current_fingerprint(tmp_path, config) 1265 1265 + first = begin_brain_prerequisite_renewal( 1266 1266 + NOW + timedelta(seconds=1), 1267 1267 + expected_fingerprint_sha256=expected, 1268 1268 + journal_path=tmp_path, 1269 1269 + ) 1270 1270 + assert first["status"] == "started" 1271 1271 + first["permit"].release() 1272 1272 + orphaned = _read_raw_record(tmp_path) 1273 1273 + assert orphaned["aggregate_state"] == "checking" 1274 1274 + 1275 1275 + second = begin_brain_prerequisite_renewal( 1276 1276 + NOW + timedelta(seconds=2), 1277 1277 + expected_fingerprint_sha256=expected, 1278 1278 + journal_path=tmp_path, 1279 1279 + ) 1280 1280 + assert second["status"] == "started" 1281 1281 + recovered = _read_raw_record(tmp_path) 1282 1282 + assert recovered["revision"] == orphaned["revision"] + 1 1283 1283 + assert recovered["checking"]["run_id"] != orphaned["checking"]["run_id"] 1284 1284 + second["permit"].release() 1285 1285 + 1286 1286 + 1287 1287 + def test_prerequisite_renewal_expired_permit_conflicts_and_releases( 1288 1288 + tmp_path: Path, 1289 1289 + ) -> None: 1290 1290 + config = _spp_config() 1291 1291 + _write_ready_record(tmp_path, config) 1292 1292 + expected = _current_fingerprint(tmp_path, config) 1293 1293 + begin = begin_brain_prerequisite_renewal( 1294 1294 + NOW, 1295 1295 + expected_fingerprint_sha256=expected, 1296 1296 + journal_path=tmp_path, 1297 1297 + ) 1298 1298 + assert begin["status"] == "started" 1299 1299 + permit = begin["permit"] 1300 1300 + 1301 1301 + with pytest.raises(BrainStateConflictError): 1302 1302 + finish_brain_prerequisite_renewal( 1303 1303 + permit, 1304 1304 + _component( 1305 1305 + permit.expires_at, expires_at=permit.expires_at + timedelta(minutes=10) 1306 1306 + ), 1307 1307 + permit.expires_at, 1308 1308 + journal_path=tmp_path, 1309 1309 + ) 1310 1310 + 1311 1311 + retry = begin_brain_prerequisite_renewal( 1312 1312 + NOW + timedelta(seconds=1), 1313 1313 + expected_fingerprint_sha256=expected, 1314 1314 + journal_path=tmp_path, 1315 1315 + ) 1316 1316 + assert retry["status"] == "started" 1317 1317 + retry["permit"].release() 1318 1318 + 1319 1319 + 1320 1320 + def test_prerequisite_renewal_conflicts_on_revision_drift_and_releases( 1321 1321 + tmp_path: Path, 1322 1322 + ) -> None: 1323 1323 + config = _spp_config() 1324 1324 + _write_ready_record(tmp_path, config) 1325 1325 + expected = _current_fingerprint(tmp_path, config) 1326 1326 + begin = begin_brain_prerequisite_renewal( 1327 1327 + NOW, 1328 1328 + expected_fingerprint_sha256=expected, 1329 1329 + journal_path=tmp_path, 1330 1330 + ) 1331 1331 + assert begin["status"] == "started" 1332 1332 + raw = _read_raw_record(tmp_path) 1333 1333 + raw["revision"] += 1 1334 1334 + raw["checking"]["checking_revision"] = raw["revision"] 1335 1335 + atomic_replace(brain_state_path(journal_path=tmp_path), json.dumps(raw), mode=0o600) 1336 1336 + 1337 1337 + with pytest.raises(BrainStateConflictError): 1338 1338 + finish_brain_prerequisite_renewal( 1339 1339 + begin["permit"], 1340 1340 + _component(NOW + timedelta(seconds=1)), 1341 1341 + NOW + timedelta(seconds=1), 1342 1342 + journal_path=tmp_path, 1343 1343 + ) 1344 1344 + 1345 1345 + retry = begin_brain_prerequisite_renewal( 1346 1346 + NOW + timedelta(seconds=2), 1347 1347 + expected_fingerprint_sha256=expected, 1348 1348 + journal_path=tmp_path, 1349 1349 + ) 1350 1350 + assert retry["status"] == "started" 1351 1351 + retry["permit"].release() 1352 1352 + 1353 1353 + 1354 1354 + def test_prerequisite_renewal_conflicts_on_runtime_marker_drift_and_releases( 1355 1355 + tmp_path: Path, 1356 1356 + ) -> None: 1357 1357 + config = _spp_config() 1358 1358 + _write_ready_record(tmp_path, config) 1359 1359 + expected = _current_fingerprint(tmp_path, config) 1360 1360 + begin = begin_brain_prerequisite_renewal( 1361 1361 + NOW, 1362 1362 + expected_fingerprint_sha256=expected, 1363 1363 + journal_path=tmp_path, 1364 1364 + ) 1365 1365 + assert begin["status"] == "started" 1366 1366 + raw = _read_raw_record(tmp_path) 1367 1367 + raw["checking"]["runtime_failure_marker_seen"] = "changed-marker" 1368 1368 + atomic_replace(brain_state_path(journal_path=tmp_path), json.dumps(raw), mode=0o600) 1369 1369 + 1370 1370 + with pytest.raises(BrainStateConflictError): 1371 1371 + finish_brain_prerequisite_renewal( 1372 1372 + begin["permit"], 1373 1373 + _component(NOW + timedelta(seconds=1)), 1374 1374 + NOW + timedelta(seconds=1), 1375 1375 + journal_path=tmp_path, 1376 1376 + ) 1377 1377 + 1378 1378 + retry = begin_brain_prerequisite_renewal( 1379 1379 + NOW + timedelta(seconds=2), 1380 1380 + expected_fingerprint_sha256=expected, 1381 1381 + journal_path=tmp_path, 1382 1382 + ) 1383 1383 + assert retry["status"] == "started" 1384 1384 + retry["permit"].release() 1385 1385 + 1386 1386 + 1387 1387 + def test_prerequisite_renewal_conflicts_on_fingerprint_drift(tmp_path: Path) -> None: 1388 1388 + config = _spp_config(account_id="acct-a") 1389 1389 + _write_ready_record(tmp_path, config) 1390 1390 + expected = _current_fingerprint(tmp_path, config) 1391 1391 + begin = begin_brain_prerequisite_renewal( 1392 1392 + NOW, 1393 1393 + expected_fingerprint_sha256=expected, 1394 1394 + journal_path=tmp_path, 1395 1395 + ) 1396 1396 + assert begin["status"] == "started" 1397 1397 + _write_config(tmp_path, _spp_config(account_id="acct-b")) 1398 1398 + 1399 1399 + with pytest.raises(BrainStateConflictError): 1400 1400 + finish_brain_prerequisite_renewal( 1401 1401 + begin["permit"], 1402 1402 + _component(NOW, expires_at=NOW + timedelta(minutes=10)), 1403 1403 + NOW, 1404 1404 + journal_path=tmp_path, 1405 1405 + ) 1406 1406 + 1407 1407 + retry = begin_brain_prerequisite_renewal( 1408 1408 + NOW + timedelta(seconds=1), 1409 1409 + expected_fingerprint_sha256=_current_fingerprint( 1410 1410 + tmp_path, _spp_config(account_id="acct-b") 1411 1411 + ), 1412 1412 + journal_path=tmp_path, 1413 1413 + ) 1414 1414 + assert retry["status"] == "unsafe" 1415 1415 + 1416 1416 + 1417 1417 + def test_prerequisite_renewal_abandon_records_failure_without_losing_model_evidence( 1418 1418 + tmp_path: Path, 1419 1419 + ) -> None: 1420 1420 + config = _spp_config() 1421 1421 + _write_ready_record(tmp_path, config) 1422 1422 + before = _read_raw_record(tmp_path) 1423 1423 + begin = begin_brain_prerequisite_renewal( 1424 1424 + NOW, 1425 1425 + expected_fingerprint_sha256=_current_fingerprint(tmp_path, config), 1426 1426 + journal_path=tmp_path, 1427 1427 + ) 1428 1428 + assert begin["status"] == "started" 1429 1429 + 1430 1430 + record = abandon_brain_prerequisite_renewal( 1431 1431 + begin["permit"], 1432 1432 + "probe_internal_error", 1433 1433 + NOW + timedelta(seconds=1), 1434 1434 + journal_path=tmp_path, 1435 1435 + ) 1436 1436 + 1437 1437 + assert record["reason_code"] == "probe_internal_error" 1438 1438 + assert record["evidence"]["lane_prerequisites"]["reason_code"] == ( 1439 1439 + "probe_internal_error" 1440 1440 + ) 1441 1441 + assert record["evidence"]["generate"] == before["evidence"]["generate"] 1442 1442 + assert record["evidence"]["cogitate"] == before["evidence"]["cogitate"] 1053 1443 1054 1444 1055 1445 def test_key_replacement_invalidates_prior_ready_record(tmp_path: Path) -> None:

+142 -2

tests/test_brain_state_multiprocess.py

View file Reviewed

··· 3 3 4 4 from __future__ import annotations 5 5 6 6 + import hashlib 6 7 import json 7 8 import os 8 9 import subprocess 9 10 import sys 10 11 import time 11 11 - from datetime import datetime, timezone 12 12 + from datetime import datetime, timedelta, timezone 12 13 from pathlib import Path 13 14 14 14 - from solstone.think.providers.brain_state import inspect_brain_state 15 15 + from solstone.think.models import LOCAL_MODEL 16 16 + from solstone.think.providers.brain_state import ( 17 17 + DEFAULT_READY_EVIDENCE_TTL, 18 18 + begin_brain_refresh, 19 19 + finish_brain_refresh, 20 20 + inspect_brain_state, 21 21 + ) 15 22 16 23 NOW = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc) 17 24 ··· 36 43 ) 37 44 38 45 46 46 + def _write_spp_config(journal: Path) -> None: 47 47 + credential = "endpoint-secret" 48 48 + path = journal / "config" / "journal.json" 49 49 + path.parent.mkdir(parents=True, exist_ok=True) 50 50 + path.write_text( 51 51 + json.dumps( 52 52 + { 53 53 + "providers": { 54 54 + "active": {"provider": "local", "model": LOCAL_MODEL}, 55 55 + "local": { 56 56 + "endpoint_url": "https://brain.example.test/v1", 57 57 + "served_model_id": "served-model", 58 58 + "credential": credential, 59 59 + }, 60 60 + }, 61 61 + "services": { 62 62 + "confidential": { 63 63 + "enabled_at": NOW.isoformat(), 64 64 + "account_id": "acct-a", 65 65 + "endpoint_url": "https://brain.example.test/v1", 66 66 + "served_model_id": "served-model", 67 67 + "credential_created_at": NOW.isoformat(), 68 68 + "credential_fingerprint_sha256": hashlib.sha256( 69 69 + credential.encode("utf-8") 70 70 + ).hexdigest(), 71 71 + "prior_active": { 72 72 + "provider": "google", 73 73 + "model": "gemini-flash-latest", 74 74 + }, 75 75 + "prior_local_endpoint": None, 76 76 + } 77 77 + }, 78 78 + "env": {}, 79 79 + } 80 80 + ), 81 81 + encoding="utf-8", 82 82 + ) 83 83 + 84 84 + 85 85 + def _component(now: datetime) -> dict[str, str]: 86 86 + return { 87 87 + "status": "ok", 88 88 + "observed_at": now.isoformat(), 89 89 + "expires_at": (now + DEFAULT_READY_EVIDENCE_TTL).isoformat(), 90 90 + } 91 91 + 92 92 + 93 93 + def _write_spp_ready_record(journal: Path) -> None: 94 94 + _write_spp_config(journal) 95 95 + permit = begin_brain_refresh(NOW, journal_path=journal) 96 96 + assert permit is not None 97 97 + finish_brain_refresh( 98 98 + permit, 99 99 + { 100 100 + "configuration": _component(NOW), 101 101 + "lane_prerequisites": _component(NOW), 102 102 + "generate": _component(NOW), 103 103 + "cogitate": _component(NOW), 104 104 + }, 105 105 + NOW, 106 106 + journal_path=journal, 107 107 + ) 108 108 + 109 109 + 39 110 def test_refresh_permit_excludes_contender_and_crash_releases(tmp_path: Path) -> None: 40 111 _write_config(tmp_path) 41 112 ready = tmp_path / "ready" ··· 107 178 if holder.poll() is None: 108 179 holder.terminate() 109 180 holder.wait(timeout=5) 181 181 + 182 182 + 183 183 + def test_prerequisite_renewal_permit_excludes_contender_and_crash_releases( 184 184 + tmp_path: Path, 185 185 + ) -> None: 186 186 + _write_spp_ready_record(tmp_path) 187 187 + ready = tmp_path / "ready" 188 188 + holder_code = f""" 189 189 + import pathlib 190 190 + import time 191 191 + from datetime import datetime 192 192 + from solstone.think.providers.brain_state import begin_brain_prerequisite_renewal 193 193 + now = datetime.fromisoformat({(NOW + timedelta(seconds=1)).isoformat()!r}) 194 194 + result = begin_brain_prerequisite_renewal(now, journal_path={str(tmp_path)!r}) 195 195 + assert result["status"] == "started", result 196 196 + pathlib.Path({str(ready)!r}).write_text("ready") 197 197 + while True: 198 198 + time.sleep(0.05) 199 199 + """ 200 200 + contender_code = f""" 201 201 + from datetime import datetime 202 202 + from solstone.think.providers.brain_state import begin_brain_prerequisite_renewal 203 203 + now = datetime.fromisoformat({(NOW + timedelta(seconds=2)).isoformat()!r}) 204 204 + result = begin_brain_prerequisite_renewal(now, journal_path={str(tmp_path)!r}) 205 205 + print(result["status"], flush=True) 206 206 + if result["status"] == "started": 207 207 + result["permit"].release() 208 208 + """ 209 209 + holder = subprocess.Popen( 210 210 + [sys.executable, "-c", holder_code], 211 211 + cwd=Path.cwd(), 212 212 + env=_env(tmp_path), 213 213 + stdout=subprocess.PIPE, 214 214 + stderr=subprocess.PIPE, 215 215 + text=True, 216 216 + ) 217 217 + try: 218 218 + deadline = time.monotonic() + 10 219 219 + while not ready.exists() and time.monotonic() < deadline: 220 220 + time.sleep(0.05) 221 221 + assert ready.exists() 222 222 + 223 223 + busy = subprocess.run( 224 224 + [sys.executable, "-c", contender_code], 225 225 + cwd=Path.cwd(), 226 226 + env=_env(tmp_path), 227 227 + capture_output=True, 228 228 + text=True, 229 229 + check=True, 230 230 + ) 231 231 + assert busy.stdout.strip() == "busy" 232 232 + 233 233 + holder.terminate() 234 234 + stdout, stderr = holder.communicate(timeout=10) 235 235 + assert holder.returncode is not None, (stdout, stderr) 236 236 + 237 237 + free = subprocess.run( 238 238 + [sys.executable, "-c", contender_code], 239 239 + cwd=Path.cwd(), 240 240 + env=_env(tmp_path), 241 241 + capture_output=True, 242 242 + text=True, 243 243 + check=True, 244 244 + ) 245 245 + assert free.stdout.strip() == "started" 246 246 + finally: 247 247 + if holder.poll() is None: 248 248 + holder.terminate() 249 249 + holder.wait(timeout=5)

+998

tests/test_cortex.py

View file Reviewed

··· 10 10 import sys 11 11 import threading 12 12 import time 13 13 + from datetime import datetime, timedelta, timezone 13 14 from pathlib import Path 14 15 from unittest.mock import MagicMock, patch 15 16 ··· 68 69 return journal_path / "talents" / name.replace(":", "--") / f"{use_id}.jsonl" 69 70 70 71 72 72 + class _FakeClock: 73 73 + def __init__(self, now: datetime): 74 74 + self.now = now 75 75 + self.waits: list[float] = [] 76 76 + 77 77 + def __call__(self) -> datetime: 78 78 + return self.now 79 79 + 80 80 + def advance(self, seconds: float) -> None: 81 81 + self.now += timedelta(seconds=seconds) 82 82 + 83 83 + def wait(self, seconds: float) -> bool: 84 84 + self.waits.append(seconds) 85 85 + self.advance(seconds) 86 86 + return False 87 87 + 88 88 + 89 89 + class _FakeCallosum: 90 90 + def __init__(self) -> None: 91 91 + self.emitted: list[tuple[tuple, dict]] = [] 92 92 + 93 93 + def emit(self, *args, **kwargs) -> None: 94 94 + self.emitted.append((args, kwargs)) 95 95 + 96 96 + 97 97 + def _spp_inspector(state: dict): 98 98 + def inspect(now: datetime, *, journal_path=None): 99 99 + lane = state.get("lane", "spp") 100 100 + aggregate = state.get("aggregate", "ready") 101 101 + fingerprint = state.get("fingerprint", "a" * 64) 102 102 + record_fingerprint = state.get("record_fingerprint", fingerprint) 103 103 + observed = state.get("observed", now) 104 104 + expires = state.get("expires", now + timedelta(minutes=10)) 105 105 + record = None 106 106 + if lane == "spp" and state.get("record_present", True): 107 107 + component = None 108 108 + if state.get("component_present", True): 109 109 + component = { 110 110 + "status": state.get("component_status", "ok"), 111 111 + "observed_at": observed.isoformat(), 112 112 + "expires_at": expires.isoformat(), 113 113 + } 114 114 + reason = state.get("component_reason") 115 115 + if reason is not None: 116 116 + component["reason_code"] = reason 117 117 + record = { 118 118 + "active_lane": "spp", 119 119 + "fingerprint_sha256": record_fingerprint, 120 120 + "evidence": {"lane_prerequisites": component}, 121 121 + } 122 122 + return { 123 123 + "status": "ok", 124 124 + "path": "/redacted", 125 125 + "record": record, 126 126 + "projection": { 127 127 + "aggregate_state": aggregate, 128 128 + "active_lane": lane, 129 129 + "fingerprint_sha256": fingerprint, 130 130 + "runtime_transition_in_progress": False, 131 131 + }, 132 132 + "reason_code": None, 133 133 + "error": None, 134 134 + } 135 135 + 136 136 + return inspect 137 137 + 138 138 + 139 139 + def _make_spp_controller( 140 140 + tmp_path: Path, 141 141 + monkeypatch: pytest.MonkeyPatch, 142 142 + *, 143 143 + state: dict, 144 144 + clock: _FakeClock | None = None, 145 145 + logger: MagicMock | None = None, 146 146 + callosum: _FakeCallosum | None = None, 147 147 + ): 148 148 + from solstone.think import cortex 149 149 + 150 150 + clock = clock or _FakeClock(datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc)) 151 151 + callosum = callosum or _FakeCallosum() 152 152 + logger = logger or MagicMock() 153 153 + monkeypatch.setattr(cortex, "inspect_brain_state", _spp_inspector(state)) 154 154 + monkeypatch.setattr( 155 155 + cortex, 156 156 + "read_active_brain_fingerprint_sha256", 157 157 + lambda *, journal_path=None: state.get("fingerprint", "a" * 64), 158 158 + ) 159 159 + controller = cortex.SppRenewalController( 160 160 + callosum=callosum, 161 161 + stop_event=threading.Event(), 162 162 + logger=logger, 163 163 + clock=clock, 164 164 + wait=clock.wait, 165 165 + journal_path=tmp_path, 166 166 + ) 167 167 + return controller, clock, callosum, logger 168 168 + 169 169 + 170 170 + def _commands(callosum: _FakeCallosum) -> list[list[str]]: 171 171 + return [kwargs["cmd"] for _args, kwargs in callosum.emitted] 172 172 + 173 173 + 174 174 + def _assert_fenced_refresh_command(command: list[str], fingerprint: str) -> None: 175 175 + assert command[:4] == ["journal", "brain", "refresh", "--json"] 176 176 + assert "--expected-active-fingerprint" in command 177 177 + expected_index = command.index("--expected-fingerprint") 178 178 + assert command[expected_index + 1] == fingerprint 179 179 + 180 180 + 181 181 + def _assert_absence_fenced_refresh_command(command: list[str]) -> None: 182 182 + assert command == [ 183 183 + "journal", 184 184 + "brain", 185 185 + "refresh", 186 186 + "--json", 187 187 + "--expect-active-fingerprint-absent", 188 188 + ] 189 189 + 190 190 + 71 191 @pytest.fixture 72 192 def mock_journal(tmp_path, monkeypatch): 73 193 """Set up a temporary journal directory.""" ··· 173 293 if cortex_service._spawn_worker is not None: 174 294 cortex_service._spawn_worker.join(timeout=1) 175 295 service_thread.join(timeout=1) 296 296 + 297 297 + 298 298 + def test_spp_renewal_controller_replaces_prerequisites_for_seventy_virtual_minutes( 299 299 + tmp_path, 300 300 + monkeypatch, 301 301 + ): 302 302 + from solstone.think import cortex 303 303 + 304 304 + start = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 305 305 + clock = _FakeClock(start) 306 306 + state = { 307 307 + "fingerprint": "a" * 64, 308 308 + "observed": start, 309 309 + "expires": start + timedelta(minutes=10), 310 310 + } 311 311 + monkeypatch.setattr(cortex, "inspect_brain_state", _spp_inspector(state)) 312 312 + monkeypatch.setattr( 313 313 + cortex, 314 314 + "read_active_brain_fingerprint_sha256", 315 315 + lambda *, journal_path=None: state["fingerprint"], 316 316 + ) 317 317 + callosum = _FakeCallosum() 318 318 + controller = cortex.SppRenewalController( 319 319 + callosum=callosum, 320 320 + stop_event=threading.Event(), 321 321 + logger=MagicMock(), 322 322 + clock=clock, 323 323 + wait=clock.wait, 324 324 + journal_path=tmp_path, 325 325 + ) 326 326 + 327 327 + committed: list[tuple[datetime, datetime]] = [] 328 328 + while clock.now < start + timedelta(minutes=72): 329 329 + delay = controller.step() 330 330 + if delay > 0: 331 331 + clock.advance(delay) 332 332 + assert state["expires"] > clock.now 333 333 + controller.step() 334 334 + ref = callosum.emitted[-1][1]["ref"] 335 335 + controller.handle_supervisor_message( 336 336 + {"tract": "supervisor", "event": "started", "ref": ref} 337 337 + ) 338 338 + state["observed"] = clock.now 339 339 + state["expires"] = clock.now + timedelta(minutes=10) 340 340 + committed.append((state["observed"], state["expires"])) 341 341 + controller.handle_supervisor_message( 342 342 + {"tract": "supervisor", "event": "stopped", "ref": ref, "exit_code": 0} 343 343 + ) 344 344 + 345 345 + commands = _commands(callosum) 346 346 + assert commands 347 347 + assert len(committed) >= 9 348 348 + assert clock.now >= start + timedelta(minutes=72) 349 349 + assert start + timedelta(minutes=72) > start + timedelta(minutes=60) 350 350 + assert any(observed >= start + timedelta(minutes=30) for observed, _ in committed) 351 351 + assert any(observed >= start + timedelta(minutes=60) for observed, _ in committed) 352 352 + assert all( 353 353 + later[0] > earlier[0] and later[1] > earlier[1] 354 354 + for earlier, later in zip(committed, committed[1:]) 355 355 + ) 356 356 + assert all( 357 357 + command[:3] == ["journal", "brain", "renew-prerequisites"] 358 358 + for command in commands 359 359 + ) 360 360 + assert not any( 361 361 + command[:3] == ["journal", "brain", "refresh"] for command in commands 362 362 + ) 363 363 + assert not any( 364 364 + "generate" in command or "cogitate" in command for command in commands 365 365 + ) 366 366 + 367 367 + 368 368 + def test_spp_renewal_controller_tracks_skipped_active_ref_successor( 369 369 + tmp_path, 370 370 + monkeypatch, 371 371 + ): 372 372 + from solstone.think import cortex 373 373 + 374 374 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 375 375 + clock = _FakeClock(now) 376 376 + state = { 377 377 + "fingerprint": "b" * 64, 378 378 + "observed": now - timedelta(minutes=9), 379 379 + "expires": now + timedelta(seconds=30), 380 380 + } 381 381 + monkeypatch.setattr(cortex, "inspect_brain_state", _spp_inspector(state)) 382 382 + monkeypatch.setattr( 383 383 + cortex, 384 384 + "read_active_brain_fingerprint_sha256", 385 385 + lambda *, journal_path=None: state["fingerprint"], 386 386 + ) 387 387 + callosum = _FakeCallosum() 388 388 + controller = cortex.SppRenewalController( 389 389 + callosum=callosum, 390 390 + stop_event=threading.Event(), 391 391 + logger=MagicMock(), 392 392 + clock=clock, 393 393 + wait=clock.wait, 394 394 + journal_path=tmp_path, 395 395 + ) 396 396 + 397 397 + controller.step() 398 398 + first_ref = callosum.emitted[-1][1]["ref"] 399 399 + controller.handle_supervisor_message( 400 400 + { 401 401 + "tract": "supervisor", 402 402 + "event": "skipped", 403 403 + "ref": first_ref, 404 404 + "active_ref": "active-brain", 405 405 + "reason": "still_running", 406 406 + } 407 407 + ) 408 408 + controller.step() 409 409 + assert len(callosum.emitted) == 1 410 410 + 411 411 + controller.handle_supervisor_message( 412 412 + { 413 413 + "tract": "supervisor", 414 414 + "event": "stopped", 415 415 + "ref": "active-brain", 416 416 + "exit_code": 0, 417 417 + } 418 418 + ) 419 419 + controller.step() 420 420 + assert len(callosum.emitted) == 2 421 421 + assert callosum.emitted[-1][1]["ref"] != first_ref 422 422 + 423 423 + 424 424 + def test_spp_renewal_controller_times_out_missed_successor_stop( 425 425 + tmp_path, 426 426 + monkeypatch, 427 427 + ): 428 428 + from solstone.think import cortex 429 429 + 430 430 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 431 431 + state = { 432 432 + "fingerprint": "b" * 64, 433 433 + "observed": now - timedelta(minutes=9), 434 434 + "expires": now + timedelta(seconds=30), 435 435 + } 436 436 + controller, clock, callosum, _logger = _make_spp_controller( 437 437 + tmp_path, 438 438 + monkeypatch, 439 439 + state=state, 440 440 + clock=_FakeClock(now), 441 441 + ) 442 442 + 443 443 + controller.step() 444 444 + first_ref = callosum.emitted[-1][1]["ref"] 445 445 + controller.handle_supervisor_message( 446 446 + { 447 447 + "tract": "supervisor", 448 448 + "event": "skipped", 449 449 + "ref": first_ref, 450 450 + "active_ref": "active-brain", 451 451 + "reason": "still_running", 452 452 + } 453 453 + ) 454 454 + clock.advance(cortex.SPP_REFRESH_OBSERVATION_BOUND_S + 1) 455 455 + 456 456 + controller.step() 457 457 + 458 458 + assert controller._successor_after_ref is None 459 459 + assert len(callosum.emitted) == 2 460 460 + assert callosum.emitted[-1][1]["ref"] != first_ref 461 461 + 462 462 + 463 463 + def test_spp_renewal_controller_accepts_full_refresh_exit_zero_without_retry( 464 464 + tmp_path, 465 465 + monkeypatch, 466 466 + ): 467 467 + from solstone.think import cortex 468 468 + 469 469 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 470 470 + state = { 471 471 + "aggregate": "unknown", 472 472 + "fingerprint": "e" * 64, 473 473 + "observed": now - timedelta(minutes=9), 474 474 + "expires": now + timedelta(seconds=30), 475 475 + } 476 476 + controller, clock, callosum, _logger = _make_spp_controller( 477 477 + tmp_path, 478 478 + monkeypatch, 479 479 + state=state, 480 480 + clock=_FakeClock(now), 481 481 + ) 482 482 + 483 483 + controller.step() 484 484 + _assert_fenced_refresh_command(callosum.emitted[-1][1]["cmd"], state["fingerprint"]) 485 485 + ref = callosum.emitted[-1][1]["ref"] 486 486 + controller.handle_supervisor_message( 487 487 + {"tract": "supervisor", "event": "started", "ref": ref} 488 488 + ) 489 489 + assert controller._running_deadline is not None 490 490 + assert ( 491 491 + controller._running_deadline - clock.now 492 492 + ).total_seconds() == cortex.SPP_REFRESH_OBSERVATION_BOUND_S 493 493 + 494 494 + clock.advance(1) 495 495 + state["aggregate"] = "ready" 496 496 + state["observed"] = clock.now 497 497 + state["expires"] = clock.now + timedelta(minutes=10) 498 498 + controller.handle_supervisor_message( 499 499 + {"tract": "supervisor", "event": "stopped", "ref": ref, "exit_code": 0} 500 500 + ) 501 501 + 502 502 + assert controller._retry_after is None 503 503 + assert controller._running_ref is None 504 504 + delay = controller.step() 505 505 + assert len(callosum.emitted) == 1 506 506 + assert delay > 0 507 507 + 508 508 + 509 509 + @pytest.mark.parametrize("case", ("unchanged", "unhealthy", "wrong_fingerprint")) 510 510 + def test_spp_renewal_controller_rejects_exit_zero_without_new_ready_refresh_proof( 511 511 + tmp_path, 512 512 + monkeypatch, 513 513 + case, 514 514 + ): 515 515 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 516 516 + original_fingerprint = "e" * 64 517 517 + state = { 518 518 + "aggregate": "unknown", 519 519 + "fingerprint": original_fingerprint, 520 520 + "observed": now - timedelta(minutes=9), 521 521 + "expires": now + timedelta(seconds=30), 522 522 + } 523 523 + controller, clock, callosum, logger = _make_spp_controller( 524 524 + tmp_path, 525 525 + monkeypatch, 526 526 + state=state, 527 527 + clock=_FakeClock(now), 528 528 + ) 529 529 + 530 530 + controller.step() 531 531 + _assert_fenced_refresh_command(callosum.emitted[-1][1]["cmd"], original_fingerprint) 532 532 + ref = callosum.emitted[-1][1]["ref"] 533 533 + controller.handle_supervisor_message( 534 534 + {"tract": "supervisor", "event": "started", "ref": ref} 535 535 + ) 536 536 + 537 537 + clock.advance(1) 538 538 + if case == "unchanged": 539 539 + state["aggregate"] = "ready" 540 540 + elif case == "unhealthy": 541 541 + state["aggregate"] = "unhealthy" 542 542 + state["component_status"] = "failed" 543 543 + state["component_reason"] = "attestation_rejected" 544 544 + state["observed"] = clock.now 545 545 + state["expires"] = clock.now + timedelta(minutes=10) 546 546 + else: 547 547 + state["aggregate"] = "ready" 548 548 + state["fingerprint"] = "f" * 64 549 549 + state["record_fingerprint"] = "f" * 64 550 550 + state["observed"] = clock.now 551 551 + state["expires"] = clock.now + timedelta(minutes=10) 552 552 + 553 553 + controller.handle_supervisor_message( 554 554 + {"tract": "supervisor", "event": "stopped", "ref": ref, "exit_code": 0} 555 555 + ) 556 556 + 557 557 + assert controller._retry_after is not None 558 558 + logged = "\n".join( 559 559 + " ".join(str(part) for part in call.args) for call in logger.info.call_args_list 560 560 + ) 561 561 + assert "verified" not in logged 562 562 + assert "failed" in logged 563 563 + 564 564 + 565 565 + def test_spp_renewal_controller_running_timeout_clears_and_retries( 566 566 + tmp_path, 567 567 + monkeypatch, 568 568 + ): 569 569 + from solstone.think import cortex 570 570 + 571 571 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 572 572 + state = { 573 573 + "fingerprint": "f" * 64, 574 574 + "observed": now - timedelta(minutes=9), 575 575 + "expires": now + timedelta(seconds=30), 576 576 + } 577 577 + controller, clock, callosum, _logger = _make_spp_controller( 578 578 + tmp_path, 579 579 + monkeypatch, 580 580 + state=state, 581 581 + clock=_FakeClock(now), 582 582 + ) 583 583 + 584 584 + controller.step() 585 585 + ref = callosum.emitted[-1][1]["ref"] 586 586 + controller.handle_supervisor_message( 587 587 + {"tract": "supervisor", "event": "started", "ref": ref} 588 588 + ) 589 589 + assert controller._running_deadline is not None 590 590 + assert ( 591 591 + controller._running_deadline - clock.now 592 592 + ).total_seconds() == cortex.SPP_RENEWAL_ATTEMPT_BOUND_S 593 593 + clock.advance(cortex.SPP_RENEWAL_ATTEMPT_BOUND_S + 1) 594 594 + delay = controller.step() 595 595 + 596 596 + assert controller._running_ref is None 597 597 + assert controller._retry_after is not None 598 598 + assert delay == 5.0 599 599 + 600 600 + clock.advance(delay) 601 601 + controller.step() 602 602 + assert len(callosum.emitted) == 2 603 603 + assert callosum.emitted[-1][1]["ref"] != ref 604 604 + 605 605 + 606 606 + @pytest.mark.parametrize( 607 607 + "state_update", 608 608 + ( 609 609 + {"record_present": False}, 610 610 + {"component_status": "failed", "component_reason": "attestation_not_verified"}, 611 611 + {"record_fingerprint": "0" * 64}, 612 612 + {"expires": datetime(2026, 7, 24, 11, 59, tzinfo=timezone.utc)}, 613 613 + ), 614 614 + ) 615 615 + def test_spp_renewal_controller_unsafe_spp_records_fall_back_to_full_refresh( 616 616 + tmp_path, 617 617 + monkeypatch, 618 618 + state_update, 619 619 + ): 620 620 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 621 621 + state = { 622 622 + "fingerprint": "9" * 64, 623 623 + "observed": now - timedelta(minutes=9), 624 624 + "expires": now + timedelta(seconds=30), 625 625 + **state_update, 626 626 + } 627 627 + _controller, _clock, callosum, _logger = _make_spp_controller( 628 628 + tmp_path, 629 629 + monkeypatch, 630 630 + state=state, 631 631 + clock=_FakeClock(now), 632 632 + ) 633 633 + 634 634 + _controller.step() 635 635 + 636 636 + _assert_fenced_refresh_command(callosum.emitted[-1][1]["cmd"], state["fingerprint"]) 637 637 + 638 638 + 639 639 + def test_spp_renewal_controller_bootstraps_with_absence_fenced_refresh( 640 640 + tmp_path, 641 641 + monkeypatch, 642 642 + ): 643 643 + from solstone.think import cortex 644 644 + 645 645 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 646 646 + state = { 647 647 + "aggregate": "unknown", 648 648 + "fingerprint": None, 649 649 + "observed": now - timedelta(minutes=9), 650 650 + "expires": now + timedelta(seconds=30), 651 651 + } 652 652 + controller, clock, callosum, logger = _make_spp_controller( 653 653 + tmp_path, 654 654 + monkeypatch, 655 655 + state=state, 656 656 + clock=_FakeClock(now), 657 657 + ) 658 658 + 659 659 + assert controller.step() == cortex.SPP_RENEWAL_ACK_TIMEOUT_S 660 660 + _assert_absence_fenced_refresh_command(callosum.emitted[-1][1]["cmd"]) 661 661 + ref = callosum.emitted[-1][1]["ref"] 662 662 + controller.handle_supervisor_message( 663 663 + {"tract": "supervisor", "event": "started", "ref": ref} 664 664 + ) 665 665 + clock.advance(1) 666 666 + state["aggregate"] = "ready" 667 667 + state["fingerprint"] = "a" * 64 668 668 + state["record_fingerprint"] = "a" * 64 669 669 + state["observed"] = clock.now 670 670 + state["expires"] = clock.now + timedelta(minutes=10) 671 671 + controller.handle_supervisor_message( 672 672 + {"tract": "supervisor", "event": "stopped", "ref": ref, "exit_code": 0} 673 673 + ) 674 674 + 675 675 + assert controller._retry_after is None 676 676 + logged = "\n".join(str(call.args) for call in logger.info.call_args_list) 677 677 + assert "verified" in logged 678 678 + 679 679 + 680 680 + def test_spp_renewal_controller_absence_fenced_refresh_retries_when_key_appears( 681 681 + tmp_path, 682 682 + monkeypatch, 683 683 + ): 684 684 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 685 685 + state = { 686 686 + "aggregate": "unknown", 687 687 + "fingerprint": None, 688 688 + "observed": now - timedelta(minutes=9), 689 689 + "expires": now + timedelta(seconds=30), 690 690 + } 691 691 + controller, clock, callosum, _logger = _make_spp_controller( 692 692 + tmp_path, 693 693 + monkeypatch, 694 694 + state=state, 695 695 + clock=_FakeClock(now), 696 696 + ) 697 697 + 698 698 + controller.step() 699 699 + _assert_absence_fenced_refresh_command(callosum.emitted[-1][1]["cmd"]) 700 700 + ref = callosum.emitted[-1][1]["ref"] 701 701 + controller.handle_supervisor_message( 702 702 + {"tract": "supervisor", "event": "started", "ref": ref} 703 703 + ) 704 704 + state["fingerprint"] = "b" * 64 705 705 + state["record_fingerprint"] = "b" * 64 706 706 + controller.handle_supervisor_message( 707 707 + {"tract": "supervisor", "event": "stopped", "ref": ref, "exit_code": 3} 708 708 + ) 709 709 + 710 710 + assert controller._retry_after is not None 711 711 + clock.advance((controller._retry_after - clock.now).total_seconds()) 712 712 + controller.step() 713 713 + _assert_fenced_refresh_command(callosum.emitted[-1][1]["cmd"], "b" * 64) 714 714 + 715 715 + 716 716 + def test_spp_renewal_controller_restarts_from_ready_vs_stale_record( 717 717 + tmp_path, 718 718 + monkeypatch, 719 719 + ): 720 720 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 721 721 + ready_state = { 722 722 + "fingerprint": "1" * 64, 723 723 + "observed": now, 724 724 + "expires": now + timedelta(minutes=10), 725 725 + } 726 726 + ready, _clock, ready_callosum, _logger = _make_spp_controller( 727 727 + tmp_path, 728 728 + monkeypatch, 729 729 + state=ready_state, 730 730 + clock=_FakeClock(now), 731 731 + ) 732 732 + 733 733 + delay = ready.step() 734 734 + assert ready_callosum.emitted == [] 735 735 + assert delay > 0 736 736 + 737 737 + stale_state = { 738 738 + "fingerprint": "1" * 64, 739 739 + "observed": now - timedelta(minutes=20), 740 740 + "expires": now - timedelta(seconds=1), 741 741 + } 742 742 + stale, _clock, stale_callosum, _logger = _make_spp_controller( 743 743 + tmp_path, 744 744 + monkeypatch, 745 745 + state=stale_state, 746 746 + clock=_FakeClock(now), 747 747 + ) 748 748 + 749 749 + stale.step() 750 750 + _assert_fenced_refresh_command( 751 751 + stale_callosum.emitted[-1][1]["cmd"], 752 752 + stale_state["fingerprint"], 753 753 + ) 754 754 + 755 755 + 756 756 + def test_spp_renewal_controller_pending_fingerprint_switch_is_refenced( 757 757 + tmp_path, 758 758 + monkeypatch, 759 759 + ): 760 760 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 761 761 + first_fingerprint = "2" * 64 762 762 + second_fingerprint = "3" * 64 763 763 + state = { 764 764 + "fingerprint": first_fingerprint, 765 765 + "observed": now - timedelta(minutes=9), 766 766 + "expires": now + timedelta(seconds=30), 767 767 + } 768 768 + controller, clock, callosum, _logger = _make_spp_controller( 769 769 + tmp_path, 770 770 + monkeypatch, 771 771 + state=state, 772 772 + clock=_FakeClock(now), 773 773 + ) 774 774 + 775 775 + controller.step() 776 776 + first_ref = callosum.emitted[-1][1]["ref"] 777 777 + assert first_fingerprint in callosum.emitted[-1][1]["cmd"] 778 778 + controller.handle_supervisor_message( 779 779 + {"tract": "supervisor", "event": "started", "ref": first_ref} 780 780 + ) 781 781 + state["fingerprint"] = second_fingerprint 782 782 + state["record_fingerprint"] = second_fingerprint 783 783 + state["observed"] = clock.now 784 784 + state["expires"] = clock.now + timedelta(seconds=30) 785 785 + controller.handle_supervisor_message( 786 786 + {"tract": "supervisor", "event": "stopped", "ref": first_ref, "exit_code": 3} 787 787 + ) 788 788 + assert controller._retry_after is not None 789 789 + 790 790 + clock.advance((controller._retry_after - clock.now).total_seconds()) 791 791 + controller.step() 792 792 + 793 793 + assert len(callosum.emitted) == 2 794 794 + assert second_fingerprint in callosum.emitted[-1][1]["cmd"] 795 795 + 796 796 + 797 797 + def test_spp_renewal_controller_refresh_fallback_is_refenced_after_switch( 798 798 + tmp_path, 799 799 + monkeypatch, 800 800 + ): 801 801 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 802 802 + first_fingerprint = "2" * 64 803 803 + second_fingerprint = "3" * 64 804 804 + state = { 805 805 + "aggregate": "unknown", 806 806 + "fingerprint": first_fingerprint, 807 807 + "observed": now - timedelta(minutes=9), 808 808 + "expires": now + timedelta(seconds=30), 809 809 + } 810 810 + controller, clock, callosum, _logger = _make_spp_controller( 811 811 + tmp_path, 812 812 + monkeypatch, 813 813 + state=state, 814 814 + clock=_FakeClock(now), 815 815 + ) 816 816 + 817 817 + controller.step() 818 818 + first_cmd = callosum.emitted[-1][1]["cmd"] 819 819 + _assert_fenced_refresh_command(first_cmd, first_fingerprint) 820 820 + first_ref = callosum.emitted[-1][1]["ref"] 821 821 + controller.handle_supervisor_message( 822 822 + {"tract": "supervisor", "event": "started", "ref": first_ref} 823 823 + ) 824 824 + state["fingerprint"] = second_fingerprint 825 825 + state["record_fingerprint"] = second_fingerprint 826 826 + state["observed"] = clock.now 827 827 + state["expires"] = clock.now + timedelta(seconds=30) 828 828 + controller.handle_supervisor_message( 829 829 + {"tract": "supervisor", "event": "stopped", "ref": first_ref, "exit_code": 3} 830 830 + ) 831 831 + assert controller._retry_after is not None 832 832 + 833 833 + clock.advance((controller._retry_after - clock.now).total_seconds()) 834 834 + controller.step() 835 835 + 836 836 + assert len(callosum.emitted) == 2 837 837 + _assert_fenced_refresh_command(callosum.emitted[-1][1]["cmd"], second_fingerprint) 838 838 + 839 839 + 840 840 + def test_spp_renewal_controller_clock_jumps_do_not_duplicate( 841 841 + tmp_path, 842 842 + monkeypatch, 843 843 + ): 844 844 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 845 845 + state = { 846 846 + "fingerprint": "4" * 64, 847 847 + "observed": now, 848 848 + "expires": now + timedelta(minutes=10), 849 849 + } 850 850 + controller, clock, callosum, _logger = _make_spp_controller( 851 851 + tmp_path, 852 852 + monkeypatch, 853 853 + state=state, 854 854 + clock=_FakeClock(now), 855 855 + ) 856 856 + 857 857 + delay = controller.step() 858 858 + assert delay >= 0 859 859 + assert callosum.emitted == [] 860 860 + 861 861 + clock.advance(delay + 1) 862 862 + controller.step() 863 863 + assert callosum.emitted[-1][1]["cmd"][:3] == [ 864 864 + "journal", 865 865 + "brain", 866 866 + "renew-prerequisites", 867 867 + ] 868 868 + 869 869 + backward_state = { 870 870 + "fingerprint": "5" * 64, 871 871 + "observed": now, 872 872 + "expires": now + timedelta(minutes=10), 873 873 + } 874 874 + backward, backward_clock, backward_callosum, _logger = _make_spp_controller( 875 875 + tmp_path, 876 876 + monkeypatch, 877 877 + state=backward_state, 878 878 + clock=_FakeClock(now), 879 879 + ) 880 880 + backward_delay = backward.step() 881 881 + backward_clock.advance(-600) 882 882 + later_delay = backward.step() 883 883 + 884 884 + assert backward_delay >= 0 885 885 + assert later_delay >= 0 886 886 + assert backward_callosum.emitted == [] 887 887 + 888 888 + 889 889 + def test_spp_renewal_controller_recovers_from_inspection_and_fingerprint_errors( 890 890 + tmp_path, 891 891 + monkeypatch, 892 892 + ): 893 893 + from solstone.think import cortex 894 894 + 895 895 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 896 896 + state = { 897 897 + "fingerprint": "6" * 64, 898 898 + "observed": now - timedelta(minutes=9), 899 899 + "expires": now + timedelta(seconds=30), 900 900 + } 901 901 + clock = _FakeClock(now) 902 902 + callosum = _FakeCallosum() 903 903 + logger = MagicMock() 904 904 + inspect_calls = 0 905 905 + 906 906 + def flaky_inspect(current: datetime, *, journal_path=None): 907 907 + nonlocal inspect_calls 908 908 + inspect_calls += 1 909 909 + if inspect_calls == 1: 910 910 + raise OSError("SECRET-SENTINEL path") 911 911 + return _spp_inspector(state)(current, journal_path=journal_path) 912 912 + 913 913 + monkeypatch.setattr(cortex, "inspect_brain_state", flaky_inspect) 914 914 + monkeypatch.setattr( 915 915 + cortex, 916 916 + "read_active_brain_fingerprint_sha256", 917 917 + lambda *, journal_path=None: state["fingerprint"], 918 918 + ) 919 919 + controller = cortex.SppRenewalController( 920 920 + callosum=callosum, 921 921 + stop_event=threading.Event(), 922 922 + logger=logger, 923 923 + clock=clock, 924 924 + wait=clock.wait, 925 925 + journal_path=tmp_path, 926 926 + ) 927 927 + 928 928 + assert controller.step() == 5.0 929 929 + clock.advance(5.0) 930 930 + controller.step() 931 931 + assert callosum.emitted[-1][1]["cmd"][:3] == [ 932 932 + "journal", 933 933 + "brain", 934 934 + "renew-prerequisites", 935 935 + ] 936 936 + 937 937 + fingerprint_calls = 0 938 938 + 939 939 + def flaky_fingerprint(*, journal_path=None): 940 940 + nonlocal fingerprint_calls 941 941 + fingerprint_calls += 1 942 942 + if fingerprint_calls == 1: 943 943 + raise OSError("SECRET-SENTINEL path") 944 944 + return state["fingerprint"] 945 945 + 946 946 + callosum = _FakeCallosum() 947 947 + logger = MagicMock() 948 948 + monkeypatch.setattr(cortex, "inspect_brain_state", _spp_inspector(state)) 949 949 + monkeypatch.setattr( 950 950 + cortex, "read_active_brain_fingerprint_sha256", flaky_fingerprint 951 951 + ) 952 952 + controller = cortex.SppRenewalController( 953 953 + callosum=callosum, 954 954 + stop_event=threading.Event(), 955 955 + logger=logger, 956 956 + clock=clock, 957 957 + wait=clock.wait, 958 958 + journal_path=tmp_path, 959 959 + ) 960 960 + 961 961 + assert controller.step() == 5.0 962 962 + clock.advance(5.0) 963 963 + controller.step() 964 964 + assert callosum.emitted[-1][1]["cmd"][:3] == [ 965 965 + "journal", 966 966 + "brain", 967 967 + "renew-prerequisites", 968 968 + ] 969 969 + 970 970 + logged = "\n".join( 971 971 + " ".join(str(part) for part in call.args) for call in logger.info.call_args_list 972 972 + ) 973 973 + assert "SECRET-SENTINEL" not in logged 974 974 + assert str(tmp_path) not in logged 975 975 + 976 976 + 977 977 + def test_spp_renewal_controller_run_contains_unexpected_step_exception( 978 978 + tmp_path, 979 979 + monkeypatch, 980 980 + ): 981 981 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 982 982 + state = { 983 983 + "fingerprint": "8" * 64, 984 984 + "observed": now, 985 985 + "expires": now + timedelta(minutes=10), 986 986 + } 987 987 + controller, _clock, _callosum, logger = _make_spp_controller( 988 988 + tmp_path, 989 989 + monkeypatch, 990 990 + state=state, 991 991 + clock=_FakeClock(now), 992 992 + ) 993 993 + controller.step = MagicMock(side_effect=RuntimeError("SECRET-SENTINEL")) 994 994 + 995 995 + def stop_after_wait(_seconds: float) -> bool: 996 996 + controller.stop_event.set() 997 997 + return True 998 998 + 999 999 + controller.wait = stop_after_wait 1000 1000 + controller.run() 1001 1001 + 1002 1002 + logged = "\n".join( 1003 1003 + " ".join(str(part) for part in call.args) for call in logger.info.call_args_list 1004 1004 + ) 1005 1005 + assert "RuntimeError" in logged 1006 1006 + assert "retrying" in logged 1007 1007 + assert "SECRET-SENTINEL" not in logged 1008 1008 + 1009 1009 + 1010 1010 + def test_spp_renewal_controller_retries_with_cap_and_clears_on_disable( 1011 1011 + tmp_path, 1012 1012 + monkeypatch, 1013 1013 + ): 1014 1014 + from solstone.think import cortex 1015 1015 + 1016 1016 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 1017 1017 + clock = _FakeClock(now) 1018 1018 + state = { 1019 1019 + "fingerprint": "c" * 64, 1020 1020 + "observed": now - timedelta(minutes=9), 1021 1021 + "expires": now + timedelta(seconds=30), 1022 1022 + } 1023 1023 + monkeypatch.setattr(cortex, "inspect_brain_state", _spp_inspector(state)) 1024 1024 + monkeypatch.setattr( 1025 1025 + cortex, 1026 1026 + "read_active_brain_fingerprint_sha256", 1027 1027 + lambda *, journal_path=None: state["fingerprint"], 1028 1028 + ) 1029 1029 + controller = cortex.SppRenewalController( 1030 1030 + callosum=_FakeCallosum(), 1031 1031 + stop_event=threading.Event(), 1032 1032 + logger=MagicMock(), 1033 1033 + clock=clock, 1034 1034 + wait=clock.wait, 1035 1035 + journal_path=tmp_path, 1036 1036 + ) 1037 1037 + 1038 1038 + delays: list[float] = [] 1039 1039 + for _ in range(6): 1040 1040 + controller.step() 1041 1041 + clock.advance(cortex.SPP_RENEWAL_ACK_TIMEOUT_S + 1) 1042 1042 + controller.step() 1043 1043 + assert controller._retry_after is not None 1044 1044 + delays.append((controller._retry_after - clock.now).total_seconds()) 1045 1045 + clock.advance(delays[-1]) 1046 1046 + assert delays == [5.0, 10.0, 20.0, 40.0, 60.0, 60.0] 1047 1047 + 1048 1048 + controller.step() 1049 1049 + assert controller._pending_ref is not None 1050 1050 + state["lane"] = "byo-cloud" 1051 1051 + controller.step() 1052 1052 + assert controller._pending_ref is None 1053 1053 + assert controller._retry_after is None 1054 1054 + 1055 1055 + 1056 1056 + def test_spp_renewal_controller_logs_lifecycle_events_without_secrets( 1057 1057 + tmp_path, 1058 1058 + monkeypatch, 1059 1059 + ): 1060 1060 + from solstone.think import cortex 1061 1061 + 1062 1062 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 1063 1063 + clock = _FakeClock(now) 1064 1064 + state = { 1065 1065 + "lane": "byo-cloud", 1066 1066 + "fingerprint": "d" * 64, 1067 1067 + "observed": now - timedelta(minutes=9), 1068 1068 + "expires": now + timedelta(seconds=30), 1069 1069 + "secret": "SECRET-SENTINEL", 1070 1070 + } 1071 1071 + monkeypatch.setattr(cortex, "inspect_brain_state", _spp_inspector(state)) 1072 1072 + monkeypatch.setattr( 1073 1073 + cortex, 1074 1074 + "read_active_brain_fingerprint_sha256", 1075 1075 + lambda *, journal_path=None: state["fingerprint"], 1076 1076 + ) 1077 1077 + logger = MagicMock() 1078 1078 + callosum = _FakeCallosum() 1079 1079 + controller = cortex.SppRenewalController( 1080 1080 + callosum=callosum, 1081 1081 + stop_event=threading.Event(), 1082 1082 + logger=logger, 1083 1083 + clock=clock, 1084 1084 + wait=clock.wait, 1085 1085 + journal_path=tmp_path, 1086 1086 + ) 1087 1087 + 1088 1088 + assert controller.step() == 30.0 1089 1089 + assert callosum.emitted == [] 1090 1090 + 1091 1091 + state["lane"] = "spp" 1092 1092 + state["expires"] = clock.now + timedelta(minutes=10) 1093 1093 + scheduled_delay = controller.step() 1094 1094 + assert scheduled_delay > 0 1095 1095 + clock.advance(scheduled_delay) 1096 1096 + controller.step() 1097 1097 + assert callosum.emitted[-1][1]["cmd"][:3] == [ 1098 1098 + "journal", 1099 1099 + "brain", 1100 1100 + "renew-prerequisites", 1101 1101 + ] 1102 1102 + verified_ref = callosum.emitted[-1][1]["ref"] 1103 1103 + controller.handle_supervisor_message( 1104 1104 + {"tract": "supervisor", "event": "started", "ref": verified_ref} 1105 1105 + ) 1106 1106 + clock.advance(1) 1107 1107 + state["observed"] = clock.now 1108 1108 + state["expires"] = clock.now + timedelta(minutes=10) 1109 1109 + controller.handle_supervisor_message( 1110 1110 + { 1111 1111 + "tract": "supervisor", 1112 1112 + "event": "stopped", 1113 1113 + "ref": verified_ref, 1114 1114 + "exit_code": 0, 1115 1115 + } 1116 1116 + ) 1117 1117 + 1118 1118 + state["expires"] = clock.now + timedelta(seconds=30) 1119 1119 + controller.step() 1120 1120 + failed_ref = callosum.emitted[-1][1]["ref"] 1121 1121 + clock.advance(cortex.SPP_RENEWAL_ACK_TIMEOUT_S + 1) 1122 1122 + controller.step() 1123 1123 + clock.advance(5.0) 1124 1124 + controller.step() 1125 1125 + stale_ref = callosum.emitted[-1][1]["ref"] 1126 1126 + assert stale_ref != failed_ref 1127 1127 + controller.handle_supervisor_message( 1128 1128 + {"tract": "supervisor", "event": "started", "ref": stale_ref} 1129 1129 + ) 1130 1130 + clock.advance(cortex.SPP_RENEWAL_ATTEMPT_BOUND_S + 1) 1131 1131 + controller.step() 1132 1132 + 1133 1133 + logged = "\n".join( 1134 1134 + " ".join(str(part) for part in call.args) for call in logger.info.call_args_list 1135 1135 + ) 1136 1136 + for event in ( 1137 1137 + "disabled", 1138 1138 + "scheduled", 1139 1139 + "in_flight", 1140 1140 + "verified", 1141 1141 + "failed", 1142 1142 + "stale", 1143 1143 + "retrying", 1144 1144 + ): 1145 1145 + assert event in logged 1146 1146 + assert "SECRET-SENTINEL" not in logged 1147 1147 + assert str(tmp_path) not in logged 1148 1148 + assert "SECRET-SENTINEL" not in json.dumps(callosum.emitted) 1149 1149 + 1150 1150 + 1151 1151 + def test_cortex_stop_wakes_and_joins_spp_renewal_worker( 1152 1152 + mock_journal, 1153 1153 + monkeypatch, 1154 1154 + ): 1155 1155 + from solstone.think import cortex 1156 1156 + from solstone.think.cortex import CortexService 1157 1157 + 1158 1158 + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) 1159 1159 + state = {"lane": "byo-cloud", "fingerprint": "7" * 64} 1160 1160 + monkeypatch.setattr(cortex, "inspect_brain_state", _spp_inspector(state)) 1161 1161 + service = CortexService(str(mock_journal), clock=lambda: now) 1162 1162 + service.callosum = MagicMock() 1163 1163 + 1164 1164 + service._start_spp_renewal_controller() 1165 1165 + assert service._spp_renewal_worker is not None 1166 1166 + deadline = time.monotonic() + 1.0 1167 1167 + while not service._spp_renewal_worker.is_alive() and time.monotonic() < deadline: 1168 1168 + time.sleep(0.01) 1169 1169 + assert service._spp_renewal_worker.is_alive() 1170 1170 + 1171 1171 + service.stop() 1172 1172 + 1173 1173 + assert not service._spp_renewal_worker.is_alive() 176 1174 177 1175 178 1176 def test_handle_request_dedups_existing_active_file(

+8 -2

tests/test_no_implicit_cloud.py

View file Reviewed

··· 823 823 establish.assert_called_once() 824 824 825 825 826 826 - def test_confidential_stt_stale_session_defers_before_egress(tmp_path, monkeypatch): 826 826 + def test_confidential_stt_stale_session_replacement_failure_defers_before_egress( 827 827 + tmp_path, monkeypatch 828 828 + ): 827 829 from solstone.think.services import spp, spp_transport 830 830 + from solstone.think.services.spp_attest.ratls.channel import RatlsChannelError 828 831 829 832 _empty_journal(tmp_path, monkeypatch) 830 833 config = _confidential_config(provider_pins=False) ··· 835 838 spp_transport._FORWARDER_BASE_URL = "http://127.0.0.1:4567" 836 839 spp.record_attestation_verified(_stale_session(object())) 837 840 mocks = _install_stt_backend_mocks(monkeypatch, confidential_result=None) 841 841 + establish = Mock(side_effect=RatlsChannelError("gateway_unreachable")) 842 842 + monkeypatch.setattr(spp_transport, "establish_attested_channel", establish) 838 843 httpx_post = Mock(side_effect=AssertionError("audio egress attempted")) 839 844 monkeypatch.setattr("httpx.post", httpx_post) 840 845 ··· 843 848 with pytest.raises(ConfidentialTranscribeDeferral) as exc_info: 844 849 transcribe("confidential", _stt_audio(), 16000, {}) 845 850 846 846 - assert exc_info.value.reason_code == "attestation_stale" 851 851 + assert exc_info.value.reason_code == "attestation_unreachable" 847 852 mocks["parakeet"].assert_not_called() 848 853 httpx_post.assert_not_called() 854 854 + establish.assert_called_once() 849 855 850 856 851 857 def test_confidential_stt_setting_off_gate_blocks_confidential_only(

+1

tests/test_supervisor.py

View file Reviewed

··· 588 588 assert get(["journal", "indexer", "--rescan"]) == "indexer" 589 589 assert get(["sol", "insight", "20240101"]) == "insight" 590 590 assert get(["journal", "think", "--day", "20240101"]) == "daily" 591 591 + assert get(["journal", "brain", "renew-prerequisites"]) == "brain" 591 592 assert get(["journal", "maintenance", "list"]) == "maintenance" 592 593 assert get(["journal", "maintenance", "run", "foo:bar"]) == "maintenance:foo:bar" 593 594 assert get(["journal", "maintenance", "run", "baz:qux"]) == "maintenance:baz:qux"