solpbc.org / solstone
personal memory agent
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

fix(thinking): add local runtime recovery action

author
Jer Miller date (Jul 19, 2026, 10:21 AM -0600) commit 5dd92075 parent 9aa17a8e
+1209 -32
+85 -1
solstone/apps/thinking/copy.py
··· 278 278 }, 279 279 "pill_inflight": "setting up", 280 280 "pill_failed": STATE_LABELS["failed"], 281 - "retry": "try again", 281 + "failed_verdict": "local setup didn't finish", 282 + "failed_reason": "local setup stopped before it finished.", 283 + "retry": "try setup again", 282 284 "install": "install local model", 283 285 "notice_inflight": "local thinking will stay in your journal once setup finishes.", 284 286 } 287 + LOCAL_RECOVERY = { 288 + "retry": "try starting local again", 289 + "states": { 290 + "checking": { 291 + "pill": "checking", 292 + "verdict": "checking local setup", 293 + "reason": "sol is checking the selected model and this computer.", 294 + }, 295 + "starting": { 296 + "pill": "starting", 297 + "verdict": "starting local thinking", 298 + "reason": "sol is checking the model before it uses it.", 299 + }, 300 + "recovering": { 301 + "pill": "recovering", 302 + "verdict": "local thinking is recovering", 303 + "reason": "sol will try starting it again shortly.", 304 + }, 305 + "retrying": { 306 + "pill": "retrying", 307 + "verdict": "trying local thinking again", 308 + "reason": "sol is starting a new recovery attempt.", 309 + }, 310 + "waiting": { 311 + "pill": "waiting", 312 + "verdict": "local thinking is waiting", 313 + "reason": "it will start when this computer is ready.", 314 + }, 315 + "unsupported": { 316 + "pill": "unavailable", 317 + "verdict": "this computer can't run this local model", 318 + "reason": "local thinking needs supported hardware on this computer.", 319 + }, 320 + "failed": { 321 + "pill": "needs attention", 322 + "verdict": "local thinking couldn't start", 323 + "reason": "the local model stopped before it became ready.", 324 + }, 325 + "changing": { 326 + "pill": "changing", 327 + "verdict": "finishing the local change", 328 + "reason": "sol is waiting for current local work to finish.", 329 + }, 330 + "cleanup_failed": { 331 + "pill": "needs attention", 332 + "verdict": "local couldn't finish changing state", 333 + "reason": ( 334 + "sol couldn't confirm that local thinking stopped. " 335 + "it will keep checking safely." 336 + ), 337 + }, 338 + "corrupt": { 339 + "pill": "can't verify", 340 + "verdict": "local status can't be confirmed", 341 + "reason": "check again before changing local setup.", 342 + }, 343 + "unavailable": { 344 + "pill": "can't verify", 345 + "verdict": "local status can't be read", 346 + "reason": "correct the local file-access problem, then check again.", 347 + }, 348 + "ready": { 349 + "pill": "on", 350 + "verdict": "local thinking is ready", 351 + "reason": "sol is thinking on this computer.", 352 + }, 353 + "ready_proof_unavailable": { 354 + "pill": "on, needs a check", 355 + "verdict": "local thinking is still running", 356 + "reason": ( 357 + "sol couldn't refresh the local file check. " 358 + "it won't start a replacement until the check returns." 359 + ), 360 + }, 361 + }, 362 + } 285 363 CONFIDENTIAL_MORE_LABEL = "how it works →" 286 364 SCOUT_STATE_OFF = "off" 287 365 SCOUT_STATE_REQUESTED = "requested" ··· 341 419 "local_install": { 342 420 **LOCAL_INSTALL, 343 421 "phases": dict(LOCAL_INSTALL["phases"]), 422 + }, 423 + "local_recovery": { 424 + "retry": LOCAL_RECOVERY["retry"], 425 + "states": { 426 + key: dict(value) for key, value in LOCAL_RECOVERY["states"].items() 427 + }, 344 428 }, 345 429 "scout": { 346 430 "state_labels": dict(SCOUT_STATE_LABELS),
+127
solstone/apps/thinking/local_recovery.py
··· 1 + # SPDX-License-Identifier: AGPL-3.0-only 2 + # Copyright (c) 2026 sol pbc 3 + 4 + """Owner-safe local runtime recovery projection and actions.""" 5 + 6 + from __future__ import annotations 7 + 8 + from typing import Any 9 + 10 + from solstone.think.providers.runtime_health import ( 11 + inspect_retry_token, 12 + inspect_runtime_health, 13 + request_runtime_retry, 14 + ) 15 + 16 + _TRANSIENT_PHASES = frozenset( 17 + { 18 + "observing", 19 + "artifact-not-ready", 20 + "host-blocked", 21 + "starting", 22 + "warming", 23 + "backoff", 24 + "retry-requested", 25 + "stop-deferred", 26 + "stopping", 27 + } 28 + ) 29 + 30 + 31 + def runtime_view() -> dict[str, Any]: 32 + """Return the small owner-safe projection consumed by Thinking.""" 33 + health_inspection = inspect_runtime_health("local") 34 + retry_inspection = inspect_retry_token("local") 35 + 36 + unavailable = _unavailable_view(health_inspection, retry_inspection) 37 + if unavailable is not None: 38 + return unavailable 39 + 40 + health = health_inspection["record"] 41 + retry = retry_inspection["record"] 42 + assert isinstance(health, dict) 43 + assert isinstance(retry, dict) 44 + 45 + current_health = health 46 + current_retry = retry 47 + fingerprint = current_health["desired_fingerprint_sha256"] 48 + retry_present = current_retry["token_id"] is not None 49 + retry_matches = ( 50 + retry_present and current_retry["desired_fingerprint_sha256"] == fingerprint 51 + ) 52 + phase = current_health["phase"] 53 + reason_code = current_health["reason_code"] 54 + poll = phase in _TRANSIENT_PHASES 55 + 56 + if retry_matches: 57 + phase = "retry-requested" 58 + reason_code = "retry-token-requested" 59 + poll = True 60 + return { 61 + "status": "ok", 62 + "phase": phase, 63 + "reason_code": reason_code, 64 + "health_revision": current_health["revision"], 65 + "desired_fingerprint_sha256": fingerprint, 66 + "retry_revision": current_retry["revision"], 67 + "retry_pending": retry_matches, 68 + "can_retry": ( 69 + current_health["phase"] == "failed" 70 + and fingerprint is not None 71 + and not retry_matches 72 + ), 73 + "poll": poll, 74 + "updated_at": current_health["updated_at"], 75 + } 76 + 77 + 78 + def request_retry( 79 + *, 80 + health_revision: int, 81 + retry_revision: int, 82 + desired_fingerprint_sha256: str, 83 + ) -> dict[str, Any]: 84 + """Request one guarded retry, then return the committed projection.""" 85 + request_runtime_retry( 86 + "local", 87 + expected_health_revision=health_revision, 88 + expected_retry_revision=retry_revision, 89 + desired_fingerprint_sha256=desired_fingerprint_sha256, 90 + owner={ 91 + "module": "solstone.apps.thinking.local_recovery", 92 + "source": "owner-recovery", 93 + }, 94 + ) 95 + return runtime_view() 96 + 97 + 98 + def _unavailable_view( 99 + health_inspection: dict[str, Any], 100 + retry_inspection: dict[str, Any], 101 + ) -> dict[str, Any] | None: 102 + statuses = { 103 + str(health_inspection["status"]), 104 + str(retry_inspection["status"]), 105 + } 106 + if "unavailable" in statuses: 107 + return _failed_read_view("unavailable", "state-unavailable") 108 + if "corrupt" in statuses: 109 + return _failed_read_view("corrupt", "state-corrupt") 110 + return None 111 + 112 + 113 + def _failed_read_view(status: str, phase: str) -> dict[str, Any]: 114 + return { 115 + "status": status, 116 + "phase": phase, 117 + "reason_code": ( 118 + "record-unavailable" if status == "unavailable" else "record-malformed" 119 + ), 120 + "health_revision": None, 121 + "desired_fingerprint_sha256": None, 122 + "retry_revision": None, 123 + "retry_pending": False, 124 + "can_retry": False, 125 + "poll": False, 126 + "updated_at": None, 127 + }
+71 -1
solstone/apps/thinking/routes.py
··· 16 16 from flask import Blueprint, current_app, jsonify, request 17 17 18 18 from solstone.apps.thinking import copy as thinking_copy 19 - from solstone.apps.thinking import local_bootstrap, scout_lane 19 + from solstone.apps.thinking import local_bootstrap, local_recovery, scout_lane 20 20 from solstone.apps.thinking.copy import thinking_copy_payload 21 21 from solstone.apps.thinking.model_tiers import MODEL_TIERS 22 22 from solstone.apps.utils import log_app_action ··· 55 55 normalize_local_endpoint_url, 56 56 resolve_local_endpoint, 57 57 ) 58 + from solstone.think.providers.runtime_health import ( 59 + RuntimeHealthConflictError, 60 + RuntimeHealthMalformedError, 61 + RuntimeHealthUnavailableError, 62 + ) 58 63 from solstone.think.services import ( 59 64 operations, 60 65 scout, ··· 517 522 "api_keys": _api_key_status(config), 518 523 "key_validation": _filtered_ai_key_validation(config), 519 524 "local": local_status, 525 + "local_runtime": local_recovery.runtime_view(), 520 526 "local_override": _local_override_payload(config), 521 527 "local_backend": "mlx" if local_bootstrap._is_mlx_backend() else "local", 522 528 "scout_enabled": scout.is_scout_enabled(), ··· 1000 1006 return jsonify(local_bootstrap.get_state(model)) 1001 1007 except Exception: 1002 1008 logger.exception("error loading local provider bootstrap status") 1009 + return _thinking_operation_failed() 1010 + 1011 + 1012 + @thinking_bp.route("/api/local/runtime") 1013 + def get_local_runtime() -> Any: 1014 + try: 1015 + return jsonify(local_recovery.runtime_view()) 1016 + except Exception: 1017 + logger.exception("error loading local runtime recovery state") 1018 + return _thinking_operation_failed() 1019 + 1020 + 1021 + @thinking_bp.route("/api/local/runtime/retry", methods=["POST"]) 1022 + def retry_local_runtime() -> Any: 1023 + request_data = request.get_json(silent=True) 1024 + if not isinstance(request_data, dict): 1025 + return error_response(MISSING_REQUEST_BODY, detail="No data provided") 1026 + expected_fields = { 1027 + "health_revision", 1028 + "retry_revision", 1029 + "desired_fingerprint_sha256", 1030 + } 1031 + if set(request_data) != expected_fields: 1032 + return error_response( 1033 + INVALID_REQUEST_VALUE, 1034 + detail="runtime retry requires the current recovery state", 1035 + ) 1036 + health_revision = request_data["health_revision"] 1037 + retry_revision = request_data["retry_revision"] 1038 + desired_fingerprint = request_data["desired_fingerprint_sha256"] 1039 + if ( 1040 + isinstance(health_revision, bool) 1041 + or not isinstance(health_revision, int) 1042 + or health_revision < 0 1043 + or isinstance(retry_revision, bool) 1044 + or not isinstance(retry_revision, int) 1045 + or retry_revision < 0 1046 + or not isinstance(desired_fingerprint, str) 1047 + or not desired_fingerprint 1048 + ): 1049 + return error_response( 1050 + INVALID_REQUEST_VALUE, 1051 + detail="runtime retry requires the current recovery state", 1052 + ) 1053 + try: 1054 + return jsonify( 1055 + local_recovery.request_retry( 1056 + health_revision=health_revision, 1057 + retry_revision=retry_revision, 1058 + desired_fingerprint_sha256=desired_fingerprint, 1059 + ) 1060 + ) 1061 + except RuntimeHealthConflictError: 1062 + return error_response( 1063 + INVALID_OPERATION_FOR_STATE, 1064 + detail="local status changed; check again", 1065 + ) 1066 + except (RuntimeHealthMalformedError, RuntimeHealthUnavailableError): 1067 + logger.exception("local runtime retry state is unavailable") 1068 + return _thinking_operation_failed( 1069 + "local status can't be changed right now; check again" 1070 + ) 1071 + except Exception: 1072 + logger.exception("error requesting local runtime retry") 1003 1073 return _thinking_operation_failed() 1004 1074 1005 1075
+214 -9
solstone/apps/thinking/static/thinking.js
··· 10 10 scout: null, 11 11 install: null, 12 12 installPollGeneration: 0, 13 + runtimePollGeneration: 0, 13 14 confidentialPollGeneration: 0, 14 15 confidentialDetailOpen: false, 15 16 selectedByoProvider: '', ··· 131 132 return { 132 133 pill: text?.pill_failed || '', 133 134 title: 'local', 134 - sub: text?.pill_failed || '', 135 - message: status?.install_error || '', 136 - notice: '', 135 + sub: text?.failed_verdict || '', 136 + message: '', 137 + notice: text?.failed_reason || '', 137 138 activate: false, 138 139 bootstrap: true, 139 140 bootstrapLabel: text?.retry || '', ··· 143 144 return null; 144 145 } 145 146 147 + function localRuntimeCopy(runtime, active, text) { 148 + if (!runtime) return null; 149 + const states = text?.states || {}; 150 + const view = (key, options = {}) => { 151 + const value = states[key] || {}; 152 + return { 153 + pill: value.pill || '', 154 + title: 'local', 155 + sub: value.verdict || '', 156 + message: '', 157 + notice: value.reason || '', 158 + activate: false, 159 + bootstrap: false, 160 + retryRuntime: !!options.retryRuntime, 161 + retryRuntimeLabel: text?.retry || '', 162 + tone: options.tone || '', 163 + }; 164 + }; 165 + 166 + if (runtime.status === 'corrupt' || runtime.phase === 'state-corrupt') { 167 + return view('corrupt', {tone: 'bad'}); 168 + } 169 + if (runtime.status === 'unavailable' || runtime.phase === 'state-unavailable') { 170 + return view('unavailable', {tone: 'bad'}); 171 + } 172 + if (!active) { 173 + return runtime.phase === 'cleanup-failed' 174 + ? view('cleanup_failed', {tone: 'bad'}) 175 + : null; 176 + } 177 + 178 + if (runtime.phase === 'ready') return view('ready'); 179 + if (runtime.phase === 'ready-proof-unavailable') { 180 + return view('ready_proof_unavailable'); 181 + } 182 + if (runtime.phase === 'starting' || runtime.phase === 'warming') { 183 + return view('starting'); 184 + } 185 + if (runtime.phase === 'backoff') return view('recovering'); 186 + if (runtime.phase === 'retry-requested') return view('retrying'); 187 + if (runtime.phase === 'host-blocked') { 188 + if (runtime.reason_code === 'platform-unsupported' || runtime.reason_code === 'package-unavailable') { 189 + return view('unsupported', {tone: 'bad'}); 190 + } 191 + return view('waiting'); 192 + } 193 + if (runtime.phase === 'failed') { 194 + return view('failed', { 195 + retryRuntime: runtime.can_retry === true, 196 + tone: 'bad', 197 + }); 198 + } 199 + if (runtime.phase === 'stop-deferred' || runtime.phase === 'stopping') { 200 + return view('changing'); 201 + } 202 + if (runtime.phase === 'cleanup-failed') { 203 + return view('cleanup_failed', {tone: 'bad'}); 204 + } 205 + if (runtime.phase === 'artifact-not-ready') return null; 206 + return view('checking'); 207 + } 208 + 146 209 async function pollLocalInstallUntilTerminal({ 147 210 fetchStatus, 148 211 sleepFn, ··· 163 226 await sleepFn(intervalMs); 164 227 } 165 228 return null; 229 + } 230 + 231 + async function pollLocalRuntimeUntilStable({ 232 + fetchStatus, 233 + sleepFn, 234 + applyStatus, 235 + isCurrent, 236 + intervalMs, 237 + initialStatus = null, 238 + }) { 239 + let status = initialStatus; 240 + while (isCurrent() && status?.poll === true) { 241 + await sleepFn(intervalMs); 242 + if (!isCurrent()) return null; 243 + status = await fetchStatus(); 244 + applyStatus(status); 245 + } 246 + return status; 166 247 } 167 248 168 249 function handleInstallPollError({ ··· 724 805 } 725 806 if (target !== 'local-setup') { 726 807 stopInstallPoll(); 808 + stopRuntimePoll(); 727 809 } else if (state.localModels.length > 0) { 728 810 refreshInstallStatus({autoResume: true}).catch((err) => { 811 + setMessage('localSetupMessage', err.message, 'error'); 812 + }); 813 + refreshLocalRuntime({autoResume: true}).catch((err) => { 729 814 setMessage('localSetupMessage', err.message, 'error'); 730 815 }); 731 816 } ··· 1565 1650 } 1566 1651 1567 1652 function localCopy() { 1568 - const installOverride = installCopyForStatus(state.install, copy.local_install || {}); 1569 - if (installOverride) return installOverride; 1570 - const local = localReadiness(); 1571 - const reason = local.reason; 1572 1653 if (localEndpointConfigured()) { 1573 1654 return { 1574 1655 pill: 'endpoint', ··· 1582 1663 endpointOverride: true, 1583 1664 }; 1584 1665 } 1666 + const installOverride = installCopyForStatus(state.install, copy.local_install || {}); 1667 + const runtimeOverride = localRuntimeCopy( 1668 + state.providers.local_runtime, 1669 + state.providers.active_lane?.lane === 'local', 1670 + copy.local_recovery || {}, 1671 + ); 1672 + if (installIsInFlight(state.install)) return installOverride; 1673 + if ( 1674 + runtimeOverride 1675 + && ['ready', 'ready-proof-unavailable'].includes(state.providers.local_runtime?.phase) 1676 + ) { 1677 + return runtimeOverride; 1678 + } 1679 + if (installOverride) return installOverride; 1680 + if (runtimeOverride) return runtimeOverride; 1681 + const local = localReadiness(); 1682 + const reason = local.reason; 1585 1683 if (local.status === 'ready' || reason === 'ready') { 1586 1684 return { 1587 1685 pill: 'off', ··· 1713 1811 setHidden('localOverrideNotice', !local.endpointOverride); 1714 1812 setButtonState('localBootstrap', local.bootstrap, !local.bootstrap); 1715 1813 setButtonText('localBootstrap', local.bootstrapLabel || copy.local_install?.install || ''); 1814 + setButtonState('localRuntimeRetry', local.retryRuntime, !local.retryRuntime); 1815 + setButtonText('localRuntimeRetry', local.retryRuntimeLabel || copy.local_recovery?.retry || ''); 1716 1816 setButtonState('localActivate', local.activate, !local.activate); 1717 1817 setButtonState('localRefresh', true, false); 1718 1818 const links = $('localSetupLinks'); ··· 1875 1975 state.installPollGeneration += 1; 1876 1976 } 1877 1977 1978 + function stopRuntimePoll() { 1979 + state.runtimePollGeneration += 1; 1980 + } 1981 + 1878 1982 function stopConfidentialPoll(options = {}) { 1879 1983 state.confidentialPollGeneration += 1; 1880 1984 if (options.clearOperation && clearConfidentialInProgressOperation(state.providers.active_lane)) { ··· 1932 2036 return api(`api/local/bootstrap/status?model=${encodeURIComponent(model)}`); 1933 2037 } 1934 2038 2039 + async function fetchLocalRuntime() { 2040 + return api('api/local/runtime'); 2041 + } 2042 + 2043 + function applyLocalRuntime(status, generation) { 2044 + if (generation !== undefined && generation !== state.runtimePollGeneration) return false; 2045 + const currentRevision = state.providers.local_runtime?.health_revision; 2046 + const nextRevision = status?.health_revision; 2047 + const currentRetryRevision = state.providers.local_runtime?.retry_revision; 2048 + const nextRetryRevision = status?.retry_revision; 2049 + if ( 2050 + ( 2051 + Number.isInteger(currentRevision) 2052 + && Number.isInteger(nextRevision) 2053 + && nextRevision < currentRevision 2054 + ) 2055 + || ( 2056 + Number.isInteger(currentRetryRevision) 2057 + && Number.isInteger(nextRetryRevision) 2058 + && nextRetryRevision < currentRetryRevision 2059 + ) 2060 + ) { 2061 + return false; 2062 + } 2063 + state.providers.local_runtime = status || null; 2064 + renderAll(); 2065 + return true; 2066 + } 2067 + 2068 + function startRuntimePoll(initialStatus = state.providers.local_runtime) { 2069 + stopRuntimePoll(); 2070 + const generation = state.runtimePollGeneration; 2071 + return pollLocalRuntimeUntilStable({ 2072 + fetchStatus: fetchLocalRuntime, 2073 + sleepFn: sleep, 2074 + applyStatus: (status) => applyLocalRuntime(status, generation), 2075 + isCurrent: () => generation === state.runtimePollGeneration, 2076 + intervalMs: pollIntervalMs, 2077 + initialStatus, 2078 + }) 2079 + .then(() => { 2080 + if (generation === state.runtimePollGeneration) stopRuntimePoll(); 2081 + }) 2082 + .catch((err) => { 2083 + if (generation !== state.runtimePollGeneration) return; 2084 + stopRuntimePoll(); 2085 + setMessage('localSetupMessage', err.message, 'error'); 2086 + }); 2087 + } 2088 + 2089 + async function refreshLocalRuntime({autoResume = false} = {}) { 2090 + const status = await fetchLocalRuntime(); 2091 + applyLocalRuntime(status); 2092 + if (status?.poll === true && autoResume) { 2093 + startRuntimePoll(status); 2094 + } else { 2095 + stopRuntimePoll(); 2096 + } 2097 + return status; 2098 + } 2099 + 1935 2100 function applyLocalInstallStatus(status, generation) { 1936 2101 if (generation !== undefined && generation !== state.installPollGeneration) return false; 1937 2102 state.install = status || null; ··· 1957 2122 if (installIsTerminal(status)) { 1958 2123 stopInstallPoll(); 1959 2124 if (status?.install_state === 'installed') { 1960 - Promise.all([refreshProviders(), refreshLocalAvailability()]).catch((err) => { 2125 + Promise.all([ 2126 + refreshProviders(), 2127 + refreshLocalAvailability(), 2128 + refreshLocalRuntime({autoResume: true}), 2129 + ]).catch((err) => { 1961 2130 setMessage('localSetupMessage', err.message, 'error'); 1962 2131 }); 1963 2132 } ··· 2007 2176 return; 2008 2177 } 2009 2178 await switchLane(target); 2179 + if (target === 'local') { 2180 + await refreshLocalRuntime({autoResume: true}); 2181 + showView('local-setup'); 2182 + return; 2183 + } 2010 2184 showView('main'); 2011 2185 } 2012 2186 ··· 2283 2457 } else { 2284 2458 await refreshInstallStatus({autoResume: true}); 2285 2459 } 2286 - await Promise.all([refreshProviders(), refreshLocalAvailability()]); 2460 + await Promise.all([ 2461 + refreshProviders(), 2462 + refreshLocalAvailability(), 2463 + refreshLocalRuntime({autoResume: true}), 2464 + ]); 2465 + } 2466 + 2467 + async function retryLocalRuntime() { 2468 + const runtime = state.providers.local_runtime; 2469 + if (!runtime?.can_retry) return; 2470 + const button = $('localRuntimeRetry'); 2471 + if (button) button.disabled = true; 2472 + try { 2473 + const status = await api('api/local/runtime/retry', { 2474 + method: 'POST', 2475 + body: JSON.stringify({ 2476 + health_revision: runtime.health_revision, 2477 + retry_revision: runtime.retry_revision, 2478 + desired_fingerprint_sha256: runtime.desired_fingerprint_sha256, 2479 + }), 2480 + }); 2481 + applyLocalRuntime(status); 2482 + startRuntimePoll(status); 2483 + } catch (_err) { 2484 + await refreshLocalRuntime({autoResume: true}); 2485 + } 2287 2486 } 2288 2487 2289 2488 function openLane(lane) { ··· 2408 2607 $('confidentialAudioToggle')?.addEventListener('change', (event) => setConfidentialAudio(event.target.checked)); 2409 2608 $('localRefresh')?.addEventListener('click', () => { 2410 2609 stopInstallPoll(); 2610 + stopRuntimePoll(); 2411 2611 stopConfidentialPoll(); 2412 2612 Promise.all([ 2413 2613 refreshProviders(), 2414 2614 refreshLocalAvailability(), 2415 2615 refreshInstallStatus({autoResume: true}), 2616 + refreshLocalRuntime({autoResume: true}), 2416 2617 ]).catch((err) => setMessage('localSetupMessage', err.message, 'error')); 2417 2618 }); 2418 2619 $('localBootstrap')?.addEventListener('click', () => startLocalBootstrap().catch((err) => setMessage('localSetupMessage', err.message, 'error'))); 2620 + $('localRuntimeRetry')?.addEventListener('click', () => retryLocalRuntime().catch((err) => setMessage('localSetupMessage', err.message, 'error'))); 2419 2621 $('localActivate')?.addEventListener('click', () => activateLane('local').catch((err) => setMessage('localSetupMessage', err.message, 'error'))); 2420 2622 $('localModelSelect')?.addEventListener('change', () => { 2421 2623 stopInstallPoll(); 2624 + stopRuntimePoll(); 2422 2625 stopConfidentialPoll(); 2423 2626 state.install = null; 2424 2627 renderAll(); ··· 2426 2629 refreshLocalAvailability(), 2427 2630 refreshProviders(), 2428 2631 refreshInstallStatus({autoResume: true}), 2632 + refreshLocalRuntime({autoResume: true}), 2429 2633 ]).catch((err) => setMessage('localSetupMessage', err.message, 'error')); 2430 2634 }); 2431 2635 $('localEndpointSave')?.addEventListener('click', () => saveLocalEndpoint().catch((err) => setMessage('localEndpointStatus', err.message, 'error'))); ··· 2444 2648 try { 2445 2649 await refreshLocalModels(); 2446 2650 await refreshInstallStatus({autoResume: true}); 2651 + await refreshLocalRuntime({autoResume: viewFromHash() === 'local-setup'}); 2447 2652 await refreshLocalAvailability(); 2448 2653 await Promise.all([refreshProviders(), refreshKeys(), refreshScout()]); 2449 2654 } catch (err) {
+207
solstone/apps/thinking/tests/test_local_runtime_recovery_routes.py
··· 1 + # SPDX-License-Identifier: AGPL-3.0-only 2 + # Copyright (c) 2026 sol pbc 3 + 4 + from __future__ import annotations 5 + 6 + import json 7 + from pathlib import Path 8 + 9 + from solstone.apps.thinking import local_recovery 10 + from solstone.think.providers.runtime_health import ( 11 + make_synthetic_runtime_health, 12 + read_retry_token, 13 + request_retry_token, 14 + runtime_health_path, 15 + write_runtime_health, 16 + ) 17 + 18 + 19 + def _write_failed_runtime(journal_path: Path): 20 + record = make_synthetic_runtime_health("local") 21 + record["phase"] = "failed" 22 + record["reason_code"] = "launch-budget-exhausted" 23 + record["desired_fingerprint_sha256"] = "fp-local" 24 + record["attempt"] = 6 25 + record["detail"] = { 26 + "error": "sensitive backend exception", 27 + "path": "/private/model/path", 28 + } 29 + record["process"] = { 30 + "name": "llama-server", 31 + "pid": 4242, 32 + "ref": "private-ref", 33 + "port": 43210, 34 + } 35 + record["updated_at"] = "2026-07-19T10:00:00+00:00" 36 + return write_runtime_health(record, journal_path=journal_path) 37 + 38 + 39 + def _finish_setup(journal_path: Path, config: dict) -> None: 40 + config["setup"] = {"completed_at": 1700000000000} 41 + (journal_path / "config" / "journal.json").write_text( 42 + json.dumps(config, indent=2) + "\n", 43 + encoding="utf-8", 44 + ) 45 + 46 + 47 + def test_runtime_view_exposes_only_owner_safe_recovery_projection( 48 + settings_env, 49 + ) -> None: 50 + journal_path, config = settings_env() 51 + _finish_setup(journal_path, config) 52 + stored = _write_failed_runtime(journal_path) 53 + 54 + view = local_recovery.runtime_view() 55 + 56 + assert view == { 57 + "status": "ok", 58 + "phase": "failed", 59 + "reason_code": "launch-budget-exhausted", 60 + "health_revision": stored["revision"], 61 + "desired_fingerprint_sha256": "fp-local", 62 + "retry_revision": 0, 63 + "retry_pending": False, 64 + "can_retry": True, 65 + "poll": False, 66 + "updated_at": "2026-07-19T10:00:00+00:00", 67 + } 68 + assert "process" not in view 69 + assert "detail" not in view 70 + assert "token_id" not in view 71 + 72 + 73 + def test_runtime_view_distinguishes_corrupt_and_unavailable_state( 74 + settings_env, 75 + monkeypatch, 76 + ) -> None: 77 + journal_path, config = settings_env() 78 + _finish_setup(journal_path, config) 79 + path = runtime_health_path("local", journal_path=journal_path) 80 + path.parent.mkdir(parents=True, exist_ok=True) 81 + path.write_text("{not-json", encoding="utf-8") 82 + 83 + corrupt = local_recovery.runtime_view() 84 + assert corrupt["status"] == "corrupt" 85 + assert corrupt["phase"] == "state-corrupt" 86 + assert corrupt["can_retry"] is False 87 + 88 + def unavailable(_provider): 89 + return { 90 + "status": "unavailable", 91 + "provider": "local", 92 + "record_kind": "health", 93 + "path": "/not-owner-visible", 94 + "record": None, 95 + "reason_code": "record-unavailable", 96 + "error": "permission denied", 97 + } 98 + 99 + monkeypatch.setattr(local_recovery, "inspect_runtime_health", unavailable) 100 + unavailable_view = local_recovery.runtime_view() 101 + assert unavailable_view["status"] == "unavailable" 102 + assert unavailable_view["phase"] == "state-unavailable" 103 + assert "path" not in unavailable_view 104 + 105 + 106 + def test_runtime_view_does_not_block_current_target_on_old_retry_token( 107 + settings_env, 108 + ) -> None: 109 + journal_path, config = settings_env() 110 + _finish_setup(journal_path, config) 111 + request_retry_token( 112 + "local", 113 + desired_fingerprint_sha256="fp-old", 114 + journal_path=journal_path, 115 + ) 116 + _write_failed_runtime(journal_path) 117 + 118 + view = local_recovery.runtime_view() 119 + 120 + assert view["phase"] == "failed" 121 + assert view["retry_pending"] is False 122 + assert view["can_retry"] is True 123 + 124 + 125 + def test_runtime_retry_route_is_single_use_compare_and_set( 126 + settings_env, 127 + thinking_app, 128 + ) -> None: 129 + from solstone.convey import state 130 + 131 + journal_path, config = settings_env() 132 + _finish_setup(journal_path, config) 133 + state.journal_root = str(journal_path) 134 + _write_failed_runtime(journal_path) 135 + client = thinking_app.test_client() 136 + 137 + current_response = client.get("/app/thinking/api/local/runtime") 138 + assert current_response.status_code == 200 139 + current = current_response.get_json() 140 + assert current["can_retry"] is True 141 + 142 + request_body = { 143 + "health_revision": current["health_revision"], 144 + "retry_revision": current["retry_revision"], 145 + "desired_fingerprint_sha256": current["desired_fingerprint_sha256"], 146 + } 147 + first = client.post( 148 + "/app/thinking/api/local/runtime/retry", 149 + json=request_body, 150 + ) 151 + assert first.status_code == 200 152 + requested = first.get_json() 153 + assert requested["phase"] == "retry-requested" 154 + assert requested["retry_pending"] is True 155 + assert requested["can_retry"] is False 156 + 157 + token = read_retry_token("local", journal_path=journal_path) 158 + assert token["token_id"] is not None 159 + assert token["desired_fingerprint_sha256"] == "fp-local" 160 + 161 + replay = client.post( 162 + "/app/thinking/api/local/runtime/retry", 163 + json=request_body, 164 + ) 165 + assert replay.status_code == 400 166 + assert replay.get_json()["reason_code"] == "invalid_operation_for_state" 167 + assert read_retry_token("local", journal_path=journal_path) == token 168 + 169 + 170 + def test_runtime_retry_route_rejects_untrusted_or_nonterminal_requests( 171 + settings_env, 172 + thinking_app, 173 + ) -> None: 174 + from solstone.convey import state 175 + 176 + journal_path, config = settings_env() 177 + _finish_setup(journal_path, config) 178 + state.journal_root = str(journal_path) 179 + client = thinking_app.test_client() 180 + 181 + missing = client.post("/app/thinking/api/local/runtime/retry") 182 + assert missing.status_code == 400 183 + assert missing.get_json()["reason_code"] == "missing_request_body" 184 + 185 + extra = client.post( 186 + "/app/thinking/api/local/runtime/retry", 187 + json={ 188 + "health_revision": 0, 189 + "retry_revision": 0, 190 + "desired_fingerprint_sha256": "fp-local", 191 + "launch": True, 192 + }, 193 + ) 194 + assert extra.status_code == 400 195 + assert extra.get_json()["reason_code"] == "invalid_request_value" 196 + 197 + stopped = client.post( 198 + "/app/thinking/api/local/runtime/retry", 199 + json={ 200 + "health_revision": 0, 201 + "retry_revision": 0, 202 + "desired_fingerprint_sha256": "fp-local", 203 + }, 204 + ) 205 + assert stopped.status_code == 400 206 + assert stopped.get_json()["reason_code"] == "invalid_operation_for_state" 207 + assert read_retry_token("local", journal_path=journal_path)["token_id"] is None
+2 -1
solstone/apps/thinking/tests/test_thinking_install_poll_js.py
··· 342 342 assert(result.install_state === 'failed', 'failed status should be terminal'); 343 343 assert(fetchCalls === 2, 'poll should stop after failed'); 344 344 assert(sleeps.length === 1, 'poll should sleep only before failed'); 345 - assert(failed.message === serverInstallError, 'server install_error should render verbatim'); 345 + assert(failed.message === '', 'raw backend install_error must stay out of owner copy'); 346 + assert(failed.notice === text.failed_reason, 'failed install should use stable copy'); 346 347 assert(failed.bootstrap === true, 'failed install should offer retry'); 347 348 assert(failed.bootstrapLabel === text.retry, 'retry label should come from copy'); 348 349 assert(failed.tone === 'bad', 'failed install should render as an error');
+217
solstone/apps/thinking/tests/test_thinking_local_runtime_recovery_js.py
··· 1 + # SPDX-License-Identifier: AGPL-3.0-only 2 + # Copyright (c) 2026 sol pbc 3 + 4 + from __future__ import annotations 5 + 6 + import json 7 + import shutil 8 + import subprocess 9 + from pathlib import Path 10 + 11 + import pytest 12 + 13 + from solstone.apps.thinking import copy as thinking_copy 14 + from solstone.apps.thinking.tests.js_extract import extract_js_function 15 + 16 + STATIC = Path(__file__).resolve().parents[1] / "static" / "thinking.js" 17 + 18 + 19 + def _run_node(body: str) -> None: 20 + node = shutil.which("node") 21 + if node is None: 22 + pytest.skip("node is not available") 23 + source = STATIC.read_text(encoding="utf-8") 24 + script = "\n".join( 25 + [ 26 + extract_js_function(source, "localRuntimeCopy"), 27 + extract_js_function(source, "pollLocalRuntimeUntilStable"), 28 + extract_js_function(source, "applyLocalRuntime"), 29 + "function assert(condition, message) { if (!condition) throw new Error(message); }", 30 + f"const text = {json.dumps(thinking_copy.LOCAL_RECOVERY)};", 31 + body, 32 + ] 33 + ) 34 + subprocess.run( 35 + [node, "-e", script], 36 + check=True, 37 + text=True, 38 + capture_output=True, 39 + ) 40 + 41 + 42 + def test_runtime_copy_has_one_honest_action_for_terminal_failure() -> None: 43 + _run_node( 44 + """ 45 + const failed = localRuntimeCopy({ 46 + status: 'ok', 47 + phase: 'failed', 48 + reason_code: 'launch-budget-exhausted', 49 + can_retry: true, 50 + }, true, text); 51 + assert(failed.pill === 'needs attention', 'failed pill'); 52 + assert(failed.sub === "local thinking couldn't start", 'failed verdict'); 53 + assert(failed.retryRuntime === true, 'failed state should expose retry'); 54 + assert(failed.retryRuntimeLabel === 'try starting local again', 'component action'); 55 + assert(!failed.bootstrap && !failed.activate, 'failure should have one action'); 56 + 57 + const requested = localRuntimeCopy({ 58 + status: 'ok', 59 + phase: 'retry-requested', 60 + can_retry: false, 61 + }, true, text); 62 + assert(requested.pill === 'retrying', 'retry request should be visible'); 63 + assert(requested.retryRuntime === false, 'retry request cannot double spend'); 64 + """ 65 + ) 66 + 67 + 68 + def test_runtime_copy_distinguishes_transient_ready_and_fail_closed_states() -> None: 69 + _run_node( 70 + """ 71 + const starting = localRuntimeCopy({status: 'ok', phase: 'warming'}, true, text); 72 + assert(starting.pill === 'starting', 'warming should say starting'); 73 + assert(starting.retryRuntime === false, 'warming is non-actionable'); 74 + 75 + const waiting = localRuntimeCopy({ 76 + status: 'ok', 77 + phase: 'host-blocked', 78 + reason_code: 'ram-insufficient', 79 + }, true, text); 80 + assert(waiting.pill === 'waiting', 'capacity pressure is recheckable'); 81 + 82 + const unsupported = localRuntimeCopy({ 83 + status: 'ok', 84 + phase: 'host-blocked', 85 + reason_code: 'platform-unsupported', 86 + }, true, text); 87 + assert(unsupported.pill === 'unavailable', 'permanent host block is distinct'); 88 + 89 + const ready = localRuntimeCopy({status: 'ok', phase: 'ready'}, true, text); 90 + assert(ready.pill === 'on', 'ready is earned'); 91 + assert(ready.sub === 'local thinking is ready', 'ready verdict'); 92 + 93 + const degraded = localRuntimeCopy({ 94 + status: 'ok', 95 + phase: 'ready-proof-unavailable', 96 + }, true, text); 97 + assert(degraded.pill === 'on, needs a check', 'live process stays on'); 98 + assert(degraded.retryRuntime === false, 'degraded ready must not replace'); 99 + 100 + const corrupt = localRuntimeCopy({ 101 + status: 'corrupt', 102 + phase: 'state-corrupt', 103 + }, true, text); 104 + assert(corrupt.pill === "can't verify", 'corrupt state fails closed'); 105 + assert(corrupt.retryRuntime === false, 'corrupt state cannot retry'); 106 + """ 107 + ) 108 + 109 + 110 + def test_runtime_copy_does_not_override_inactive_artifact_setup() -> None: 111 + _run_node( 112 + """ 113 + assert( 114 + localRuntimeCopy({status: 'ok', phase: 'stopped'}, false, text) === null, 115 + 'inactive stopped runtime should leave artifact/activation rendering alone', 116 + ); 117 + assert( 118 + localRuntimeCopy({status: 'ok', phase: 'artifact-not-ready'}, true, text) === null, 119 + 'artifact owner should render artifact recovery', 120 + ); 121 + """ 122 + ) 123 + 124 + 125 + def test_runtime_poll_stops_at_truthful_terminal_state_and_fences_navigation() -> None: 126 + _run_node( 127 + """ 128 + async function main() { 129 + const statuses = [ 130 + {phase: 'warming', poll: true}, 131 + {phase: 'ready', poll: false}, 132 + ]; 133 + let index = 0; 134 + const applied = []; 135 + const result = await pollLocalRuntimeUntilStable({ 136 + fetchStatus: async () => statuses[index++], 137 + sleepFn: async () => {}, 138 + applyStatus: (status) => applied.push(status.phase), 139 + isCurrent: () => true, 140 + intervalMs: 1500, 141 + initialStatus: {phase: 'starting', poll: true}, 142 + }); 143 + assert(result.phase === 'ready', 'poll should return ready'); 144 + assert(JSON.stringify(applied) === JSON.stringify(['warming', 'ready']), 'phase order'); 145 + 146 + let fetched = false; 147 + const cancelled = await pollLocalRuntimeUntilStable({ 148 + fetchStatus: async () => { fetched = true; return {phase: 'ready', poll: false}; }, 149 + sleepFn: async () => {}, 150 + applyStatus: () => {}, 151 + isCurrent: () => false, 152 + intervalMs: 1500, 153 + initialStatus: {phase: 'warming', poll: true}, 154 + }); 155 + assert(cancelled.phase === 'warming', 'cancelled poll keeps its last state'); 156 + assert(fetched === false, 'cancelled poll must not fetch'); 157 + } 158 + main().catch((error) => { console.error(error.stack || error); process.exit(1); }); 159 + """ 160 + ) 161 + 162 + 163 + def test_runtime_apply_fences_health_and_retry_revision_races() -> None: 164 + _run_node( 165 + """ 166 + const state = { 167 + providers: { 168 + local_runtime: { 169 + health_revision: 8, 170 + retry_revision: 4, 171 + phase: 'retry-requested', 172 + }, 173 + }, 174 + runtimePollGeneration: 3, 175 + }; 176 + let renders = 0; 177 + function renderAll() { renders += 1; } 178 + 179 + assert( 180 + applyLocalRuntime({ 181 + health_revision: 8, 182 + retry_revision: 3, 183 + phase: 'failed', 184 + }) === false, 185 + 'older retry truth must not restore a spent retry action', 186 + ); 187 + assert(state.providers.local_runtime.phase === 'retry-requested', 'retry state preserved'); 188 + assert(renders === 0, 'stale retry truth must not render'); 189 + 190 + assert( 191 + applyLocalRuntime({ 192 + health_revision: 7, 193 + retry_revision: 5, 194 + phase: 'failed', 195 + }) === false, 196 + 'older health truth must stay fenced', 197 + ); 198 + assert( 199 + applyLocalRuntime({ 200 + health_revision: 9, 201 + retry_revision: 5, 202 + phase: 'starting', 203 + }, 2) === false, 204 + 'an obsolete poll generation must stay fenced', 205 + ); 206 + assert( 207 + applyLocalRuntime({ 208 + health_revision: 9, 209 + retry_revision: 5, 210 + phase: 'starting', 211 + }, 3) === true, 212 + 'current monotonic truth should apply', 213 + ); 214 + assert(state.providers.local_runtime.phase === 'starting', 'fresh truth applied'); 215 + assert(renders === 1, 'fresh truth renders once'); 216 + """ 217 + )
+12 -1
solstone/apps/thinking/tests/test_workspace_html.py
··· 88 88 assert 'id="lane-detail-confidential"' in html 89 89 assert 'data-open-view="byo-setup"' in html 90 90 assert 'data-open-view="local-setup"' in html 91 + assert 'id="localSetupCard" aria-live="polite" aria-atomic="true"' in html 92 + assert 'id="localRuntimeRetry"' in html 91 93 assert "data-switch-lane" in html 92 94 assert html.count('id="byoLaneStatus"') == 1 93 95 assert 'id="byoKeyStatus"' in html ··· 379 381 }, 380 382 "pill_inflight": "setting up", 381 383 "pill_failed": "couldn't finish", 382 - "retry": "try again", 384 + "failed_verdict": "local setup didn't finish", 385 + "failed_reason": "local setup stopped before it finished.", 386 + "retry": "try setup again", 383 387 "install": "install local model", 384 388 "notice_inflight": ( 385 389 "local thinking will stay in your journal once setup finishes." ··· 505 509 assert payload["local_install"] == { 506 510 **thinking_copy.LOCAL_INSTALL, 507 511 "phases": dict(thinking_copy.LOCAL_INSTALL["phases"]), 512 + } 513 + assert payload["local_recovery"] == { 514 + "retry": thinking_copy.LOCAL_RECOVERY["retry"], 515 + "states": { 516 + key: dict(value) 517 + for key, value in thinking_copy.LOCAL_RECOVERY["states"].items() 518 + }, 508 519 } 509 520 assert "byo" not in payload 510 521
+2 -1
solstone/apps/thinking/workspace.html
··· 803 803 <p class="intro">run a thinking model right in your journal — nothing leaves this computer.</p> 804 804 </div> 805 805 806 - <div class="app-onoff" id="localSetupCard"> 806 + <div class="app-onoff" id="localSetupCard" aria-live="polite" aria-atomic="true"> 807 807 <div class="cardtop"> 808 808 <h3 id="localSetupTitle">local</h3> 809 809 <span class="pill" id="localSetupPill">checking</span> ··· 825 825 <div class="actions"> 826 826 <button type="button" class="btn" id="localRefresh">check again</button> 827 827 <button type="button" class="btn" id="localBootstrap"></button> 828 + <button type="button" class="btn" id="localRuntimeRetry">try starting local again</button> 828 829 <button type="button" class="btn primary" id="localActivate">turn on local</button> 829 830 </div> 830 831 <p class="micro" id="localSetupLinks"></p>
+63
solstone/think/providers/runtime_health.py
··· 487 487 ) from exc 488 488 489 489 490 + def request_runtime_retry( 491 + provider: str, 492 + *, 493 + expected_health_revision: int, 494 + expected_retry_revision: int, 495 + desired_fingerprint_sha256: str, 496 + owner: dict[str, Any] | None = None, 497 + journal_path: str | Path | None = None, 498 + ) -> RuntimeRetryTokenRecord: 499 + """Request one retry for a current terminal runtime failure. 500 + 501 + This is the owner-facing compare-and-set operation. Internal supervisor 502 + recovery may still use ``request_retry_token`` directly, but routes must 503 + not compose health and token reads around that lower-level primitive. 504 + """ 505 + validated = _validate_provider(provider) 506 + _validate_owner(owner) 507 + health_path = runtime_health_path(validated, journal_path=journal_path) 508 + retry_path = runtime_retry_token_path(validated, journal_path=journal_path) 509 + lock_target = runtime_operation_path(validated, journal_path=journal_path) 510 + try: 511 + with hold_lock(lock_target, mode=RUNTIME_RECORD_MODE): 512 + health = _read_health_unlocked(health_path, validated) 513 + retry = _read_retry_unlocked(retry_path, validated) 514 + if health["revision"] != expected_health_revision: 515 + raise RuntimeHealthConflictError("stale runtime health revision") 516 + if retry["revision"] != expected_retry_revision: 517 + raise RuntimeHealthConflictError("stale retry-token revision") 518 + if health["desired_fingerprint_sha256"] != desired_fingerprint_sha256: 519 + raise RuntimeHealthConflictError("runtime desired fingerprint changed") 520 + if health["phase"] != "failed": 521 + raise RuntimeHealthConflictError( 522 + "runtime retry requires a terminal failure" 523 + ) 524 + if ( 525 + retry["token_id"] is not None 526 + and retry["desired_fingerprint_sha256"] == desired_fingerprint_sha256 527 + ): 528 + raise RuntimeHealthConflictError("runtime retry already requested") 529 + 530 + stored: RuntimeRetryTokenRecord = { 531 + "schema_version": SCHEMA_VERSION, 532 + "provider": validated, 533 + "revision": retry["revision"] + 1, 534 + "token_id": uuid.uuid4().hex, 535 + "desired_fingerprint_sha256": desired_fingerprint_sha256, 536 + "requested_at": now_iso(), 537 + "reason_code": "retry-token-requested", 538 + "owner": owner, 539 + } 540 + _write_json_unlocked(retry_path, _persistable_retry(stored)) 541 + return stored 542 + except LockTimeout as exc: 543 + raise RuntimeHealthUnavailableError( 544 + f"runtime retry lock unavailable: {lock_target}" 545 + ) from exc 546 + except OSError as exc: 547 + raise RuntimeHealthUnavailableError( 548 + f"runtime retry write unavailable: {retry_path}" 549 + ) from exc 550 + 551 + 490 552 def consume_retry_token( 491 553 provider: str, 492 554 *, ··· 987 1049 "read_runtime_health", 988 1050 "repair_corrupt_record", 989 1051 "request_retry_token", 1052 + "request_runtime_retry", 990 1053 "runtime_directory", 991 1054 "runtime_health_path", 992 1055 "runtime_operation_lock_path",
+29 -18
solstone/think/supervisor.py
··· 5404 5404 return 5405 5405 if token["desired_fingerprint_sha256"] not in {None, state.desired_fingerprint}: 5406 5406 return 5407 + if state.latest_phase in { 5408 + "not-desired", 5409 + "artifact-not-ready", 5410 + "host-blocked", 5411 + "stopped", 5412 + "failed", 5413 + "backoff", 5414 + "observing", 5415 + }: 5416 + retry_phase: RuntimePhase = "observing" 5417 + else: 5418 + retry_phase = "retry-requested" 5419 + 5420 + # Publish the nonterminal transition before clearing the token. Both writes 5421 + # use the provider operation lock, so an owner retry can never observe a 5422 + # cleared token while the durable health record still accepts another 5423 + # terminal-failure request. 5424 + state.latest_phase = retry_phase 5425 + if ( 5426 + _write_provider_runtime( 5427 + state, 5428 + phase=retry_phase, 5429 + reason_code="retry-token-requested", 5430 + detail={"token_revision": token["revision"]}, 5431 + ) 5432 + is None 5433 + ): 5434 + return 5407 5435 try: 5408 5436 consume_retry_token( 5409 5437 state.provider, ··· 5420 5448 if state.provider == "parakeet": 5421 5449 _parakeet_admission_retry_epoch += 1 5422 5450 state.retry = ProviderRetryState(desired_fingerprint=state.desired_fingerprint) 5423 - if state.latest_phase in { 5424 - "not-desired", 5425 - "artifact-not-ready", 5426 - "host-blocked", 5427 - "stopped", 5428 - "failed", 5429 - "backoff", 5430 - "observing", 5431 - }: 5432 - state.latest_phase = "observing" 5433 - else: 5434 - state.latest_phase = "retry-requested" 5451 + state.latest_phase = retry_phase 5435 5452 state.next_truth_at = 0.0 5436 - _write_provider_runtime( 5437 - state, 5438 - phase=state.latest_phase, 5439 - reason_code="retry-token-requested", 5440 - detail={"token_revision": token["revision"]}, 5441 - ) 5442 5453 5443 5454 5444 5455 async def _reconcile_provider_runtime(
+12
tests/baselines/api/thinking/providers.json
··· 125 125 "endpoint_url": "", 126 126 "served_model_id": "" 127 127 }, 128 + "local_runtime": { 129 + "can_retry": false, 130 + "desired_fingerprint_sha256": null, 131 + "health_revision": 0, 132 + "phase": "stopped", 133 + "poll": false, 134 + "reason_code": null, 135 + "retry_pending": false, 136 + "retry_revision": 0, 137 + "status": "ok", 138 + "updated_at": null 139 + }, 128 140 "model_tiers": { 129 141 "anthropic": [ 130 142 {
+118
tests/test_provider_runtime_health.py
··· 27 27 read_runtime_health, 28 28 repair_corrupt_record, 29 29 request_retry_token, 30 + request_runtime_retry, 30 31 runtime_directory, 31 32 runtime_health_path, 32 33 runtime_operation_lock_path, ··· 417 418 assert consumed["token_id"] is None 418 419 assert consumed["revision"] == second["revision"] + 1 419 420 assert read_retry_token("local", journal_path=tmp_path) == consumed 421 + 422 + 423 + def test_owner_runtime_retry_requires_current_terminal_failure(tmp_path: Path) -> None: 424 + failed = _health(phase="failed") 425 + failed["reason_code"] = "launch-budget-exhausted" 426 + stored = write_runtime_health(failed, journal_path=tmp_path) 427 + 428 + requested = request_runtime_retry( 429 + "local", 430 + expected_health_revision=stored["revision"], 431 + expected_retry_revision=0, 432 + desired_fingerprint_sha256="fp-1", 433 + owner={"source": "owner-recovery"}, 434 + journal_path=tmp_path, 435 + ) 436 + 437 + assert requested["revision"] == 1 438 + assert requested["token_id"] is not None 439 + assert requested["desired_fingerprint_sha256"] == "fp-1" 440 + assert requested["reason_code"] == "retry-token-requested" 441 + assert requested["owner"] == {"source": "owner-recovery"} 442 + assert read_runtime_health("local", journal_path=tmp_path) == stored 443 + 444 + 445 + @pytest.mark.parametrize("phase", ["ready", "starting", "backoff", "host-blocked"]) 446 + def test_owner_runtime_retry_rejects_nonterminal_state( 447 + tmp_path: Path, 448 + phase: str, 449 + ) -> None: 450 + health = _health(phase=phase) 451 + stored = write_runtime_health(health, journal_path=tmp_path) 452 + 453 + with pytest.raises( 454 + RuntimeHealthConflictError, 455 + match="terminal failure", 456 + ): 457 + request_runtime_retry( 458 + "local", 459 + expected_health_revision=stored["revision"], 460 + expected_retry_revision=0, 461 + desired_fingerprint_sha256="fp-1", 462 + journal_path=tmp_path, 463 + ) 464 + 465 + assert read_retry_token("local", journal_path=tmp_path)["token_id"] is None 466 + 467 + 468 + def test_owner_runtime_retry_rejects_stale_and_outstanding_requests( 469 + tmp_path: Path, 470 + ) -> None: 471 + failed = _health(phase="failed") 472 + failed["reason_code"] = "launch-budget-exhausted" 473 + stored = write_runtime_health(failed, journal_path=tmp_path) 474 + 475 + with pytest.raises(RuntimeHealthConflictError, match="health revision"): 476 + request_runtime_retry( 477 + "local", 478 + expected_health_revision=stored["revision"] - 1, 479 + expected_retry_revision=0, 480 + desired_fingerprint_sha256="fp-1", 481 + journal_path=tmp_path, 482 + ) 483 + with pytest.raises(RuntimeHealthConflictError, match="fingerprint"): 484 + request_runtime_retry( 485 + "local", 486 + expected_health_revision=stored["revision"], 487 + expected_retry_revision=0, 488 + desired_fingerprint_sha256="fp-stale", 489 + journal_path=tmp_path, 490 + ) 491 + 492 + requested = request_runtime_retry( 493 + "local", 494 + expected_health_revision=stored["revision"], 495 + expected_retry_revision=0, 496 + desired_fingerprint_sha256="fp-1", 497 + journal_path=tmp_path, 498 + ) 499 + with pytest.raises(RuntimeHealthConflictError, match="retry-token revision"): 500 + request_runtime_retry( 501 + "local", 502 + expected_health_revision=stored["revision"], 503 + expected_retry_revision=0, 504 + desired_fingerprint_sha256="fp-1", 505 + journal_path=tmp_path, 506 + ) 507 + with pytest.raises(RuntimeHealthConflictError, match="already requested"): 508 + request_runtime_retry( 509 + "local", 510 + expected_health_revision=stored["revision"], 511 + expected_retry_revision=requested["revision"], 512 + desired_fingerprint_sha256="fp-1", 513 + journal_path=tmp_path, 514 + ) 515 + 516 + 517 + def test_owner_runtime_retry_replaces_an_old_target_token(tmp_path: Path) -> None: 518 + stale = request_retry_token( 519 + "local", 520 + desired_fingerprint_sha256="fp-old", 521 + journal_path=tmp_path, 522 + ) 523 + failed = _health(phase="failed") 524 + failed["reason_code"] = "launch-budget-exhausted" 525 + stored = write_runtime_health(failed, journal_path=tmp_path) 526 + 527 + requested = request_runtime_retry( 528 + "local", 529 + expected_health_revision=stored["revision"], 530 + expected_retry_revision=stale["revision"], 531 + desired_fingerprint_sha256="fp-1", 532 + journal_path=tmp_path, 533 + ) 534 + 535 + assert requested["token_id"] != stale["token_id"] 536 + assert requested["desired_fingerprint_sha256"] == "fp-1" 537 + assert requested["revision"] == stale["revision"] + 1 420 538 421 539 422 540 def test_repair_handle_allows_repair_without_parseable_token(tmp_path: Path) -> None:
+50
tests/test_supervisor_provider_runtime.py
··· 30 30 read_retry_token, 31 31 read_runtime_health, 32 32 request_retry_token, 33 + request_runtime_retry, 33 34 write_runtime_health, 34 35 ) 35 36 ··· 921 922 assert state.retry.attempt_count == 0 922 923 assert state.latest_phase == "stopped" 923 924 assert observations == 1 925 + 926 + 927 + def test_owner_retry_is_nonterminal_before_token_is_cleared(monkeypatch) -> None: 928 + state = supervisor._provider_runtime_states["local"] 929 + state.latest_phase = "failed" 930 + state.desired_fingerprint = "fp-local" 931 + state.retry = supervisor.ProviderRetryState( 932 + attempt_count=len(supervisor.PROVIDER_RETRY_SCHEDULE_SECONDS), 933 + desired_fingerprint="fp-local", 934 + ) 935 + failed = supervisor._write_provider_runtime( 936 + state, 937 + phase="failed", 938 + reason_code="launch-budget-exhausted", 939 + detail={}, 940 + ) 941 + assert failed is not None 942 + token = request_runtime_retry( 943 + "local", 944 + expected_health_revision=failed["revision"], 945 + expected_retry_revision=0, 946 + desired_fingerprint_sha256="fp-local", 947 + ) 948 + 949 + real_consume = supervisor.consume_retry_token 950 + 951 + def assert_nonterminal_before_consume(*args, **kwargs): 952 + health = read_runtime_health("local") 953 + assert health["phase"] == "observing" 954 + with pytest.raises( 955 + supervisor.RuntimeHealthConflictError, 956 + match="terminal failure", 957 + ): 958 + request_runtime_retry( 959 + "local", 960 + expected_health_revision=health["revision"], 961 + expected_retry_revision=token["revision"], 962 + desired_fingerprint_sha256="fp-local", 963 + ) 964 + return real_consume(*args, **kwargs) 965 + 966 + monkeypatch.setattr( 967 + supervisor, "consume_retry_token", assert_nonterminal_before_consume 968 + ) 969 + 970 + supervisor._handle_provider_retry_token(state) 971 + 972 + assert read_retry_token("local")["token_id"] is None 973 + assert state.retry.attempt_count == 0 924 974 925 975 926 976 @pytest.mark.parametrize(