solpbc.org / solstone
personal memory agent
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

fix(think): contain local output tails

Cap the Qwen-heavy describe and Sense output paths so truncated local runs have less tail room to leak partial output, while leaving the category loader default unchanged. Add explicit Sense collection bounds that survive runtime enum hydration and local GBNF schema prep instead of falling back to the global array cap.

Normalize local finish reasons and make the Batch boundary reject any present non-stop finish while preserving hard schema-invalid JSON failures. This keeps truncated, malformed, timed-out, or otherwise bad local responses from landing as normal-looking journal artifacts at frame categorization, browsing extraction, and Segment Sense boundaries.

Carry local JSON length retry counts into the terminal error when the retry itself fails.

Co-Authored-By: OpenAI Codex <codex@openai.com>

author
Jer Miller co-author
OpenAI Codex date (Jul 11, 2026, 8:49 AM -0600) commit d81bb318 parent 2423c6d4
+856 -146
+6 -3
solstone/apps/timeline/tests/conftest.py
··· 113 113 114 114 async def _fake_agenerate(**kwargs): 115 115 if not responses: 116 - return json.dumps({"picks": [0], "rationale": "default"}) 116 + return { 117 + "text": json.dumps({"picks": [0], "rationale": "default"}), 118 + "finish_reason": "stop", 119 + } 117 120 item = responses.pop(0) 118 121 if isinstance(item, Exception): 119 122 raise item 120 - return json.dumps(item) 123 + return {"text": json.dumps(item), "finish_reason": "stop"} 121 124 122 125 mock = AsyncMock(side_effect=_fake_agenerate) 123 - monkeypatch.setattr("solstone.think.batch.agenerate", mock) 126 + monkeypatch.setattr("solstone.think.batch.agenerate_with_result", mock) 124 127 return mock 125 128 126 129 return _install
+9
solstone/convey/provider_readiness.py
··· 274 274 ), 275 275 recovery_action=None, 276 276 ), 277 + "incomplete_text_length": _Entry( 278 + klass="generic", 279 + summary="the answer ran out of room before it finished", 280 + detail=( 281 + "The reply hit its length limit before it could finish. Try again " 282 + "with less at once or choose another provider." 283 + ), 284 + recovery_action=None, 285 + ), 277 286 "max_turns_exhausted": _Entry( 278 287 klass="generic", 279 288 summary="this took too many steps to finish",
+4
solstone/convey/static/chat_reasons.js
··· 122 122 "template": "the answer ran out of room before it finished", 123 123 "action": null 124 124 }, 125 + "incomplete_text_length": { 126 + "template": "the answer ran out of room before it finished", 127 + "action": null 128 + }, 125 129 "max_turns_exhausted": { 126 130 "template": "this took too many steps to finish", 127 131 "action": null
+2 -1
solstone/observe/categories/browsing.md
··· 2 2 3 3 "description": "General web browsing, news, shopping, or reference pages without a dominant social feed or media viewer", 4 4 "output": "markdown", 5 - "extraction": "Extract when visiting distinctly different websites or search results" 5 + "extraction": "Extract when visiting distinctly different websites or search results", 6 + "max_output_tokens": 2048 6 7 7 8 } 8 9
+1 -1
solstone/observe/describe.py
··· 792 792 json_output=True, 793 793 json_schema=_SCHEMA, 794 794 temperature=0.7, 795 - max_output_tokens=1024, 795 + max_output_tokens=512, 796 796 thinking_budget=1024, 797 797 ) 798 798
+1 -1
solstone/talent/sense.md
··· 9 9 "tier": 3, 10 10 "output": "json", 11 11 "schema": "sense.schema.json", 12 - "max_output_tokens": 12288, 12 + "max_output_tokens": 6144, 13 13 "timeout_s": 480, 14 14 "load": {"transcripts": true, "percepts": true, "talents": false} 15 15 }
+3
solstone/talent/sense.schema.json
··· 49 49 }, 50 50 "entities": { 51 51 "type": "array", 52 + "maxItems": 96, 52 53 "items": { 53 54 "type": "object", 54 55 "additionalProperties": false, ··· 106 107 }, 107 108 "facets": { 108 109 "type": "array", 110 + "maxItems": 16, 109 111 "items": { 110 112 "type": "object", 111 113 "additionalProperties": false, ··· 146 148 }, 147 149 "speakers": { 148 150 "type": "array", 151 + "maxItems": 16, 149 152 "items": { 150 153 "type": "string" 151 154 }
+23 -6
solstone/think/batch.py
··· 6 6 7 7 Provides Batch for concurrent execution of multiple LLM API calls 8 8 with dynamic request queuing and result streaming via async iterator. 9 - Routes requests to providers based on context via the unified agenerate() API. 9 + Routes requests to providers based on context via the unified async generate API. 10 10 11 11 Example: 12 12 batch = Batch(max_concurrent=5) ··· 26 26 import time 27 27 from typing import Any, List, Optional, Union 28 28 29 - from solstone.think.models import agenerate, resolve_provider 29 + from solstone.think.models import ( 30 + SchemaValidationError, 31 + agenerate_with_result, 32 + finish_reason_error, 33 + resolve_provider, 34 + ) 30 35 from solstone.think.providers.shared import classify_provider_error 31 36 32 37 ··· 34 39 """ 35 40 Mutable request object for a single LLM API call. 36 41 37 - Core attributes are passed to agenerate(). Callers can add 42 + Core attributes are passed to agenerate_with_result(). Callers can add 38 43 arbitrary attributes for tracking (e.g., frame_id, stage, etc). 39 44 40 45 After execution, these attributes are populated: ··· 83 88 Async batch processor for LLM API requests. 84 89 85 90 Manages concurrent execution with dynamic request queuing and result 86 - streaming via async iterator pattern. Routes to providers via agenerate(). 91 + streaming via async iterator pattern. Routes to providers via async generation. 87 92 88 93 Example: 89 94 batch = Batch(max_concurrent=5) ··· 265 270 if request.model is not None: 266 271 kwargs["model"] = request.model 267 272 268 - response = await agenerate( 273 + result = await agenerate_with_result( 269 274 contents=request.contents, 270 275 context=request.context, 271 276 temperature=request.temperature, ··· 277 282 timeout_s=request.timeout_s, 278 283 **kwargs, 279 284 ) 285 + error = finish_reason_error( 286 + result, 287 + json_output=request.json_output, 288 + ) 289 + if error is not None: 290 + raise error 291 + validation = result.get("schema_validation") 292 + if isinstance(validation, dict) and validation.get("valid") is False: 293 + raise SchemaValidationError( 294 + validation.get("errors") or [], 295 + result.get("text", ""), 296 + ) 280 297 request.duration = time.time() - start_time 281 - request.response = response 298 + request.response = result["text"] 282 299 request.error = None 283 300 284 301 # Track which model was actually used
+123 -8
solstone/think/models.py
··· 232 232 super().__init__(f"JSON response incomplete (reason: {reason})") 233 233 234 234 235 + class IncompleteTextError(ValueError): 236 + """Raised when a non-JSON response is truncated due to token limits.""" 237 + 238 + def __init__(self, reason: str, partial_text: str): 239 + self.reason = reason 240 + self.partial_text = partial_text 241 + self.reason_code = "incomplete_text_length" 242 + super().__init__(f"Text response incomplete (reason: {reason})") 243 + 244 + 245 + class ProviderResponseInvalidError(ValueError): 246 + """Raised when a provider reports a non-success finish for plain text.""" 247 + 248 + reason_code = "provider_response_invalid" 249 + 250 + def __init__(self, reason: str): 251 + self.reason = reason 252 + super().__init__(f"Provider response did not finish cleanly (reason: {reason})") 253 + 254 + 235 255 class SchemaValidationError(ValueError): 236 256 """Raised when JSON response text fails local schema validation. 237 257 ··· 1293 1313 # --------------------------------------------------------------------------- 1294 1314 1295 1315 1316 + def finish_reason_error( 1317 + result: Dict[str, Any], 1318 + *, 1319 + json_output: bool, 1320 + ) -> Exception | None: 1321 + """Map a finish reason to the error it should raise, or None if acceptable.""" 1322 + finish_reason = result.get("finish_reason") 1323 + if not finish_reason or finish_reason == "stop": 1324 + return None 1325 + 1326 + if json_output: 1327 + return IncompleteJSONError( 1328 + reason=finish_reason, 1329 + partial_text=result.get("text", ""), 1330 + ) 1331 + 1332 + if str(finish_reason).strip().lower() in _LENGTH_FINISH_REASONS: 1333 + return IncompleteTextError( 1334 + reason=finish_reason, 1335 + partial_text=result.get("text", ""), 1336 + ) 1337 + return ProviderResponseInvalidError(reason=finish_reason) 1338 + 1339 + 1296 1340 def _validate_json_response(result: Dict[str, Any], json_output: bool) -> None: 1297 1341 """Validate response for JSON output mode. 1298 1342 1299 - Raises IncompleteJSONError if finish_reason indicates truncation. 1343 + Raises IncompleteJSONError if finish_reason is a present non-stop value. 1300 1344 """ 1345 + # Non-JSON generate() callers (planner, depict, importers, enrich, extract, 1346 + # transcribe, detect_*) keep today's leniency; the Batch boundary is strict. 1301 1347 if not json_output: 1302 1348 return 1303 - 1304 - finish_reason = result.get("finish_reason") 1305 - if finish_reason and finish_reason != "stop": 1306 - raise IncompleteJSONError( 1307 - reason=finish_reason, 1308 - partial_text=result.get("text", ""), 1309 - ) 1349 + error = finish_reason_error(result, json_output=True) 1350 + if error is not None: 1351 + raise error 1310 1352 1311 1353 1312 1354 def _validate_schema(text: str, schema: dict) -> dict: ··· 1717 1759 return result 1718 1760 1719 1761 1762 + async def agenerate_with_result( 1763 + contents: Union[str, List[Any]], 1764 + context: str, 1765 + temperature: float = 0.3, 1766 + max_output_tokens: int = 8192 * 2, 1767 + system_instruction: Optional[str] = None, 1768 + json_output: bool = False, 1769 + *, 1770 + json_schema: dict | None = None, 1771 + thinking_budget: Optional[int] = None, 1772 + timeout_s: Optional[float] = None, 1773 + **kwargs: Any, 1774 + ) -> dict: 1775 + """Async generate text and return the full GenerateResult dict.""" 1776 + from solstone.think.providers import get_provider_module 1777 + 1778 + if json_schema is not None: 1779 + json_output = True 1780 + 1781 + model_override = kwargs.pop("model", None) 1782 + provider_override = kwargs.pop("provider", None) 1783 + 1784 + provider, model = resolve_provider(context, "generate") 1785 + if provider_override: 1786 + provider = provider_override 1787 + if not model_override: 1788 + model = resolve_model_for_provider(context, provider, "generate") 1789 + if model_override: 1790 + model = model_override 1791 + 1792 + _raise_if_no_brain(provider) 1793 + _reject_local_cloud_model_override(provider, model_override) 1794 + _raise_if_confidential_unverified() 1795 + 1796 + provider_mod = get_provider_module(provider) 1797 + provider_schema = prepare_provider_schema(json_schema, provider) 1798 + 1799 + timeout_s = DEFAULT_PROVIDER_TIMEOUT_S if timeout_s is None else timeout_s 1800 + 1801 + result = await provider_mod.run_agenerate( 1802 + contents=contents, 1803 + model=model, 1804 + provider=provider, 1805 + temperature=temperature, 1806 + max_output_tokens=max_output_tokens, 1807 + system_instruction=system_instruction, 1808 + json_output=json_output, 1809 + json_schema=provider_schema, 1810 + thinking_budget=thinking_budget, 1811 + timeout_s=timeout_s, 1812 + **kwargs, 1813 + ) 1814 + 1815 + if result.get("usage"): 1816 + log_token_usage( 1817 + model=result.get("model") or model, 1818 + usage=result["usage"], 1819 + context=context, 1820 + type="generate", 1821 + ) 1822 + 1823 + _validate_json_response(result, json_output) 1824 + 1825 + if json_schema is not None: 1826 + result["schema_validation"] = _validate_schema(result["text"], json_schema) 1827 + 1828 + return result 1829 + 1830 + 1720 1831 async def agenerate( 1721 1832 contents: Union[str, List[Any]], 1722 1833 context: str, ··· 1855 1966 "generate", 1856 1967 "generate_with_result", 1857 1968 "agenerate", 1969 + "agenerate_with_result", 1970 + "finish_reason_error", 1971 + "IncompleteTextError", 1972 + "ProviderResponseInvalidError", 1858 1973 "resolve_provider", 1859 1974 "resolve_effective_route", 1860 1975 "is_local_provider_needed",
+29 -1
solstone/think/providers/local.py
··· 60 60 _QWEN_TOP_K = 20 61 61 _QWEN_MIN_P = 0.0 62 62 _QWEN_PRESENCE_PENALTY = 1.5 63 + _LOCAL_FINISH_REASON_MAP = { 64 + "stop": "stop", 65 + "length": "max_tokens", 66 + "max_tokens": "max_tokens", 67 + "content_filter": "content_filter", 68 + } 69 + _LOCAL_UNSUPPORTED_FINISH_REASONS = frozenset({"tool_calls", "function_call"}) 63 70 64 71 65 72 @dataclass(frozen=True) ··· 280 287 return normalized 281 288 282 289 290 + def _normalize_finish_reason(raw: Any) -> str: 291 + if not isinstance(raw, str) or not raw.strip(): 292 + raise LocalProviderError( 293 + "provider_response_invalid", 294 + "Local model response did not include a finish reason.", 295 + ) 296 + reason = raw.strip().lower() 297 + normalized = _LOCAL_FINISH_REASON_MAP.get(reason) 298 + if normalized is not None: 299 + return normalized 300 + if reason in _LOCAL_UNSUPPORTED_FINISH_REASONS: 301 + raise LocalProviderError( 302 + "provider_response_invalid", 303 + f"Local model returned unsupported finish reason: {reason}", 304 + ) 305 + raise LocalProviderError( 306 + "provider_response_invalid", 307 + f"Local model returned unknown finish reason: {reason}", 308 + ) 309 + 310 + 283 311 def _parse_response(data: dict[str, Any]) -> GenerateResult: 284 312 choices = data.get("choices") 285 313 if not isinstance(choices, list) or not choices: ··· 298 326 text=text, 299 327 model=LOCAL_MODEL, 300 328 usage=_extract_usage(data), 301 - finish_reason=choice.get("finish_reason"), 329 + finish_reason=_normalize_finish_reason(choice.get("finish_reason")), 302 330 thinking=None, 303 331 ) 304 332
+1
solstone/think/providers/shared.py
··· 222 222 "provider_unavailable", 223 223 "provider_response_invalid", 224 224 "incomplete_json_length", 225 + "incomplete_text_length", 225 226 "unknown", 226 227 } 227 228 )
+21 -14
solstone/think/talents.py
··· 1607 1607 exc.reason, 1608 1608 ) 1609 1609 retries = 1 1610 - gen_result = generate_with_result( 1611 - contents=contents, 1612 - context=context, 1613 - temperature=max(temperature, _LOCAL_LENGTH_RETRY_TEMPERATURE_FLOOR), 1614 - max_output_tokens=max_output_tokens, 1615 - thinking_budget=thinking_budget, 1616 - system_instruction=system_instruction, 1617 - json_output=is_json_output, 1618 - json_schema=runtime_json_schema, 1619 - timeout_s=timeout_s, 1620 - provider=config.get("provider"), 1621 - model=config.get("model"), 1622 - inference_retry_index=1, 1623 - ) 1610 + try: 1611 + gen_result = generate_with_result( 1612 + contents=contents, 1613 + context=context, 1614 + temperature=max(temperature, _LOCAL_LENGTH_RETRY_TEMPERATURE_FLOOR), 1615 + max_output_tokens=max_output_tokens, 1616 + thinking_budget=thinking_budget, 1617 + system_instruction=system_instruction, 1618 + json_output=is_json_output, 1619 + json_schema=runtime_json_schema, 1620 + timeout_s=timeout_s, 1621 + provider=config.get("provider"), 1622 + model=config.get("model"), 1623 + inference_retry_index=1, 1624 + ) 1625 + except Exception as retry_exc: 1626 + retry_exc.retries = retries 1627 + raise 1624 1628 else: 1625 1629 if config.get("fallback_from") or not _should_fallback(exc): 1626 1630 raise ··· 2024 2028 event["partial_text_tail"] = e.partial_text[-500:] 2025 2029 name = config.get("name", "unknown") if config else "unknown" 2026 2030 log_extraction_failure(e, name) 2031 + retries = getattr(e, "retries", None) 2032 + if retries: 2033 + event["retries"] = retries 2027 2034 emit_event(event) 2028 2035 2029 2036 except Exception as exc:
+1 -1
tests/baselines/api/stats/stats.json
··· 365 365 "talents": false, 366 366 "transcripts": true 367 367 }, 368 - "max_output_tokens": 12288, 368 + "max_output_tokens": 6144, 369 369 "mtime": 0, 370 370 "output": "json", 371 371 "path": "<PROJECT>/solstone/talent/sense.md",
+39 -4
tests/test_bad_media_corpus.py
··· 54 54 FIXED_NOW = "2026-06-30T12:00:00Z" 55 55 56 56 57 + def _generate_result(text: str, finish_reason: str = "stop") -> dict[str, Any]: 58 + return {"text": text, "finish_reason": finish_reason} 59 + 60 + 57 61 @pytest.fixture 58 62 def observer_env(tmp_path, monkeypatch): 59 63 """Temp journal + Flask test client factory. ··· 244 248 output_path: Path, 245 249 *, 246 250 agenerate_response: str = "{}", 251 + agenerate_finish_reason: str = "stop", 247 252 expect_runtime_error: bool = False, 248 253 ) -> tuple[dict[str, Any], dict[str, Any], AsyncMock]: 249 254 from solstone.observe import describe, processing_record 250 255 251 - agenerate = AsyncMock(return_value=agenerate_response) 256 + agenerate = AsyncMock( 257 + return_value=_generate_result(agenerate_response, agenerate_finish_reason) 258 + ) 252 259 monkeypatch.setattr( 253 260 "solstone.think.models.resolve_provider", 254 261 lambda _context, _interface: ("google", "gemini-test"), ··· 256 263 monkeypatch.setattr(describe, "callosum_send", lambda *args, **kwargs: None) 257 264 monkeypatch.setattr(describe, "select_frames_for_extraction", lambda *a, **k: []) 258 265 monkeypatch.setattr(processing_record, "now_iso_utc", lambda: FIXED_NOW) 259 - monkeypatch.setattr("solstone.think.batch.agenerate", agenerate) 266 + monkeypatch.setattr("solstone.think.batch.agenerate_with_result", agenerate) 260 267 261 268 processor = describe.VideoProcessor(video_path) 262 269 if expect_runtime_error: ··· 366 373 spawned: list[str] = [] 367 374 writer_path = journal / "chronicle" / day / "health" / f"idle_{segment}.jsonl" 368 375 writer = ThinkingJSONLWriter(str(writer_path)) 369 - agenerate = agenerate_spy or AsyncMock(return_value="{}") 376 + agenerate = agenerate_spy or AsyncMock(return_value=_generate_result("{}")) 370 377 original_callosum = think._callosum 371 378 original_jsonl = think._jsonl 372 379 try: ··· 387 394 "wait_for_uses", 388 395 lambda agent_ids, timeout=600: ({aid: "finish" for aid in agent_ids}, []), 389 396 ) 390 - monkeypatch.setattr("solstone.think.batch.agenerate", agenerate) 397 + monkeypatch.setattr("solstone.think.batch.agenerate_with_result", agenerate) 391 398 think._callosum = None 392 399 think._jsonl = writer 393 400 result = think.run_segment_sense( ··· 647 654 # Both corrupt_input and analysis_failed derive DataState.FAILED; the 648 655 # distinction survives only in the processing-record reason_code. 649 656 assert read_segment_data_state(DAY, SEGMENT) == {"screen": DataState.FAILED.value} 657 + 658 + 659 + def test_truncated_frame_categorization_retries_and_promotes_no_frame_artifact( 660 + segment_journal, 661 + monkeypatch, 662 + ): 663 + segment = _segment_dir(segment_journal) 664 + video_path = segment / "screen.mp4" 665 + output_path = segment / "screen.jsonl" 666 + _build_one_frame_mp4(video_path) 667 + 668 + _header, record, agenerate = _drive_describe( 669 + monkeypatch, 670 + video_path, 671 + output_path, 672 + agenerate_response='{"visual_description":"partial"', 673 + agenerate_finish_reason="max_tokens", 674 + expect_runtime_error=True, 675 + ) 676 + 677 + _assert_processing_record( 678 + record, 679 + state=STATE_FAILED, 680 + reason_code=REASON_ANALYSIS_FAILED, 681 + handler=HANDLER_DESCRIBE, 682 + ) 683 + assert agenerate.call_count == 5 684 + assert len(_read_jsonl(output_path)) == 1 650 685 651 686 652 687 def test_ac6_no_model_calls_on_all_empty_segment(segment_journal, monkeypatch):
+131 -33
tests/test_batch.py
··· 12 12 from solstone.think.models import GEMINI_FLASH, GEMINI_LITE, SchemaValidationError 13 13 14 14 15 + def _result(text: str = "Response", finish_reason: str = "stop", **extra): 16 + return {"text": text, "finish_reason": finish_reason, **extra} 17 + 18 + 15 19 def test_batch_request_creation(): 16 20 """Test BatchRequest can be created with required and custom params.""" 17 21 # Required params only ··· 50 54 51 55 52 56 @pytest.mark.asyncio 53 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 57 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 54 58 async def test_batch_basic(mock_agenerate): 55 59 """Test basic batch execution with single request.""" 56 - mock_agenerate.return_value = "Response 1" 60 + mock_agenerate.return_value = _result("Response 1") 57 61 58 62 # Create batch and add request 59 63 batch = Batch(max_concurrent=5) ··· 80 84 81 85 82 86 @pytest.mark.asyncio 83 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 87 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 84 88 async def test_batch_with_model_override(mock_agenerate): 85 89 """Test batch with explicit model override.""" 86 - mock_agenerate.return_value = "Response" 90 + mock_agenerate.return_value = _result("Response") 87 91 88 92 batch = Batch(max_concurrent=5) 89 93 req = batch.create(contents="Test", context="test.context", model=GEMINI_FLASH) ··· 102 106 103 107 104 108 @pytest.mark.asyncio 105 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 109 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 106 110 async def test_batch_multiple_requests(mock_agenerate): 107 111 """Test batch with multiple requests.""" 108 - mock_agenerate.side_effect = ["Response 1", "Response 2", "Response 3"] 112 + mock_agenerate.side_effect = [ 113 + _result("Response 1"), 114 + _result("Response 2"), 115 + _result("Response 3"), 116 + ] 109 117 110 118 batch = Batch(max_concurrent=2) 111 119 ··· 141 149 142 150 143 151 @pytest.mark.asyncio 144 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 152 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 145 153 async def test_batch_error_handling(mock_agenerate): 146 154 """Test that errors are captured in request.error.""" 147 155 mock_agenerate.side_effect = ValueError("API error") ··· 165 173 166 174 @pytest.mark.asyncio 167 175 @patch("solstone.think.batch.resolve_provider") 168 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 176 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 169 177 async def test_batch_preserves_exception_metadata( 170 178 mock_agenerate, mock_resolve_provider 171 179 ): ··· 197 205 198 206 @pytest.mark.asyncio 199 207 @patch("solstone.think.batch.resolve_provider") 200 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 208 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 201 209 async def test_batch_classifies_exception_metadata_when_attrs_missing( 202 210 mock_agenerate, mock_resolve_provider 203 211 ): ··· 238 246 239 247 240 248 @pytest.mark.asyncio 241 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 249 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 242 250 async def test_batch_dynamic_adding(mock_agenerate): 243 251 """Test adding requests dynamically during iteration.""" 244 - mock_agenerate.return_value = "Response" 252 + mock_agenerate.return_value = _result("Response") 245 253 246 254 batch = Batch(max_concurrent=5) 247 255 ··· 271 279 272 280 273 281 @pytest.mark.asyncio 274 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 282 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 275 283 async def test_batch_retry_pattern(mock_agenerate): 276 284 """Test retry pattern - add failed request back with different model.""" 277 285 # First call fails, second succeeds ··· 282 290 call_count += 1 283 291 if call_count == 1: 284 292 raise ValueError("Transient error") 285 - return "Success on retry" 293 + return _result("Success on retry") 286 294 287 295 mock_agenerate.side_effect = mock_response 288 296 ··· 314 322 315 323 316 324 @pytest.mark.asyncio 317 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 325 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 318 326 async def test_batch_factory_method(mock_agenerate): 319 327 """Test that batch.create() factory method works correctly.""" 320 - mock_agenerate.return_value = "Response" 328 + mock_agenerate.return_value = _result("Response") 321 329 322 330 batch = Batch() 323 331 ··· 339 347 340 348 341 349 @pytest.mark.asyncio 342 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 350 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 343 351 async def test_batch_can_add_after_draining(mock_agenerate): 344 352 """Test that adding after draining works (reusable batch).""" 345 - mock_agenerate.side_effect = ["Response 1", "Response 2"] 353 + mock_agenerate.side_effect = [_result("Response 1"), _result("Response 2")] 346 354 347 355 batch = Batch() 348 356 ··· 372 380 373 381 374 382 @pytest.mark.asyncio 375 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 383 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 376 384 async def test_batch_empty_batch(mock_agenerate): 377 385 """Test that empty batch (no requests) completes immediately.""" 378 386 batch = Batch() ··· 385 393 386 394 387 395 @pytest.mark.asyncio 388 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 396 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 389 397 async def test_batch_concurrency_limit(mock_agenerate): 390 398 """Test that semaphore limits concurrent requests.""" 391 399 # Track concurrent calls ··· 404 412 async with lock: 405 413 concurrent_calls -= 1 406 414 407 - return "Response" 415 + return _result("Response") 408 416 409 417 mock_agenerate.side_effect = mock_with_tracking 410 418 ··· 426 434 427 435 428 436 @pytest.mark.asyncio 429 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 437 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 430 438 async def test_batch_update_method(mock_agenerate): 431 439 """Test batch.update() method for modifying and re-adding requests.""" 432 440 # Track which model was used in each call ··· 434 442 435 443 async def mock_track_model(*args, **kwargs): 436 444 call_models.append(kwargs.get("model", "unknown")) 437 - return f"Response from {kwargs.get('model', 'unknown')}" 445 + return _result(f"Response from {kwargs.get('model', 'unknown')}") 438 446 439 447 mock_agenerate.side_effect = mock_track_model 440 448 ··· 494 502 495 503 496 504 @pytest.mark.asyncio 497 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 505 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 498 506 async def test_batch_timeout_passthrough(mock_agenerate): 499 - """Test that timeout_s is passed through to agenerate.""" 500 - mock_agenerate.return_value = "Response" 507 + """Test that timeout_s is passed through to agenerate_with_result.""" 508 + mock_agenerate.return_value = _result("Response") 501 509 502 510 batch = Batch(max_concurrent=5) 503 511 ··· 511 519 512 520 assert len(results) == 1 513 521 514 - # Verify timeout_s was passed to agenerate 522 + # Verify timeout_s was passed to agenerate_with_result 515 523 mock_agenerate.assert_called_once() 516 524 call_kwargs = mock_agenerate.call_args[1] 517 525 assert call_kwargs["timeout_s"] == 45 518 526 519 527 520 528 @pytest.mark.asyncio 521 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 529 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 522 530 async def test_batch_client_passthrough(mock_agenerate): 523 - """Test that client is passed through to agenerate for Google connection reuse.""" 524 - mock_agenerate.return_value = "Response" 531 + """Test that client is passed through for Google connection reuse.""" 532 + mock_agenerate.return_value = _result("Response") 525 533 526 534 # Create a mock client (would be genai.Client for Google) 527 535 mock_client = object() ··· 542 550 543 551 544 552 @pytest.mark.asyncio 545 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 553 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 546 554 async def test_batch_passes_json_schema_to_agenerate(mock_agenerate): 547 - """Test that json_schema is passed through to agenerate.""" 548 - mock_agenerate.return_value = "Response" 555 + """Test that json_schema is passed through to agenerate_with_result.""" 556 + mock_agenerate.return_value = _result("Response") 549 557 550 558 batch = Batch(max_concurrent=5) 551 559 req = batch.create( ··· 566 574 567 575 568 576 @pytest.mark.asyncio 569 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 577 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 570 578 async def test_batch_schema_validation_error_populates_request_error(mock_agenerate): 571 579 mock_agenerate.side_effect = SchemaValidationError( 572 580 [{"path": "", "constraint": "json_parse", "message": "empty"}], ··· 588 596 assert len(results) == 1 589 597 assert results[0].response is None 590 598 assert "schema validation" in results[0].error 599 + 600 + 601 + @pytest.mark.asyncio 602 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 603 + async def test_batch_non_json_length_finish_populates_text_length_error(mock_agenerate): 604 + mock_agenerate.return_value = _result("partial text", finish_reason="max_tokens") 605 + 606 + batch = Batch(max_concurrent=5) 607 + req = batch.create(contents="Test prompt", context="test.context") 608 + batch.add(req) 609 + 610 + results = [] 611 + async for completed_req in batch.drain_batch(): 612 + results.append(completed_req) 613 + 614 + assert len(results) == 1 615 + assert results[0].response is None 616 + assert results[0].reason_code == "incomplete_text_length" 617 + assert "Text response incomplete" in results[0].error 618 + 619 + 620 + @pytest.mark.asyncio 621 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 622 + async def test_batch_non_json_stop_finish_succeeds(mock_agenerate): 623 + mock_agenerate.return_value = _result("complete text", finish_reason="stop") 624 + 625 + batch = Batch(max_concurrent=1) 626 + req = batch.create( 627 + contents="Test", 628 + context="test.context", 629 + json_output=False, 630 + ) 631 + batch.add(req) 632 + 633 + results = [] 634 + async for completed_req in batch.drain_batch(): 635 + results.append(completed_req) 636 + 637 + assert len(results) == 1 638 + assert req.response == "complete text" 639 + assert req.error is None 640 + assert req.reason_code is None 641 + 642 + 643 + @pytest.mark.asyncio 644 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 645 + async def test_batch_non_json_non_length_finish_is_provider_response_invalid( 646 + mock_agenerate, 647 + ): 648 + mock_agenerate.return_value = _result("", finish_reason="content_filter") 649 + 650 + batch = Batch(max_concurrent=5) 651 + req = batch.create(contents="Test prompt", context="test.context") 652 + batch.add(req) 653 + 654 + results = [] 655 + async for completed_req in batch.drain_batch(): 656 + results.append(completed_req) 657 + 658 + assert len(results) == 1 659 + assert results[0].response is None 660 + assert results[0].reason_code == "provider_response_invalid" 661 + 662 + 663 + @pytest.mark.asyncio 664 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 665 + async def test_batch_full_result_schema_invalid_stays_hard_failure(mock_agenerate): 666 + mock_agenerate.return_value = _result( 667 + '{"field": "bad"}', 668 + schema_validation={ 669 + "valid": False, 670 + "errors": [{"path": "/field", "constraint": "type", "message": "bad"}], 671 + }, 672 + ) 673 + 674 + batch = Batch(max_concurrent=5) 675 + req = batch.create( 676 + contents="Test prompt", 677 + context="test.context", 678 + json_schema={"type": "object"}, 679 + ) 680 + batch.add(req) 681 + 682 + results = [] 683 + async for completed_req in batch.drain_batch(): 684 + results.append(completed_req) 685 + 686 + assert len(results) == 1 687 + assert results[0].response is None 688 + assert "schema validation" in results[0].error
+5 -2
tests/test_calendar_schema.py
··· 82 82 83 83 84 84 @pytest.mark.asyncio 85 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 85 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 86 86 async def test_calendar_extract_batch_call_passes_schema(mock_agenerate): 87 - mock_agenerate.return_value = json.dumps(_valid_payload()) 87 + mock_agenerate.return_value = { 88 + "text": json.dumps(_valid_payload()), 89 + "finish_reason": "stop", 90 + } 88 91 89 92 cat_meta = describe_mod.CATEGORIES["calendar"] 90 93 batch = Batch(max_concurrent=1)
+1
tests/test_chat_reasons.py
··· 39 39 "chat_timeout", 40 40 "context_window_exceeded", 41 41 "incomplete_json_length", 42 + "incomplete_text_length", 42 43 "max_turns_exhausted", 43 44 "no_output", 44 45 "token_budget_exceeded",
+100
tests/test_describe_promote.py
··· 6 6 import logging 7 7 from pathlib import Path 8 8 from types import SimpleNamespace 9 + from unittest.mock import AsyncMock 9 10 10 11 import pytest 11 12 from PIL import Image ··· 63 64 for line in path.read_text(encoding="utf-8").splitlines() 64 65 if line 65 66 ] 67 + 68 + 69 + def _generate_result(text: str, finish_reason: str = "stop") -> dict: 70 + return {"text": text, "finish_reason": finish_reason} 66 71 67 72 68 73 def _canned_detection() -> dict: ··· 280 285 assert output_path.read_text() == expected 281 286 assert output_path.name in [path.name for path in output_path.parent.iterdir()] 282 287 _assert_no_describe_temp(output_path.parent) 288 + 289 + 290 + @pytest.mark.asyncio 291 + async def test_browsing_truncation_does_not_promote_category_content( 292 + tmp_path, monkeypatch 293 + ): 294 + from solstone.think import batch as batch_module 295 + from solstone.think import models 296 + 297 + video_path = _video_path(tmp_path) 298 + output_path = video_path.with_suffix(".jsonl") 299 + frame_bytes = _png_bytes() 300 + processor = _processor( 301 + video_path, 302 + [_frame(1, 0.0, frame_bytes)], 303 + monkeypatch, 304 + ) 305 + completed = [] 306 + real_batch = batch_module.Batch 307 + 308 + class SpyBatch(real_batch): 309 + async def drain_batch(self): 310 + async for request in super().drain_batch(): 311 + completed.append( 312 + { 313 + "request_type": getattr(request, "request_type", None), 314 + "reason_code": getattr(request, "reason_code", None), 315 + "retry_count": getattr(request, "retry_count", None), 316 + "extraction_category": getattr( 317 + request, "extraction_category", None 318 + ), 319 + } 320 + ) 321 + yield request 322 + 323 + categorize = _generate_result( 324 + json.dumps( 325 + { 326 + "visual_description": "A browser page is open.", 327 + "primary": "browsing", 328 + "secondary": "none", 329 + "overlap": True, 330 + } 331 + ) 332 + ) 333 + truncated = _generate_result("partial browsing notes", "max_tokens") 334 + agenerate = AsyncMock(return_value=truncated) 335 + agenerate.side_effect = [categorize, *[truncated for _ in range(5)]] 336 + 337 + monkeypatch.setattr(batch_module, "Batch", SpyBatch) 338 + monkeypatch.setattr(batch_module, "agenerate_with_result", agenerate) 339 + monkeypatch.setattr( 340 + batch_module, 341 + "resolve_provider", 342 + lambda _context, _interface: ("google", "gemini-test"), 343 + ) 344 + monkeypatch.setattr( 345 + models, 346 + "resolve_provider", 347 + lambda _context, _interface: ("google", "gemini-test"), 348 + ) 349 + monkeypatch.setattr( 350 + processing_record_module, "now_iso_utc", lambda: "2026-06-30T12:00:00Z" 351 + ) 352 + monkeypatch.setattr(describe_module, "callosum_send", lambda *a, **k: None) 353 + monkeypatch.setattr(describe_module, "get_config", lambda: {"describe": {}}) 354 + monkeypatch.setattr( 355 + describe_module, 356 + "select_frames_for_extraction", 357 + lambda *_args, **_kwargs: [1], 358 + ) 359 + 360 + await processor.process_with_vision( 361 + max_concurrent=1, 362 + output_path=output_path, 363 + work_key="20250101/143022_300/screen", 364 + ) 365 + 366 + rows = _jsonl_rows(output_path) 367 + result = rows[1] 368 + category_requests = [ 369 + request 370 + for request in completed 371 + if request["request_type"] == describe_module.RequestType.CATEGORY 372 + ] 373 + 374 + assert agenerate.call_count == 6 375 + assert len(category_requests) == 5 376 + assert category_requests[-1]["reason_code"] == "incomplete_text_length" 377 + assert result["enhanced"] is True 378 + assert result["content"] == {} 379 + assert "browsing" not in result["content"] 380 + assert result["error"] == "Text response incomplete (reason: max_tokens)" 381 + assert result["requests"][-1]["category"] == "browsing" 382 + assert result["requests"][-1]["retries"] == 4 283 383 284 384 285 385 @pytest.mark.asyncio
+24 -16
tests/test_generate_talents.py
··· 48 48 49 49 largest_observed_legitimate_completion = 3560 50 50 51 - for name in ("sense", "participation"): 52 - config = get_talent(name) 53 - params = _generation_params(config) 54 - max_output_tokens = params["max_output_tokens"] 55 - # Mirror talents.py's resolution: frontmatter timeout_s short-circuits 56 - # the derivation. 57 - resolved = config.get("timeout_s") or min( 58 - 480, 59 - max(120, (max_output_tokens + params["thinking_budget"]) // 100), 60 - ) 51 + sense_config = get_talent("sense") 52 + sense_params = _generation_params(sense_config) 53 + assert sense_params["max_output_tokens"] == 6144 54 + assert sense_config.get("timeout_s") == 480 55 + assert "temperature" not in sense_config 61 56 62 - assert max_output_tokens >= 2 * largest_observed_legitimate_completion 63 - assert max_output_tokens < 8192 * 6 64 - assert config.get("timeout_s") == 480 65 - assert resolved == config["timeout_s"] 66 - assert resolved >= 480 67 - assert "temperature" not in config 57 + participation_config = get_talent("participation") 58 + participation_params = _generation_params(participation_config) 59 + participation_tokens = participation_params["max_output_tokens"] 60 + # Participation did not change and still keeps 2x headroom over the 61 + # largest legitimate completion observed when this guard was added. 62 + assert participation_tokens == 12288 63 + assert participation_tokens >= 2 * largest_observed_legitimate_completion 64 + assert participation_tokens < 8192 * 6 65 + assert participation_config.get("timeout_s") == 480 66 + resolved = participation_config.get("timeout_s") or min( 67 + 480, 68 + max( 69 + 120, 70 + (participation_tokens + participation_params["thinking_budget"]) // 100, 71 + ), 72 + ) 73 + assert resolved == participation_config["timeout_s"] 74 + assert resolved >= 480 75 + assert "temperature" not in participation_config
+105 -21
tests/test_local.py
··· 65 65 return found 66 66 67 67 68 + def _local_response(finish_reason): 69 + return { 70 + "choices": [ 71 + { 72 + "message": {"content": "ok"}, 73 + "finish_reason": finish_reason, 74 + } 75 + ] 76 + } 77 + 78 + 68 79 def test_local_model_prefix_maps_to_provider(): 69 80 assert get_model_provider(LOCAL_MODEL) == "local" 70 81 ··· 112 123 ) 113 124 == "context_budget_exceeded" 114 125 ) 126 + 127 + 128 + @pytest.mark.parametrize( 129 + ("raw", "expected"), 130 + [ 131 + ("stop", "stop"), 132 + ("length", "max_tokens"), 133 + ("max_tokens", "max_tokens"), 134 + ("content_filter", "content_filter"), 135 + ], 136 + ) 137 + def test_parse_response_normalizes_known_finish_reasons(raw, expected): 138 + provider = _provider() 139 + 140 + result = provider._parse_response(_local_response(raw)) 141 + 142 + assert result["finish_reason"] == expected 143 + 144 + 145 + @pytest.mark.parametrize("raw", [None, "", "weird", "tool_calls", "function_call"]) 146 + def test_parse_response_fails_closed_on_bad_finish_reasons(raw): 147 + provider = _provider() 148 + 149 + with pytest.raises(provider.LocalProviderError) as exc_info: 150 + provider._parse_response(_local_response(raw)) 151 + 152 + assert exc_info.value.reason_code == "provider_response_invalid" 115 153 116 154 117 155 def test_cloud_generate_providers_do_not_reference_local_budget(): ··· 839 877 return None 840 878 841 879 def json(self): 842 - return {"choices": [{"message": {"content": "ok"}}]} 880 + return { 881 + "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}] 882 + } 843 883 844 884 def fake_post(url, **kwargs): 845 885 captured.update({"url": url, **kwargs}) ··· 854 894 assert "headers" not in captured 855 895 856 896 857 - def test_generate_schema_files_do_not_declare_bounds(): 897 + def _load_schema(path: str) -> dict: 898 + return json.loads(Path(path).read_text(encoding="utf-8")) 899 + 900 + 901 + def _sense_collection_bounds(schema: dict) -> dict[str, int]: 902 + properties = schema["properties"] 903 + return { 904 + "entities": properties["entities"]["maxItems"], 905 + "facets": properties["facets"]["maxItems"], 906 + "speakers": properties["speakers"]["maxItems"], 907 + } 908 + 909 + 910 + def test_generate_schema_files_declare_only_safe_sense_collection_bounds(): 858 911 bounded_keys = { 859 912 "minItems", 860 913 "maxItems", ··· 863 916 "minimum", 864 917 "maximum", 865 918 } 866 - paths = [ 867 - Path("solstone/talent/sense.schema.json"), 868 - Path("solstone/talent/participation.schema.json"), 869 - Path("solstone/talent/participation_entry.schema.json"), 919 + sense = _load_schema("solstone/talent/sense.schema.json") 920 + 921 + assert _sense_collection_bounds(sense) == { 922 + "entities": 96, 923 + "facets": 16, 924 + "speakers": 16, 925 + } 926 + assert _schema_keyword_paths(sense, {"maxItems"}) == [ 927 + "$/properties/entities/maxItems", 928 + "$/properties/facets/maxItems", 929 + "$/properties/speakers/maxItems", 870 930 ] 871 - found = {} 931 + assert _schema_keyword_paths(sense, {"pattern", "minLength", "maxLength"}) == [] 872 932 873 - def walk(node, keys): 874 - if isinstance(node, dict): 875 - keys.update(bounded_keys & node.keys()) 876 - for value in node.values(): 877 - walk(value, keys) 878 - elif isinstance(node, list): 879 - for item in node: 880 - walk(item, keys) 933 + for path in ( 934 + "solstone/talent/participation.schema.json", 935 + "solstone/talent/participation_entry.schema.json", 936 + ): 937 + assert _schema_keyword_paths(_load_schema(path), bounded_keys) == [] 881 938 882 - for path in paths: 883 - keys = set() 884 - walk(json.loads(path.read_text(encoding="utf-8")), keys) 885 - if keys: 886 - found[str(path)] = sorted(keys) 887 939 888 - assert found == {} 940 + def test_sense_collection_bounds_survive_runtime_and_local_schema_prep(): 941 + from solstone.think.talent import hydrate_runtime_enums 942 + 943 + provider = _provider() 944 + sense = _load_schema("solstone/talent/sense.schema.json") 945 + 946 + hydrated = hydrate_runtime_enums(sense) 947 + prepared = provider._prepare_local_schema(hydrated) 948 + 949 + assert _sense_collection_bounds(hydrated) == { 950 + "entities": 96, 951 + "facets": 16, 952 + "speakers": 16, 953 + } 954 + assert _sense_collection_bounds(prepared) == { 955 + "entities": 96, 956 + "facets": 16, 957 + "speakers": 16, 958 + } 959 + 960 + 961 + def test_local_input_budget_reserve_for_changed_caps(): 962 + from solstone.think.providers.local_budget import compute_input_budget 963 + 964 + floor = 16384 965 + capable = 32768 966 + 967 + assert compute_input_budget(512, floor) - compute_input_budget(1024, floor) == 512 968 + assert compute_input_budget(2048, floor) - compute_input_budget(4096, floor) == 2048 969 + assert compute_input_budget(12288, floor) == compute_input_budget(6144, floor) 970 + assert compute_input_budget(12288, floor) == floor - 4096 - 256 971 + assert compute_input_budget(12288, capable) == capable - 8192 - 256 972 + assert compute_input_budget(6144, capable) == capable - 6144 - 256 889 973 890 974 891 975 def test_run_generate_byo_network_error_maps_to_unreachable(monkeypatch):
+8 -5
tests/test_meeting_schema.py
··· 127 127 128 128 129 129 @pytest.mark.asyncio 130 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 130 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 131 131 async def test_meeting_extract_batch_call_passes_schema(mock_agenerate): 132 - mock_agenerate.return_value = ( 133 - '{"platform":"zoom","participants":[{"name":"Alice","status":"active",' 134 - '"video":true}],"screen_share":null}' 135 - ) 132 + mock_agenerate.return_value = { 133 + "text": ( 134 + '{"platform":"zoom","participants":[{"name":"Alice","status":"active",' 135 + '"video":true}],"screen_share":null}' 136 + ), 137 + "finish_reason": "stop", 138 + } 136 139 137 140 cat_meta = describe_mod.CATEGORIES["meeting"] 138 141 batch = Batch(max_concurrent=1)
+5 -2
tests/test_messaging_schema.py
··· 74 74 75 75 76 76 @pytest.mark.asyncio 77 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 77 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 78 78 async def test_messaging_extract_batch_call_passes_schema(mock_agenerate): 79 - mock_agenerate.return_value = json.dumps(_valid_payload()) 79 + mock_agenerate.return_value = { 80 + "text": json.dumps(_valid_payload()), 81 + "finish_reason": "stop", 82 + } 80 83 81 84 cat_meta = describe_mod.CATEGORIES["messaging"] 82 85 batch = Batch(max_concurrent=1)
+64
tests/test_models.py
··· 35 35 TIER_PRO, 36 36 TYPE_DEFAULTS, 37 37 IncompleteJSONError, 38 + IncompleteTextError, 38 39 NoBrainConfiguredError, 40 + ProviderResponseInvalidError, 39 41 SchemaValidationError, 40 42 _Family, 41 43 _find_pricing_fallback, ··· 44 46 _parse_family_openai, 45 47 _validate_schema, 46 48 agenerate, 49 + agenerate_with_result, 47 50 calc_agent_cost, 48 51 calc_token_cost, 52 + finish_reason_error, 49 53 generate, 50 54 generate_with_result, 51 55 get_context_registry, ··· 1573 1577 1574 1578 1575 1579 class TestGenerateJsonSchemaPlumbing: 1580 + def test_finish_reason_predicate_json_rejects_any_non_stop(self): 1581 + error = finish_reason_error( 1582 + {"text": "{}", "finish_reason": "content_filter"}, 1583 + json_output=True, 1584 + ) 1585 + 1586 + assert isinstance(error, IncompleteJSONError) 1587 + 1588 + def test_finish_reason_predicate_plain_text_rejects_length(self): 1589 + error = finish_reason_error( 1590 + {"text": "partial", "finish_reason": "max_tokens"}, 1591 + json_output=False, 1592 + ) 1593 + 1594 + assert isinstance(error, IncompleteTextError) 1595 + assert error.reason_code == "incomplete_text_length" 1596 + 1597 + def test_finish_reason_predicate_plain_text_rejects_non_length(self): 1598 + error = finish_reason_error( 1599 + {"text": "", "finish_reason": "content_filter"}, 1600 + json_output=False, 1601 + ) 1602 + 1603 + assert isinstance(error, ProviderResponseInvalidError) 1604 + assert error.reason_code == "provider_response_invalid" 1605 + 1606 + def test_validate_json_response_plain_text_leniency_is_unchanged(self): 1607 + models_module._validate_json_response( 1608 + {"text": "partial", "finish_reason": "max_tokens"}, 1609 + json_output=False, 1610 + ) 1611 + 1576 1612 def test_generate_forces_json_output_with_schema(self): 1577 1613 schema = {"type": "object"} 1578 1614 provider_module = SimpleNamespace( ··· 1639 1675 "hello", 1640 1676 "test.context", 1641 1677 json_schema={"type": "object"}, 1678 + ) 1679 + 1680 + assert result["schema_validation"] == validation 1681 + 1682 + def test_agenerate_with_result_adds_schema_validation(self): 1683 + provider_module = SimpleNamespace( 1684 + run_agenerate=AsyncMock( 1685 + return_value={"text": "{}", "finish_reason": "stop"} 1686 + ) 1687 + ) 1688 + validation = {"valid": True, "errors": []} 1689 + 1690 + with ( 1691 + patch( 1692 + "solstone.think.models.resolve_provider", return_value=("fake", "model") 1693 + ), 1694 + patch( 1695 + "solstone.think.providers.get_provider_module", 1696 + return_value=provider_module, 1697 + ), 1698 + patch("solstone.think.models._validate_schema", return_value=validation), 1699 + ): 1700 + result = asyncio.run( 1701 + agenerate_with_result( 1702 + "hello", 1703 + "test.context", 1704 + json_schema={"type": "object"}, 1705 + ) 1642 1706 ) 1643 1707 1644 1708 assert result["schema_validation"] == validation
+8 -5
tests/test_observe_describe_schema.py
··· 138 138 139 139 140 140 @pytest.mark.asyncio 141 - @patch("solstone.think.batch.agenerate", new_callable=AsyncMock) 141 + @patch("solstone.think.batch.agenerate_with_result", new_callable=AsyncMock) 142 142 async def test_describe_batch_call_passes_schema(mock_agenerate): 143 - mock_agenerate.return_value = ( 144 - '{"visual_description":"A code editor is visible.","primary":"code",' 145 - '"secondary":"none","overlap":false}' 146 - ) 143 + mock_agenerate.return_value = { 144 + "text": ( 145 + '{"visual_description":"A code editor is visible.","primary":"code",' 146 + '"secondary":"none","overlap":false}' 147 + ), 148 + "finish_reason": "stop", 149 + } 147 150 148 151 batch = Batch(max_concurrent=1) 149 152 req = batch.create(
+1
tests/test_provider_readiness_presenter.py
··· 148 148 "chat_timeout", 149 149 "network_unreachable", 150 150 "provider_response_invalid", 151 + "incomplete_text_length", 151 152 "no_output", 152 153 "unknown", 153 154 "ready",
+1
tests/test_provider_state.py
··· 130 130 "provider_response_invalid", 131 131 "context_window_exceeded", 132 132 "incomplete_json_length", 133 + "incomplete_text_length", 133 134 "max_turns_exhausted", 134 135 "unknown", 135 136 }
+75 -21
tests/test_sense_schema.py
··· 6 6 from pathlib import Path 7 7 8 8 import frontmatter 9 + import pytest 9 10 from jsonschema import Draft202012Validator 10 11 11 12 from solstone.think.activities import DEFAULT_ACTIVITIES ··· 37 38 38 39 def _facet_naming_template() -> str: 39 40 return FACET_NAMING_PATH.read_text(encoding="utf-8").strip() 41 + 42 + 43 + def _valid_sense_payload() -> dict: 44 + return { 45 + "density": "active", 46 + "content_type": "coding", 47 + "activity_summary": "Writing tests.", 48 + "entities": [], 49 + "facets": [ 50 + { 51 + "facet": RUNTIME_FACETS_SENTINEL, 52 + "activity": "Writing tests.", 53 + "level": "low", 54 + } 55 + ], 56 + "speculative_facet": "test-planning", 57 + "meeting_detected": False, 58 + "speakers": [], 59 + "recommend": { 60 + "screen_record": False, 61 + "speaker_attribution": False, 62 + }, 63 + "emotional_register": "focused", 64 + } 65 + 66 + 67 + def _sense_entity(index: int) -> dict: 68 + return { 69 + "type": "Person", 70 + "name": f"Person {index}", 71 + "role": "mentioned", 72 + "source": "screen", 73 + "context": "visible in the workspace", 74 + "level": "low", 75 + } 76 + 77 + 78 + def _sense_facet(index: int) -> dict: 79 + return { 80 + "facet": RUNTIME_FACETS_SENTINEL, 81 + "activity": f"Activity {index}", 82 + "level": "low", 83 + } 40 84 41 85 42 86 def _write_prompt_journal(tmp_path: Path, *, with_facet: bool) -> None: ··· 119 163 def test_sense_schema_speculative_facet_nullable_and_required(): 120 164 schema = json.loads(SENSE_SCHEMA_PATH.read_text(encoding="utf-8")) 121 165 validator = Draft202012Validator(schema) 122 - valid = { 123 - "density": "active", 124 - "content_type": "coding", 125 - "activity_summary": "Writing tests.", 126 - "entities": [], 127 - "facets": [ 128 - { 129 - "facet": RUNTIME_FACETS_SENTINEL, 130 - "activity": "Writing tests.", 131 - "level": "low", 132 - } 133 - ], 134 - "speculative_facet": "test-planning", 135 - "meeting_detected": False, 136 - "speakers": [], 137 - "recommend": { 138 - "screen_record": False, 139 - "speaker_attribution": False, 140 - }, 141 - "emotional_register": "focused", 142 - } 166 + valid = _valid_sense_payload() 143 167 144 168 assert schema["properties"]["speculative_facet"] == {"type": ["string", "null"]} 145 169 assert "speculative_facet" in schema["required"] ··· 152 176 missing = dict(valid) 153 177 del missing["speculative_facet"] 154 178 assert list(validator.iter_errors(missing)) 179 + 180 + 181 + @pytest.mark.parametrize( 182 + ("collection", "ok_count", "bad_count", "factory"), 183 + [ 184 + ("entities", 96, 97, _sense_entity), 185 + ("facets", 16, 17, _sense_facet), 186 + ("speakers", 16, 17, lambda index: f"Speaker {index}"), 187 + ], 188 + ) 189 + def test_sense_schema_collection_bounds_independently( 190 + collection, 191 + ok_count, 192 + bad_count, 193 + factory, 194 + ): 195 + schema = json.loads(SENSE_SCHEMA_PATH.read_text(encoding="utf-8")) 196 + validator = Draft202012Validator(schema) 197 + payload = _valid_sense_payload() 198 + 199 + payload[collection] = [factory(index) for index in range(ok_count)] 200 + assert list(validator.iter_errors(payload)) == [] 201 + 202 + payload[collection] = [factory(index) for index in range(bad_count)] 203 + errors = list(validator.iter_errors(payload)) 204 + 205 + assert any( 206 + error.validator == "maxItems" and list(error.path) == [collection] 207 + for error in errors 208 + ) 155 209 156 210 157 211 def test_sense_prompt_renders_speculative_facet_instruction_in_steady_state(
+65 -1
tests/test_talent_fallback.py
··· 772 772 assert events[-1]["retries"] == 1 773 773 774 774 775 + def test_execute_generate_local_length_retry_success_writes_once_and_runs_hook_once( 776 + tmp_path, monkeypatch 777 + ): 778 + from solstone.think import models, talents 779 + from solstone.think.talents import _execute_generate 780 + 781 + output_path = tmp_path / "out.json" 782 + events = [] 783 + generate_calls = [] 784 + hook_calls = [] 785 + 786 + def mock_generate_with_result(**kwargs): 787 + generate_calls.append(kwargs) 788 + if len(generate_calls) == 1: 789 + raise IncompleteJSONError("length", '{"partial":') 790 + return { 791 + "text": '{"summary": "ok"}', 792 + "usage": {"input_tokens": 1, "output_tokens": 2}, 793 + } 794 + 795 + def post_hook(result, config): 796 + hook_calls.append((result, config["name"])) 797 + return result 798 + 799 + def fail_backup(_agent_type): 800 + raise AssertionError("local retry must not consult cloud backup") 801 + 802 + monkeypatch.setattr( 803 + "solstone.think.talent.key_to_context", lambda _name: "talent.system.default" 804 + ) 805 + monkeypatch.setattr(models, "generate_with_result", mock_generate_with_result) 806 + monkeypatch.setattr(models, "get_backup_provider", fail_backup) 807 + monkeypatch.setattr(talents, "load_post_hook", lambda _config: post_hook) 808 + 809 + asyncio.run( 810 + _execute_generate( 811 + { 812 + "name": "chat", 813 + "provider": "local", 814 + "model": LOCAL_MODEL, 815 + "prompt": "hello", 816 + "health_stale": False, 817 + "output": "json", 818 + "output_path": str(output_path), 819 + "hook": {"post": "test"}, 820 + "json_schema": { 821 + "type": "object", 822 + "additionalProperties": False, 823 + "required": ["summary"], 824 + "properties": {"summary": {"type": "string"}}, 825 + }, 826 + }, 827 + events.append, 828 + ) 829 + ) 830 + 831 + assert len(generate_calls) == 2 832 + assert hook_calls == [('{"summary": "ok"}', "chat")] 833 + assert output_path.read_text(encoding="utf-8") == '{"summary": "ok"}' 834 + assert [path.name for path in tmp_path.iterdir()] == ["out.json"] 835 + assert events[-1]["event"] == "finish" 836 + assert events[-1]["retries"] == 1 837 + 838 + 775 839 def test_main_async_local_length_retry_second_failure_emits_one_error( 776 840 monkeypatch, capsys 777 841 ): ··· 830 894 assert calls["count"] == 2 831 895 assert len(error_events) == 1 832 896 assert error_events[0]["reason_code"] == "incomplete_json_length" 897 + assert error_events[0]["retries"] == 1 833 898 assert finish_events == [] 834 - assert all("retries" not in event for event in events) 835 899 836 900 837 901 @pytest.mark.parametrize(