solpbc.org / solstone
personal memory agent
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

feat(providers): endpoint-lane request budget, reactive re-clamp, cogitate caps

Converge both non-bundled run_generate/run_agenerate branches through one shared _prepare_endpoint_request that fits input, applies image-aware output clamps with _SAFETY_MARGIN_TOKENS, and raises ContextBudgetExceeded before POST when the completion floor cannot fit. Add a single reactive completion-budget re-clamp using limit - input - 16, with one retry inside the held admission permit and remaining deadline; prompt-alone and second context 400s are terminal context_window_exceeded. Add a body-reading _classify_byo_generate_error backstop with length-capped, credential-redacted matching. Preserve generic-BYO and bundled byte-identical request bodies while adding confidential Qwen parity, and thread served windows once per cogitate run for scaled caps/condenser while unknown and sub-16384 windows remain unchanged.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

author
Jer Miller co-author
Claude Opus 4.8 (1M context) date (Jul 23, 2026, 2:09 PM -0600) commit 3d1ecb26 parent d71bb8d3
+1298 -102
+308 -37
solstone/think/providers/local.py
··· 13 13 import contextlib 14 14 import copy 15 15 import logging 16 + import re 16 17 import time 17 18 import traceback 18 19 import uuid 19 20 from collections.abc import Callable 20 21 from dataclasses import dataclass 21 - from typing import Any 22 + from typing import Any, Literal 22 23 23 24 from solstone.think.models import LOCAL_MODEL 24 25 from solstone.think.providers._image import encode_image_part, is_image_part 25 26 from solstone.think.providers.local_endpoint import ( 27 + ENDPOINT_ERROR_BODY_CAP_CHARS, 26 28 LOCAL_ENDPOINT_CONTRACT_COPY, 27 29 LOCAL_ENDPOINT_UNREACHABLE_COPY, 28 30 classify_byo_cogitate_error, ··· 30 32 is_byo_network_error, 31 33 local_endpoint_reason_copy, 32 34 redact_local_endpoint_credential, 35 + resolve_endpoint_served_window, 33 36 resolve_local_endpoint, 34 37 ) 35 38 from solstone.think.providers.shared import ( ··· 73 76 _LOCAL_CAPACITY_EXHAUSTED_MESSAGE = ( 74 77 "The local model was busy and could not finish this request. Try again in a moment." 75 78 ) 79 + _ENDPOINT_CONTEXT_WINDOW_MESSAGE = ( 80 + "The configured endpoint rejected the request: prompt and completion exceed " 81 + "the served context window." 82 + ) 83 + _ENDPOINT_MIN_COMPLETION_TOKENS = 256 84 + _ENDPOINT_RECLAMP_SLACK_TOKENS = 16 85 + _ENDPOINT_COMPLETION_ANCHOR = "tokens for the completion" 86 + _ENDPOINT_LIMIT_RE = re.compile(r"maximum context length of\s+(?P<limit>\d+)\s+tokens") 87 + _ENDPOINT_INPUT_RE = re.compile( 88 + r"(?P<input>\d+)\s+tokens?\s+from\s+the\s+input\s+messages?\s+and\s+" 89 + r"\d+\s+tokens?\s+for\s+the\s+completion" 90 + ) 76 91 77 92 78 93 @dataclass(frozen=True) ··· 127 142 super().__init__("local_capacity_exhausted", _LOCAL_CAPACITY_EXHAUSTED_MESSAGE) 128 143 129 144 145 + @dataclass(frozen=True) 146 + class _EndpointOverflowDecision: 147 + kind: Literal["retry", "context", "budget", "contract"] 148 + max_tokens: int | None = None 149 + 150 + 130 151 def normalize_model_id(model: str | None) -> str: 131 152 model_id = str(model or LOCAL_MODEL) 132 153 if model_id.startswith("openai/"): ··· 248 269 json_output: bool, 249 270 json_schema: dict | None, 250 271 is_bundled: bool, 272 + is_confidential: bool = False, 251 273 ) -> dict[str, Any]: 252 274 body: dict[str, Any] = { 253 275 "model": model_id, ··· 256 278 "max_tokens": max_output_tokens, 257 279 "stream": False, 258 280 } 259 - if is_bundled: 281 + if is_bundled or is_confidential: 260 282 body.update( 261 283 { 262 284 "chat_template_kwargs": {"enable_thinking": False}, ··· 280 302 return body 281 303 282 304 305 + def _count_image_parts(value: Any) -> int: 306 + if is_image_part(value): 307 + return 1 308 + if isinstance(value, dict): 309 + return sum(_count_image_parts(item) for item in value.values()) 310 + if isinstance(value, list | tuple): 311 + return sum(_count_image_parts(item) for item in value) 312 + return 0 313 + 314 + 315 + def _serialized_message_text(messages: list[dict[str, Any]]) -> str: 316 + text_parts: list[str] = [] 317 + for message in messages: 318 + content = message.get("content") 319 + if isinstance(content, str): 320 + text_parts.append(content) 321 + elif isinstance(content, list): 322 + for part in content: 323 + if isinstance(part, dict) and part.get("type") == "text": 324 + text = part.get("text") 325 + if isinstance(text, str): 326 + text_parts.append(text) 327 + return "\n".join(text_parts) 328 + 329 + 283 330 def _extract_usage(data: dict[str, Any]) -> dict[str, int] | None: 284 331 usage = data.get("usage") 285 332 if not isinstance(usage, dict): ··· 435 482 return record 436 483 437 484 438 - def _classify_byo_generate_error(exc: BaseException) -> LocalProviderError: 485 + def _classify_byo_generate_error( 486 + exc: BaseException, 487 + endpoint: Any, 488 + ) -> LocalProviderError: 439 489 if is_byo_capacity_error(exc): 440 490 return LocalCapacityExhausted() 441 491 if is_byo_network_error(exc): ··· 443 493 "local_endpoint_unreachable", 444 494 LOCAL_ENDPOINT_UNREACHABLE_COPY, 445 495 ) 496 + response = getattr(exc, "response", None) 497 + body_text = getattr(response, "text", None) 498 + if isinstance(body_text, str) and body_text: 499 + excerpt = redact_local_endpoint_credential( 500 + body_text[:ENDPOINT_ERROR_BODY_CAP_CHARS], 501 + endpoint, 502 + ) 503 + if _contains_any(excerpt.lower(), _CONTEXT_WINDOW_PATTERNS): 504 + return _context_window_exceeded_error() 446 505 return LocalProviderError( 447 506 "local_endpoint_contract_failed", 448 507 LOCAL_ENDPOINT_CONTRACT_COPY, ··· 460 519 return remaining 461 520 462 521 522 + def _context_window_exceeded_error() -> LocalProviderError: 523 + return LocalProviderError( 524 + "context_window_exceeded", 525 + _ENDPOINT_CONTEXT_WINDOW_MESSAGE, 526 + ) 527 + 528 + 529 + def _endpoint_overflow_decision( 530 + body_text: str, 531 + served_window: int | None, 532 + attempt: int, 533 + ) -> _EndpointOverflowDecision: 534 + body_lower = body_text.lower() 535 + if _ENDPOINT_COMPLETION_ANCHOR in body_lower: 536 + limit_match = _ENDPOINT_LIMIT_RE.search(body_lower) 537 + input_match = _ENDPOINT_INPUT_RE.search(body_lower) 538 + limit = ( 539 + int(limit_match.group("limit")) 540 + if limit_match is not None 541 + else served_window 542 + ) 543 + if limit is not None and input_match is not None: 544 + reported_input = int(input_match.group("input")) 545 + new_max = limit - reported_input - _ENDPOINT_RECLAMP_SLACK_TOKENS 546 + if attempt == 0 and new_max >= _ENDPOINT_MIN_COMPLETION_TOKENS: 547 + return _EndpointOverflowDecision("retry", new_max) 548 + if attempt == 0: 549 + return _EndpointOverflowDecision("budget") 550 + return _EndpointOverflowDecision("context") 551 + 552 + if _contains_any(body_lower, _CONTEXT_WINDOW_PATTERNS): 553 + return _EndpointOverflowDecision("context") 554 + return _EndpointOverflowDecision("contract") 555 + 556 + 463 557 def _prepare_bundled_request( 464 558 *, 465 559 server: Any, ··· 496 590 ) 497 591 498 592 593 + def _prepare_endpoint_request( 594 + *, 595 + endpoint: Any, 596 + served_window: int | None, 597 + contents: str | list[Any], 598 + system_instruction: str | None, 599 + temperature: float, 600 + max_output_tokens: int, 601 + json_output: bool, 602 + json_schema: dict | None, 603 + ) -> tuple[dict[str, Any], dict[str, Any] | None, dict[str, int | None]]: 604 + from solstone.think.providers import local_budget 605 + 606 + if served_window is None: 607 + messages = _build_messages(contents, system_instruction) 608 + return ( 609 + _build_request_body( 610 + endpoint.served_model_id, 611 + messages, 612 + temperature, 613 + max_output_tokens, 614 + json_output, 615 + json_schema, 616 + endpoint.is_bundled, 617 + endpoint.is_confidential, 618 + ), 619 + None, 620 + { 621 + "served_window": None, 622 + "estimated_prompt_tokens": None, 623 + "clamped_max_tokens": max_output_tokens, 624 + "requested_max_output_tokens": max_output_tokens, 625 + }, 626 + ) 627 + 628 + fitted_contents, input_budget = local_budget.fit_contents( 629 + contents, 630 + system_instruction, 631 + max_output_tokens, 632 + count=local_budget.estimate_tokens, 633 + window=served_window, 634 + ) 635 + messages = _build_messages(fitted_contents, system_instruction) 636 + estimated_prompt_tokens = local_budget.estimate_tokens( 637 + _serialized_message_text(messages) 638 + ) + local_budget._ESTIMATED_IMAGE_TOKENS * _count_image_parts(fitted_contents) 639 + room = served_window - estimated_prompt_tokens - local_budget._SAFETY_MARGIN_TOKENS 640 + if room < _ENDPOINT_MIN_COMPLETION_TOKENS: 641 + raise ContextBudgetExceeded( 642 + "Local endpoint request prompt content exceeds the served context window." 643 + ) 644 + clamped_max_tokens = min(max_output_tokens, room) 645 + return ( 646 + _build_request_body( 647 + endpoint.served_model_id, 648 + messages, 649 + temperature, 650 + clamped_max_tokens, 651 + json_output, 652 + json_schema, 653 + endpoint.is_bundled, 654 + endpoint.is_confidential, 655 + ), 656 + input_budget, 657 + { 658 + "served_window": served_window, 659 + "estimated_prompt_tokens": estimated_prompt_tokens, 660 + "clamped_max_tokens": clamped_max_tokens, 661 + "requested_max_output_tokens": max_output_tokens, 662 + }, 663 + ) 664 + 665 + 666 + def _prepare_endpoint_request_with_resolution( 667 + *, 668 + endpoint: Any, 669 + contents: str | list[Any], 670 + system_instruction: str | None, 671 + temperature: float, 672 + max_output_tokens: int, 673 + json_output: bool, 674 + json_schema: dict | None, 675 + ) -> tuple[dict[str, Any], dict[str, Any] | None, dict[str, int | None]]: 676 + return _prepare_endpoint_request( 677 + endpoint=endpoint, 678 + served_window=resolve_endpoint_served_window(endpoint), 679 + contents=contents, 680 + system_instruction=system_instruction, 681 + temperature=temperature, 682 + max_output_tokens=max_output_tokens, 683 + json_output=json_output, 684 + json_schema=json_schema, 685 + ) 686 + 687 + 499 688 def _raise_bundled_status(response: Any) -> None: 500 689 import httpx 501 690 ··· 644 833 ) 645 834 raise 646 835 647 - messages = _build_messages(contents, system_instruction) 648 - body = _build_request_body( 649 - endpoint.served_model_id, 650 - messages, 651 - temperature, 652 - max_output_tokens, 653 - json_output, 654 - json_schema, 655 - endpoint.is_bundled, 836 + body, input_budget, endpoint_budget = _prepare_endpoint_request_with_resolution( 837 + endpoint=endpoint, 838 + contents=contents, 839 + system_instruction=system_instruction, 840 + temperature=temperature, 841 + max_output_tokens=max_output_tokens, 842 + json_output=json_output, 843 + json_schema=json_schema, 656 844 ) 657 845 658 846 import httpx ··· 686 874 try: 687 875 with admission: 688 876 base_url = confidential_egress_base_url(endpoint.base_url) 689 - response = httpx.post( 690 - f"{base_url}/v1/chat/completions", 691 - timeout=post_timeout, 692 - **post_kwargs, 693 - ) 694 - response.raise_for_status() 695 - return _parse_response(response.json()) 877 + attempt = 0 878 + while True: 879 + response = httpx.post( 880 + f"{base_url}/v1/chat/completions", 881 + timeout=( 882 + post_timeout 883 + if attempt == 0 884 + else _remaining_timeout(started, timeout) 885 + ), 886 + **post_kwargs, 887 + ) 888 + try: 889 + response.raise_for_status() 890 + break 891 + except httpx.HTTPStatusError as exc: 892 + if response.status_code != 400: 893 + raise 894 + decision = _endpoint_overflow_decision( 895 + response.text, 896 + endpoint_budget.get("served_window"), 897 + attempt, 898 + ) 899 + if decision.kind == "retry" and decision.max_tokens is not None: 900 + post_kwargs["json"] = { 901 + **post_kwargs["json"], 902 + "max_tokens": decision.max_tokens, 903 + } 904 + endpoint_budget = { 905 + **endpoint_budget, 906 + "clamped_max_tokens": decision.max_tokens, 907 + } 908 + attempt += 1 909 + continue 910 + if decision.kind == "budget": 911 + raise ContextBudgetExceeded( 912 + "Local endpoint request exceeded the served context " 913 + "window after completion re-clamp." 914 + ) from exc 915 + if decision.kind == "context": 916 + raise _context_window_exceeded_error() from exc 917 + raise 918 + result = _parse_response(response.json()) 919 + if input_budget is not None: 920 + result["input_budget"] = input_budget 921 + if endpoint_budget.get("served_window") is not None: 922 + result["endpoint_budget"] = endpoint_budget 923 + return result 696 924 except LocalAdmissionTimeout: 697 925 raise 698 926 except LocalProviderError: 699 927 raise 700 928 except Exception as exc: 701 - raise _classify_byo_generate_error(exc) from exc 929 + raise _classify_byo_generate_error(exc, endpoint) from exc 702 930 703 931 704 932 async def run_agenerate( ··· 725 953 import httpx 726 954 727 955 if not endpoint.is_bundled: 728 - messages = _build_messages(contents, system_instruction) 729 956 from solstone.think.providers.local_admission import ( 730 957 LocalAdmissionTimeout, 731 958 acquire_local_slot_async, ··· 734 961 confidential_egress_base_url, 735 962 ) 736 963 737 - body = _build_request_body( 738 - endpoint.served_model_id, 739 - messages, 740 - temperature, 741 - max_output_tokens, 742 - json_output, 743 - json_schema, 744 - False, 964 + body, input_budget, endpoint_budget = await asyncio.to_thread( 965 + _prepare_endpoint_request_with_resolution, 966 + endpoint=endpoint, 967 + contents=contents, 968 + system_instruction=system_instruction, 969 + temperature=temperature, 970 + max_output_tokens=max_output_tokens, 971 + json_output=json_output, 972 + json_schema=json_schema, 745 973 ) 746 974 post_kwargs: dict[str, Any] = { 747 975 "json": body, ··· 766 994 try: 767 995 async with admission: 768 996 base_url = confidential_egress_base_url(endpoint.base_url) 997 + attempt = 0 769 998 async with httpx.AsyncClient() as client: 770 - response = await client.post( 771 - f"{base_url}/v1/chat/completions", 772 - timeout=post_timeout, 773 - **post_kwargs, 774 - ) 775 - response.raise_for_status() 776 - return _parse_response(response.json()) 999 + while True: 1000 + response = await client.post( 1001 + f"{base_url}/v1/chat/completions", 1002 + timeout=( 1003 + post_timeout 1004 + if attempt == 0 1005 + else _remaining_timeout(started, timeout) 1006 + ), 1007 + **post_kwargs, 1008 + ) 1009 + try: 1010 + response.raise_for_status() 1011 + break 1012 + except httpx.HTTPStatusError as exc: 1013 + if response.status_code != 400: 1014 + raise 1015 + decision = _endpoint_overflow_decision( 1016 + response.text, 1017 + endpoint_budget.get("served_window"), 1018 + attempt, 1019 + ) 1020 + if ( 1021 + decision.kind == "retry" 1022 + and decision.max_tokens is not None 1023 + ): 1024 + post_kwargs["json"] = { 1025 + **post_kwargs["json"], 1026 + "max_tokens": decision.max_tokens, 1027 + } 1028 + endpoint_budget = { 1029 + **endpoint_budget, 1030 + "clamped_max_tokens": decision.max_tokens, 1031 + } 1032 + attempt += 1 1033 + continue 1034 + if decision.kind == "budget": 1035 + raise ContextBudgetExceeded( 1036 + "Local endpoint request exceeded the served " 1037 + "context window after completion re-clamp." 1038 + ) from exc 1039 + if decision.kind == "context": 1040 + raise _context_window_exceeded_error() from exc 1041 + raise 1042 + result = _parse_response(response.json()) 1043 + if input_budget is not None: 1044 + result["input_budget"] = input_budget 1045 + if endpoint_budget.get("served_window") is not None: 1046 + result["endpoint_budget"] = endpoint_budget 1047 + return result 777 1048 except asyncio.CancelledError: 778 1049 raise 779 1050 except LocalAdmissionTimeout: ··· 781 1052 except LocalProviderError: 782 1053 raise 783 1054 except Exception as exc: 784 - raise _classify_byo_generate_error(exc) from exc 1055 + raise _classify_byo_generate_error(exc, endpoint) from exc 785 1056 786 1057 from solstone.think.providers import local_server 787 1058 from solstone.think.providers.local_admission import (
+76 -27
solstone/think/providers/openhands.py
··· 89 89 _SHELL_STDERR_CAP = 6000 90 90 _SHELL_TIMEOUT_SECONDS = 30 91 91 _COST_WARNING_TEXT = "Cost calculation failed" 92 - _LOCAL_OUTPUT_RESERVE_TOKENS = LOCAL_MIN_CONTEXT_TOKENS // 4 93 - _LOCAL_CONDENSER_MAX_TOKENS = LOCAL_MIN_CONTEXT_TOKENS * 11 // 16 94 92 _LOCAL_CONDENSER_KEEP_FIRST = 4 95 93 _GENERATE_NUM_RETRIES = 2 96 94 _GEMINI_MAX_OUTPUT_TOKENS = 65_535 ··· 151 149 return effective 152 150 153 151 152 + def _local_output_reserve_tokens(window: int) -> int: 153 + return window // 4 154 + 155 + 156 + def _local_condenser_max_tokens(window: int) -> int: 157 + return window * 11 // 16 158 + 159 + 154 160 def _resolve_allowed_roots(config: dict[str, Any]) -> list[Path]: 155 161 journal = Path(get_journal()).resolve() 156 162 project_root = Path(get_project_root()).resolve() ··· 206 212 return max_turns, cost_cap, timeout_seconds 207 213 208 214 209 - def _build_llm(provider: str, model: str, *, num_retries: int | None = None) -> Any: 215 + def _build_llm( 216 + provider: str, 217 + model: str, 218 + *, 219 + num_retries: int | None = None, 220 + endpoint: Any | None = None, 221 + served_window: int | None = None, 222 + ) -> Any: 210 223 from openhands.sdk import LLM 211 224 212 225 retry_count = LLM_NUM_RETRIES if num_retries is None else num_retries 213 226 if provider == "local": 214 - from solstone.think.providers.local_endpoint import resolve_local_endpoint 215 227 from solstone.think.services.spp_transport import confidential_egress_base_url 216 228 217 - endpoint = resolve_local_endpoint() 229 + if endpoint is None: 230 + raise ValueError("Resolved local endpoint is required to build local LLM.") 218 231 if not endpoint.is_bundled: 219 232 base_url = confidential_egress_base_url(endpoint.base_url) 220 - return LLM( 221 - model=f"openai/{endpoint.served_model_id}", 222 - base_url=f"{base_url}/v1", 223 - api_key=endpoint.credential or "EMPTY", 224 - native_tool_calling=False, 225 - timeout=LLM_TIMEOUT_S, 226 - num_retries=retry_count, 227 - retry_min_wait=1, 228 - retry_max_wait=2, 229 - retry_multiplier=1.0, 230 - input_cost_per_token=0, 231 - output_cost_per_token=0, 232 - ) 233 + llm_kwargs: dict[str, Any] = { 234 + "model": f"openai/{endpoint.served_model_id}", 235 + "base_url": f"{base_url}/v1", 236 + "api_key": endpoint.credential or "EMPTY", 237 + "native_tool_calling": False, 238 + "timeout": LLM_TIMEOUT_S, 239 + "num_retries": retry_count, 240 + "retry_min_wait": 1, 241 + "retry_max_wait": 2, 242 + "retry_multiplier": 1.0, 243 + "input_cost_per_token": 0, 244 + "output_cost_per_token": 0, 245 + } 246 + if served_window is not None and served_window >= LOCAL_MIN_CONTEXT_TOKENS: 247 + llm_kwargs["max_input_tokens"] = served_window 248 + llm_kwargs["max_output_tokens"] = _local_output_reserve_tokens( 249 + served_window 250 + ) 251 + if endpoint.is_confidential: 252 + llm_kwargs["litellm_extra_body"] = { 253 + "chat_template_kwargs": {"enable_thinking": False} 254 + } 255 + return LLM(**llm_kwargs) 233 256 234 257 from solstone.think.providers import local_server 235 258 ··· 242 265 timeout=LLM_TIMEOUT_S, 243 266 num_retries=retry_count, 244 267 max_input_tokens=local_server.LOCAL_MIN_CONTEXT_TOKENS, 245 - max_output_tokens=_LOCAL_OUTPUT_RESERVE_TOKENS, 268 + max_output_tokens=_local_output_reserve_tokens( 269 + local_server.LOCAL_MIN_CONTEXT_TOKENS 270 + ), 246 271 input_cost_per_token=0, 247 272 output_cost_per_token=0, 248 273 litellm_extra_body={"chat_template_kwargs": {"enable_thinking": False}}, ··· 677 702 return _generate_result(response, model) 678 703 679 704 680 - def _build_local_condenser(llm: Any) -> Any: 681 - """LLM-summarizing condenser for the bundled-local floor window. 705 + def _build_local_condenser(llm: Any, *, max_tokens: int) -> Any: 706 + """LLM-summarizing condenser for an explicitly resolved local window. 682 707 683 708 Reuses the agent's own LLM (shared usage_id is accepted in 684 709 openhands-sdk 1.27.1) so there is no separate summarization endpoint. ··· 687 712 688 713 return LLMSummarizingCondenser( 689 714 llm=llm, 690 - max_tokens=_LOCAL_CONDENSER_MAX_TOKENS, 715 + max_tokens=max_tokens, 691 716 keep_first=_LOCAL_CONDENSER_KEEP_FIRST, 692 717 ) 693 718 ··· 695 720 def _build_cogitate_agent( 696 721 *, 697 722 llm: Any, 698 - is_bundled_local: bool, 723 + condenser_max_tokens: int | None, 699 724 tool_specs: list[Any], 700 725 include_default_tools: list[Any], 701 726 system_prompt: str, 702 727 ) -> Any: 703 728 from openhands.sdk import Agent 704 729 705 - condenser = _build_local_condenser(llm) if is_bundled_local else None 730 + condenser = ( 731 + _build_local_condenser(llm, max_tokens=condenser_max_tokens) 732 + if condenser_max_tokens is not None 733 + else None 734 + ) 706 735 return Agent( 707 736 llm=llm, 708 737 tools=tool_specs, ··· 1597 1626 model = str(config["model"]) 1598 1627 effective_on_event = on_event 1599 1628 byo_endpoint = None 1629 + byo_served_window = None 1600 1630 if provider == "local": 1601 1631 from solstone.think.providers.local_endpoint import ( 1632 + resolve_endpoint_served_window, 1602 1633 resolve_local_endpoint, 1603 1634 wrap_on_event_redacting, 1604 1635 ) 1605 1636 1606 1637 byo_endpoint = resolve_local_endpoint() 1638 + if not byo_endpoint.is_bundled: 1639 + byo_served_window = resolve_endpoint_served_window(byo_endpoint) 1607 1640 if not byo_endpoint.is_bundled and byo_endpoint.credential: 1608 1641 effective_on_event = wrap_on_event_redacting( 1609 1642 on_event, ··· 1642 1675 config.get("read_call_budget", DEFAULT_READ_CALL_BUDGET) or 0 1643 1676 ) 1644 1677 journal = Path(get_journal()) 1645 - llm = _build_llm(provider, model, num_retries=_llm_num_retries(config)) 1678 + llm = _build_llm( 1679 + provider, 1680 + model, 1681 + num_retries=_llm_num_retries(config), 1682 + endpoint=byo_endpoint, 1683 + served_window=byo_served_window, 1684 + ) 1646 1685 usage_start = _usage_snapshot(llm) 1647 1686 tool_specs = [] 1648 1687 sol_executor = None ··· 1679 1718 tool_specs.append(Tool(name="emit_final")) 1680 1719 default_tools = [] 1681 1720 1682 - is_bundled_local = byo_endpoint is not None and byo_endpoint.is_bundled 1721 + condenser_max_tokens = None 1722 + if byo_endpoint is not None: 1723 + if byo_endpoint.is_bundled: 1724 + condenser_max_tokens = _local_condenser_max_tokens( 1725 + LOCAL_MIN_CONTEXT_TOKENS 1726 + ) 1727 + elif ( 1728 + byo_served_window is not None 1729 + and byo_served_window >= LOCAL_MIN_CONTEXT_TOKENS 1730 + ): 1731 + condenser_max_tokens = _local_condenser_max_tokens(byo_served_window) 1683 1732 agent = _build_cogitate_agent( 1684 1733 llm=llm, 1685 - is_bundled_local=is_bundled_local, 1734 + condenser_max_tokens=condenser_max_tokens, 1686 1735 tool_specs=tool_specs, 1687 1736 include_default_tools=default_tools, 1688 1737 system_prompt=system_instruction,
+38 -13
tests/test_cogitate_local_condenser.py
··· 20 20 api_key="EMPTY", 21 21 native_tool_calling=False, 22 22 max_input_tokens=LOCAL_MIN_CONTEXT_TOKENS, 23 - max_output_tokens=openhands._LOCAL_OUTPUT_RESERVE_TOKENS, 23 + max_output_tokens=openhands._local_output_reserve_tokens( 24 + LOCAL_MIN_CONTEXT_TOKENS 25 + ), 24 26 ) 25 27 26 28 27 29 def test_build_cogitate_agent_adds_bundled_local_condenser(): 30 + condenser_max_tokens = openhands._local_condenser_max_tokens( 31 + LOCAL_MIN_CONTEXT_TOKENS 32 + ) 28 33 agent = openhands._build_cogitate_agent( 29 34 llm=_local_llm(), 30 - is_bundled_local=True, 35 + condenser_max_tokens=condenser_max_tokens, 31 36 tool_specs=[], 32 37 include_default_tools=[], 33 38 system_prompt="sys", 34 39 ) 35 40 36 41 assert isinstance(agent.condenser, LLMSummarizingCondenser) 37 - assert agent.condenser.max_tokens == openhands._LOCAL_CONDENSER_MAX_TOKENS 42 + assert agent.condenser.max_tokens == condenser_max_tokens 38 43 assert agent.condenser.keep_first == openhands._LOCAL_CONDENSER_KEEP_FIRST 39 44 assert agent.condenser.llm is agent.llm 40 45 ··· 42 47 def test_build_cogitate_agent_skips_condenser_for_non_bundled_local(): 43 48 agent = openhands._build_cogitate_agent( 44 49 llm=_local_llm(), 45 - is_bundled_local=False, 50 + condenser_max_tokens=None, 51 + tool_specs=[], 52 + include_default_tools=[], 53 + system_prompt="sys", 54 + ) 55 + 56 + assert agent.condenser is None 57 + 58 + 59 + def test_build_cogitate_agent_skips_condenser_for_small_endpoint_window(): 60 + window = 8192 61 + condenser_max_tokens = ( 62 + openhands._local_condenser_max_tokens(window) 63 + if window >= LOCAL_MIN_CONTEXT_TOKENS 64 + else None 65 + ) 66 + 67 + agent = openhands._build_cogitate_agent( 68 + llm=_local_llm(), 69 + condenser_max_tokens=condenser_max_tokens, 46 70 tool_specs=[], 47 71 include_default_tools=[], 48 72 system_prompt="sys", ··· 52 76 53 77 54 78 def test_local_condenser_window_invariants(): 55 - assert ( 56 - openhands._LOCAL_CONDENSER_MAX_TOKENS // 2 57 - + openhands._LOCAL_OUTPUT_RESERVE_TOKENS 58 - < LOCAL_MIN_CONTEXT_TOKENS 79 + condenser_max_tokens = openhands._local_condenser_max_tokens( 80 + LOCAL_MIN_CONTEXT_TOKENS 59 81 ) 60 - assert ( 61 - openhands._LOCAL_CONDENSER_MAX_TOKENS + openhands._LOCAL_OUTPUT_RESERVE_TOKENS 62 - <= LOCAL_MIN_CONTEXT_TOKENS 82 + output_reserve_tokens = openhands._local_output_reserve_tokens( 83 + LOCAL_MIN_CONTEXT_TOKENS 63 84 ) 64 - assert openhands._LOCAL_CONDENSER_MAX_TOKENS < LOCAL_MIN_CONTEXT_TOKENS 65 - assert 11000 <= openhands._LOCAL_CONDENSER_MAX_TOKENS <= 11500 85 + assert condenser_max_tokens // 2 + output_reserve_tokens < LOCAL_MIN_CONTEXT_TOKENS 86 + assert condenser_max_tokens + output_reserve_tokens <= LOCAL_MIN_CONTEXT_TOKENS 87 + assert condenser_max_tokens < LOCAL_MIN_CONTEXT_TOKENS 88 + assert 11000 <= condenser_max_tokens <= 11500 89 + assert openhands._local_output_reserve_tokens(16384) == 4096 90 + assert openhands._local_condenser_max_tokens(16384) == 11264 66 91 assert openhands._LOCAL_CONDENSER_KEEP_FIRST < 240 // 2 - 1 67 92 68 93
+762 -23
tests/test_local.py
··· 38 38 monkeypatch.setattr(local_admission, "record_local_inference", lambda _record: None) 39 39 40 40 41 + @pytest.fixture(autouse=True) 42 + def _default_endpoint_models_unknown(monkeypatch): 43 + import httpx 44 + 45 + from solstone.think.providers import local_endpoint 46 + 47 + local_endpoint.reset_endpoint_served_window_cache() 48 + 49 + def fake_get(url, **_kwargs): 50 + return httpx.Response( 51 + 404, 52 + request=httpx.Request("GET", url), 53 + text="not found", 54 + ) 55 + 56 + monkeypatch.setattr(httpx, "get", fake_get) 57 + yield 58 + local_endpoint.reset_endpoint_served_window_cache() 59 + 60 + 41 61 def _provider(): 42 62 providers_pkg = importlib.import_module("solstone.think.providers") 43 63 if hasattr(providers_pkg, "local_budget"): ··· 91 111 } 92 112 ], 93 113 } 114 + 115 + 116 + _FAKE_MODELS_BODY = ( 117 + '{"object":"list","data":[{"id":"Qwen/Qwen3.5-4B","object":"model",' 118 + '"created":1784825047,"owned_by":"sglang","root":"Qwen/Qwen3.5-4B",' 119 + '"parent":null,"max_model_len":16384}]}' 120 + ) 121 + _FAKE_F1_COMPLETION_OVERFLOW_BODY = ( 122 + '{"object":"error","message":"Requested token count exceeds the model\'s ' 123 + "maximum context length of 16384 tokens. You requested a total of 16397 " 124 + "tokens: 13 tokens from the input messages and 16384 tokens for the " 125 + "completion. Please reduce the number of tokens in the input messages or " 126 + 'the completion to fit within the limit.","type":"BadRequestError",' 127 + '"param":null,"code":400}' 128 + ) 129 + _FAKE_F2_PROMPT_OVERFLOW_BODY = ( 130 + '{"object":"error","message":"The input (18010 tokens) is longer than ' 131 + 'the model\'s context length (16384 tokens).","type":"BadRequestError",' 132 + '"param":null,"code":400}' 133 + ) 134 + _FAKE_REJECT_WINDOW = 16384 135 + 136 + 137 + class _FakeRejectEndpoint: 138 + def __init__( 139 + self, 140 + *, 141 + prompt_tokens: int = 13, 142 + models_body: str = _FAKE_MODELS_BODY, 143 + completion_overflow_body: str = _FAKE_F1_COMPLETION_OVERFLOW_BODY, 144 + prompt_overflow_body: str = _FAKE_F2_PROMPT_OVERFLOW_BODY, 145 + force_completion_overflow: bool = False, 146 + force_prompt_overflow: bool = False, 147 + force_non_context_400: bool = False, 148 + force_second_context_400: bool = False, 149 + ) -> None: 150 + self.prompt_tokens = prompt_tokens 151 + self.models_body = models_body 152 + self.completion_overflow_body = completion_overflow_body 153 + self.prompt_overflow_body = prompt_overflow_body 154 + self.force_completion_overflow = force_completion_overflow 155 + self.force_prompt_overflow = force_prompt_overflow 156 + self.force_non_context_400 = force_non_context_400 157 + self.force_second_context_400 = force_second_context_400 158 + self.gets: list[dict] = [] 159 + self.posts: list[dict] = [] 160 + 161 + @property 162 + def max_tokens(self) -> list[int]: 163 + return [int(post["json"]["max_tokens"]) for post in self.posts] 164 + 165 + def install(self, monkeypatch) -> None: 166 + import httpx 167 + 168 + monkeypatch.setattr(httpx, "get", self.get) 169 + monkeypatch.setattr(httpx, "post", self.post) 170 + 171 + fake_endpoint = self 172 + 173 + class AsyncClient: 174 + async def __aenter__(self): 175 + return self 176 + 177 + async def __aexit__(self, *_args): 178 + return None 179 + 180 + async def post(self, url, **kwargs): 181 + return fake_endpoint.post(url, **kwargs) 182 + 183 + monkeypatch.setattr(httpx, "AsyncClient", AsyncClient) 184 + 185 + def get(self, url, **kwargs): 186 + import httpx 187 + 188 + self.gets.append({"url": url, **kwargs}) 189 + request = httpx.Request("GET", url) 190 + if str(url).endswith("/v1/models"): 191 + return httpx.Response(200, request=request, text=self.models_body) 192 + return httpx.Response(404, request=request, text="not found") 193 + 194 + def post(self, url, **kwargs): 195 + import httpx 196 + 197 + request = httpx.Request("POST", url) 198 + if str(url).endswith("/tokenize"): 199 + content = str((kwargs.get("json") or {}).get("content") or "") 200 + return httpx.Response( 201 + 200, 202 + request=request, 203 + json={"tokens": list(range(max(1, len(content) // 3)))}, 204 + ) 205 + if not str(url).endswith("/v1/chat/completions"): 206 + raise AssertionError(f"unexpected local provider URL: {url}") 207 + 208 + body = kwargs["json"] 209 + self.posts.append({"url": url, **kwargs}) 210 + if self.force_non_context_400: 211 + return httpx.Response(400, request=request, text="invalid temperature") 212 + if self.force_prompt_overflow or self.prompt_tokens >= _FAKE_REJECT_WINDOW: 213 + return httpx.Response( 214 + 400, 215 + request=request, 216 + text=self.prompt_overflow_body, 217 + ) 218 + if ( 219 + self.force_completion_overflow 220 + or self.force_second_context_400 221 + and len(self.posts) > 1 222 + or self.prompt_tokens + int(body["max_tokens"]) > _FAKE_REJECT_WINDOW 223 + ): 224 + return httpx.Response( 225 + 400, 226 + request=request, 227 + text=self.completion_overflow_body, 228 + ) 229 + return httpx.Response( 230 + 200, 231 + request=request, 232 + json={ 233 + "choices": [ 234 + { 235 + "message": {"content": "ok"}, 236 + "finish_reason": "stop", 237 + } 238 + ], 239 + "usage": { 240 + "prompt_tokens": self.prompt_tokens, 241 + "completion_tokens": int(body["max_tokens"]), 242 + "total_tokens": self.prompt_tokens + int(body["max_tokens"]), 243 + }, 244 + }, 245 + ) 246 + 247 + 248 + def _http_status_error(body: str): 249 + import httpx 250 + 251 + request = httpx.Request("POST", "http://byo.example/openai/v1/chat/completions") 252 + response = httpx.Response(400, request=request, text=body) 253 + return httpx.HTTPStatusError("bad request", request=request, response=response) 94 254 95 255 96 256 def test_local_model_prefix_maps_to_provider(): ··· 854 1014 lambda: SimpleNamespace(port=9876, served_model_id=served_model_id), 855 1015 ) 856 1016 857 - llm = openhands._build_llm("local", LOCAL_MODEL) 1017 + llm = openhands._build_llm( 1018 + "local", 1019 + LOCAL_MODEL, 1020 + endpoint=_bundled_endpoint(), 1021 + ) 858 1022 859 1023 assert isinstance(llm, FakeLLM) 860 1024 assert captured == { ··· 865 1029 "timeout": openhands.LLM_TIMEOUT_S, 866 1030 "num_retries": openhands.LLM_NUM_RETRIES, 867 1031 "max_input_tokens": local_server.LOCAL_MIN_CONTEXT_TOKENS, 868 - "max_output_tokens": openhands._LOCAL_OUTPUT_RESERVE_TOKENS, 1032 + "max_output_tokens": openhands._local_output_reserve_tokens( 1033 + local_server.LOCAL_MIN_CONTEXT_TOKENS 1034 + ), 869 1035 "input_cost_per_token": 0, 870 1036 "output_cost_per_token": 0, 871 1037 "litellm_extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, ··· 896 1062 ) 897 1063 898 1064 1065 + def _qwen_byo_endpoint(): 1066 + from solstone.think.providers.local_endpoint import ( 1067 + LocalEndpoint, 1068 + normalize_local_endpoint_url, 1069 + ) 1070 + 1071 + return LocalEndpoint( 1072 + base_url=normalize_local_endpoint_url("http://byo.example/openai/v1/"), 1073 + served_model_id="Qwen/Qwen3.5-4B", 1074 + credential="test-token-PLACEHOLDER", 1075 + is_bundled=False, 1076 + ) 1077 + 1078 + 899 1079 def _bundled_endpoint(): 900 1080 from solstone.think.providers.local_endpoint import LocalEndpoint 901 1081 ··· 919 1099 ) 920 1100 921 1101 1102 + def test_run_generate_endpoint_clamps_large_default_budget_against_served_window( 1103 + monkeypatch, 1104 + ): 1105 + provider = _provider() 1106 + fake_endpoint = _FakeRejectEndpoint() 1107 + fake_endpoint.install(monkeypatch) 1108 + monkeypatch.setattr(provider, "resolve_local_endpoint", _qwen_byo_endpoint) 1109 + 1110 + result = provider.run_generate( 1111 + "hello", 1112 + model=LOCAL_MODEL, 1113 + max_output_tokens=49152, 1114 + ) 1115 + 1116 + assert result["text"] == "ok" 1117 + assert fake_endpoint.gets 1118 + assert fake_endpoint.max_tokens 1119 + assert all(token <= _FAKE_REJECT_WINDOW for token in fake_endpoint.max_tokens) 1120 + 1121 + 1122 + def test_run_agenerate_endpoint_clamps_window_sized_budget_against_served_window( 1123 + monkeypatch, 1124 + ): 1125 + provider = _provider() 1126 + fake_endpoint = _FakeRejectEndpoint() 1127 + fake_endpoint.install(monkeypatch) 1128 + monkeypatch.setattr(provider, "resolve_local_endpoint", _qwen_byo_endpoint) 1129 + 1130 + result = asyncio.run( 1131 + provider.run_agenerate( 1132 + "hello", 1133 + model=LOCAL_MODEL, 1134 + max_output_tokens=16384, 1135 + ) 1136 + ) 1137 + 1138 + assert result["text"] == "ok" 1139 + assert fake_endpoint.gets 1140 + assert fake_endpoint.max_tokens 1141 + assert all(token <= _FAKE_REJECT_WINDOW for token in fake_endpoint.max_tokens) 1142 + 1143 + 1144 + def test_run_generate_endpoint_preflight_floor_skips_chat_post(monkeypatch): 1145 + provider = _provider() 1146 + fake_endpoint = _FakeRejectEndpoint() 1147 + fake_endpoint.install(monkeypatch) 1148 + monkeypatch.setattr(provider, "resolve_local_endpoint", _qwen_byo_endpoint) 1149 + 1150 + with pytest.raises(provider.ContextBudgetExceeded) as exc: 1151 + provider.run_generate( 1152 + [{"role": "user", "content": "x" * 50000}], 1153 + model=LOCAL_MODEL, 1154 + max_output_tokens=1024, 1155 + ) 1156 + 1157 + assert exc.value.reason_code == "context_budget_exceeded" 1158 + assert fake_endpoint.gets 1159 + assert fake_endpoint.posts == [] 1160 + 1161 + 1162 + def test_validate_key_known_window_keeps_tiny_max_tokens(monkeypatch): 1163 + provider = _provider() 1164 + fake_endpoint = _FakeRejectEndpoint() 1165 + fake_endpoint.install(monkeypatch) 1166 + monkeypatch.setattr(provider, "resolve_local_endpoint", _qwen_byo_endpoint) 1167 + 1168 + assert provider.validate_key("local", "") == {"valid": True} 1169 + assert fake_endpoint.max_tokens == [8] 1170 + 1171 + 1172 + def test_run_generate_endpoint_known_window_truncates_fittable_input(monkeypatch): 1173 + provider = _provider() 1174 + fake_endpoint = _FakeRejectEndpoint() 1175 + fake_endpoint.install(monkeypatch) 1176 + monkeypatch.setattr(provider, "resolve_local_endpoint", _qwen_byo_endpoint) 1177 + chunks = [ 1178 + f"## 2026-06-23 09:{minute:02d}:00\n### Transcript\n" 1179 + + (str(minute) * 3000) 1180 + + "\n" 1181 + for minute in range(20) 1182 + ] 1183 + 1184 + result = provider.run_generate( 1185 + "".join(chunks), 1186 + model=LOCAL_MODEL, 1187 + max_output_tokens=1024, 1188 + ) 1189 + 1190 + from solstone.think.providers import local_budget 1191 + 1192 + posted_content = fake_endpoint.posts[0]["json"]["messages"][0]["content"] 1193 + assert local_budget.TRUNCATION_MARKER in posted_content 1194 + assert result["input_budget"]["clipped"] is True 1195 + assert result["input_budget"]["dropped_entries"] > 0 1196 + assert result["endpoint_budget"]["served_window"] == _FAKE_REJECT_WINDOW 1197 + assert result["endpoint_budget"]["clamped_max_tokens"] == 1024 1198 + 1199 + 1200 + def test_endpoint_generate_branches_route_shared_prep(monkeypatch): 1201 + provider = _provider() 1202 + calls = [] 1203 + original = provider._prepare_endpoint_request 1204 + monkeypatch.setattr(provider, "resolve_local_endpoint", _byo_endpoint) 1205 + monkeypatch.setattr( 1206 + provider, 1207 + "resolve_endpoint_served_window", 1208 + lambda _endpoint: None, 1209 + ) 1210 + 1211 + def spy_prepare(**kwargs): 1212 + calls.append(kwargs) 1213 + return original(**kwargs) 1214 + 1215 + def fake_post(url, **kwargs): 1216 + return _ChatResponse("ok") 1217 + 1218 + class AsyncClient: 1219 + async def __aenter__(self): 1220 + return self 1221 + 1222 + async def __aexit__(self, *_args): 1223 + return None 1224 + 1225 + async def post(self, url, **kwargs): 1226 + return _ChatResponse("ok") 1227 + 1228 + import httpx 1229 + 1230 + monkeypatch.setattr(provider, "_prepare_endpoint_request", spy_prepare) 1231 + monkeypatch.setattr(httpx, "post", fake_post) 1232 + monkeypatch.setattr(httpx, "AsyncClient", AsyncClient) 1233 + 1234 + provider.run_generate("hello", model=LOCAL_MODEL, max_output_tokens=7) 1235 + asyncio.run(provider.run_agenerate("hello", model=LOCAL_MODEL, max_output_tokens=7)) 1236 + 1237 + assert [call["max_output_tokens"] for call in calls] == [7, 7] 1238 + assert [call["served_window"] for call in calls] == [None, None] 1239 + 1240 + 1241 + def _run_endpoint_generate(provider, mode: str, **kwargs): 1242 + if mode == "sync": 1243 + return provider.run_generate(**kwargs) 1244 + return asyncio.run(provider.run_agenerate(**kwargs)) 1245 + 1246 + 1247 + @pytest.mark.parametrize("mode", ["sync", "async"]) 1248 + def test_endpoint_completion_overflow_reclamps_once(monkeypatch, mode): 1249 + provider = _provider() 1250 + fake_endpoint = _FakeRejectEndpoint() 1251 + fake_endpoint.install(monkeypatch) 1252 + monkeypatch.setattr(provider, "resolve_local_endpoint", _byo_endpoint) 1253 + monkeypatch.setattr( 1254 + provider, 1255 + "resolve_endpoint_served_window", 1256 + lambda _endpoint: None, 1257 + ) 1258 + 1259 + result = _run_endpoint_generate( 1260 + provider, 1261 + mode, 1262 + contents="hello", 1263 + model=LOCAL_MODEL, 1264 + max_output_tokens=16384, 1265 + ) 1266 + 1267 + assert result["text"] == "ok" 1268 + assert fake_endpoint.max_tokens == [16384, 16355] 1269 + 1270 + 1271 + @pytest.mark.parametrize("mode", ["sync", "async"]) 1272 + def test_endpoint_completion_overflow_below_floor_is_budget_terminal( 1273 + monkeypatch, 1274 + mode, 1275 + ): 1276 + provider = _provider() 1277 + low_room_body = _FAKE_F1_COMPLETION_OVERFLOW_BODY.replace( 1278 + "13 tokens from the input messages and 16384 tokens for the completion", 1279 + "16200 tokens from the input messages and 500 tokens for the completion", 1280 + ) 1281 + fake_endpoint = _FakeRejectEndpoint( 1282 + completion_overflow_body=low_room_body, 1283 + force_completion_overflow=True, 1284 + ) 1285 + fake_endpoint.install(monkeypatch) 1286 + monkeypatch.setattr(provider, "resolve_local_endpoint", _byo_endpoint) 1287 + monkeypatch.setattr( 1288 + provider, 1289 + "resolve_endpoint_served_window", 1290 + lambda _endpoint: None, 1291 + ) 1292 + 1293 + with pytest.raises(provider.ContextBudgetExceeded) as exc: 1294 + _run_endpoint_generate( 1295 + provider, 1296 + mode, 1297 + contents="hello", 1298 + model=LOCAL_MODEL, 1299 + max_output_tokens=500, 1300 + ) 1301 + 1302 + assert exc.value.reason_code == "context_budget_exceeded" 1303 + assert fake_endpoint.max_tokens == [500] 1304 + 1305 + 1306 + @pytest.mark.parametrize("mode", ["sync", "async"]) 1307 + def test_endpoint_prompt_overflow_is_terminal_context_window(monkeypatch, mode): 1308 + provider = _provider() 1309 + fake_endpoint = _FakeRejectEndpoint(force_prompt_overflow=True) 1310 + fake_endpoint.install(monkeypatch) 1311 + monkeypatch.setattr(provider, "resolve_local_endpoint", _byo_endpoint) 1312 + monkeypatch.setattr( 1313 + provider, 1314 + "resolve_endpoint_served_window", 1315 + lambda _endpoint: None, 1316 + ) 1317 + 1318 + with pytest.raises(provider.LocalProviderError) as exc: 1319 + _run_endpoint_generate( 1320 + provider, 1321 + mode, 1322 + contents="hello", 1323 + model=LOCAL_MODEL, 1324 + max_output_tokens=1024, 1325 + ) 1326 + 1327 + assert exc.value.reason_code == "context_window_exceeded" 1328 + assert fake_endpoint.max_tokens == [1024] 1329 + 1330 + 1331 + @pytest.mark.parametrize("mode", ["sync", "async"]) 1332 + def test_endpoint_non_context_400_stays_contract_failed(monkeypatch, mode): 1333 + provider = _provider() 1334 + fake_endpoint = _FakeRejectEndpoint(force_non_context_400=True) 1335 + fake_endpoint.install(monkeypatch) 1336 + monkeypatch.setattr(provider, "resolve_local_endpoint", _byo_endpoint) 1337 + monkeypatch.setattr( 1338 + provider, 1339 + "resolve_endpoint_served_window", 1340 + lambda _endpoint: None, 1341 + ) 1342 + 1343 + with pytest.raises(provider.LocalProviderError) as exc: 1344 + _run_endpoint_generate( 1345 + provider, 1346 + mode, 1347 + contents="hello", 1348 + model=LOCAL_MODEL, 1349 + max_output_tokens=1024, 1350 + ) 1351 + 1352 + assert exc.value.reason_code == "local_endpoint_contract_failed" 1353 + assert fake_endpoint.max_tokens == [1024] 1354 + 1355 + 1356 + @pytest.mark.parametrize("mode", ["sync", "async"]) 1357 + def test_endpoint_second_context_400_after_reclamp_is_terminal(monkeypatch, mode): 1358 + provider = _provider() 1359 + fake_endpoint = _FakeRejectEndpoint(force_second_context_400=True) 1360 + fake_endpoint.install(monkeypatch) 1361 + monkeypatch.setattr(provider, "resolve_local_endpoint", _byo_endpoint) 1362 + monkeypatch.setattr( 1363 + provider, 1364 + "resolve_endpoint_served_window", 1365 + lambda _endpoint: None, 1366 + ) 1367 + 1368 + with pytest.raises(provider.LocalProviderError) as exc: 1369 + _run_endpoint_generate( 1370 + provider, 1371 + mode, 1372 + contents="hello", 1373 + model=LOCAL_MODEL, 1374 + max_output_tokens=16384, 1375 + ) 1376 + 1377 + assert exc.value.reason_code == "context_window_exceeded" 1378 + assert fake_endpoint.max_tokens == [16384, 16355] 1379 + 1380 + 1381 + def test_run_generate_byo_unknown_window_request_body_matches_golden( 1382 + monkeypatch, 1383 + ): 1384 + provider = _provider() 1385 + monkeypatch.setattr(provider, "resolve_local_endpoint", _byo_endpoint) 1386 + monkeypatch.setattr( 1387 + provider, 1388 + "resolve_endpoint_served_window", 1389 + lambda _endpoint: None, 1390 + raising=False, 1391 + ) 1392 + captured = {} 1393 + 1394 + def fake_post(url, **kwargs): 1395 + captured.update({"url": url, **kwargs}) 1396 + return _ChatResponse("ok") 1397 + 1398 + import httpx 1399 + 1400 + monkeypatch.setattr(httpx, "post", fake_post) 1401 + 1402 + provider.run_generate( 1403 + "hello", 1404 + model=LOCAL_MODEL, 1405 + temperature=0.4, 1406 + max_output_tokens=7, 1407 + ) 1408 + 1409 + assert captured["json"] == { 1410 + "model": "served-model", 1411 + "messages": [{"role": "user", "content": "hello"}], 1412 + "temperature": 0.4, 1413 + "max_tokens": 7, 1414 + "stream": False, 1415 + } 1416 + 1417 + 1418 + def test_run_generate_confidential_body_carries_qwen_block(monkeypatch): 1419 + provider = _provider() 1420 + from solstone.think.providers.local_endpoint import LocalEndpoint 1421 + 1422 + endpoint = LocalEndpoint( 1423 + base_url="https://spp.example.test", 1424 + served_model_id="confidential-model", 1425 + credential="confidential-token", 1426 + is_bundled=False, 1427 + is_confidential=True, 1428 + ) 1429 + captured = {} 1430 + monkeypatch.setattr(provider, "resolve_local_endpoint", lambda: endpoint) 1431 + monkeypatch.setattr( 1432 + provider, 1433 + "resolve_endpoint_served_window", 1434 + lambda _endpoint: None, 1435 + ) 1436 + monkeypatch.setattr( 1437 + "solstone.think.services.spp_transport.confidential_egress_base_url", 1438 + lambda _base_url: "http://127.0.0.1:4567", 1439 + ) 1440 + 1441 + def fake_post(url, **kwargs): 1442 + captured.update({"url": url, **kwargs}) 1443 + return _ChatResponse("ok") 1444 + 1445 + import httpx 1446 + 1447 + monkeypatch.setattr(httpx, "post", fake_post) 1448 + 1449 + provider.run_generate( 1450 + "hello", 1451 + model=LOCAL_MODEL, 1452 + temperature=0.4, 1453 + max_output_tokens=7, 1454 + ) 1455 + 1456 + assert captured["url"] == "http://127.0.0.1:4567/v1/chat/completions" 1457 + assert captured["json"]["chat_template_kwargs"] == {"enable_thinking": False} 1458 + assert captured["json"]["top_p"] == 0.8 1459 + assert captured["json"]["top_k"] == 20 1460 + assert captured["json"]["min_p"] == 0.0 1461 + assert captured["json"]["presence_penalty"] == 1.5 1462 + 1463 + 1464 + def test_run_generate_bundled_request_body_matches_golden(monkeypatch): 1465 + provider = _provider() 1466 + monkeypatch.setattr(provider, "resolve_local_endpoint", _bundled_endpoint) 1467 + _patch_bundled_server(monkeypatch) 1468 + captured = {} 1469 + 1470 + def fake_post(url, **kwargs): 1471 + if str(url).endswith("/tokenize"): 1472 + return _FakeRejectEndpoint().post(url, **kwargs) 1473 + if str(url).endswith("/v1/chat/completions"): 1474 + captured.update({"url": url, **kwargs}) 1475 + return _ChatResponse("ok") 1476 + raise AssertionError(f"unexpected local provider URL: {url}") 1477 + 1478 + import httpx 1479 + 1480 + monkeypatch.setattr(httpx, "post", fake_post) 1481 + 1482 + provider.run_generate( 1483 + "hello", 1484 + model=LOCAL_MODEL, 1485 + temperature=0.4, 1486 + max_output_tokens=7, 1487 + ) 1488 + 1489 + assert captured["json"] == { 1490 + "model": LOCAL_MODEL, 1491 + "messages": [{"role": "user", "content": "hello"}], 1492 + "temperature": 0.4, 1493 + "max_tokens": 7, 1494 + "stream": False, 1495 + "chat_template_kwargs": {"enable_thinking": False}, 1496 + "top_p": 0.8, 1497 + "top_k": 20, 1498 + "min_p": 0.0, 1499 + "presence_penalty": 1.5, 1500 + } 1501 + 1502 + 922 1503 def test_run_generate_bundled_encodes_image_once(monkeypatch): 923 1504 provider = _provider() 924 1505 monkeypatch.setattr(provider, "resolve_local_endpoint", _bundled_endpoint) ··· 1040 1621 "resolve_local_endpoint", 1041 1622 lambda: _byo_endpoint(parallel_slots=1), 1042 1623 ) 1624 + monkeypatch.setattr( 1625 + provider, 1626 + "resolve_endpoint_served_window", 1627 + lambda _endpoint: None, 1628 + ) 1043 1629 captured = {} 1044 1630 1045 1631 def fake_post(url, **kwargs): ··· 1068 1654 provider, 1069 1655 "resolve_local_endpoint", 1070 1656 lambda: _byo_endpoint(parallel_slots=1), 1657 + ) 1658 + monkeypatch.setattr( 1659 + provider, 1660 + "resolve_endpoint_served_window", 1661 + lambda _endpoint: None, 1071 1662 ) 1072 1663 1073 1664 def fake_post(*_args, **_kwargs): ··· 1095 1686 provider, 1096 1687 "resolve_local_endpoint", 1097 1688 lambda: _byo_endpoint(parallel_slots=1), 1689 + ) 1690 + monkeypatch.setattr( 1691 + provider, 1692 + "resolve_endpoint_served_window", 1693 + lambda _endpoint: None, 1098 1694 ) 1099 1695 captured = {} 1100 1696 times = iter([100.0, 100.0, 100.35]) ··· 1311 1907 provider, 1312 1908 "resolve_local_endpoint", 1313 1909 lambda: _byo_endpoint(parallel_slots=1), 1910 + ) 1911 + monkeypatch.setattr( 1912 + provider, 1913 + "resolve_endpoint_served_window", 1914 + lambda _endpoint: None, 1314 1915 ) 1315 1916 captured = {} 1316 1917 ··· 1716 2317 exc = RuntimeError("outer") 1717 2318 exc.__cause__ = inner 1718 2319 1719 - classified = provider._classify_byo_generate_error(exc) 2320 + classified = provider._classify_byo_generate_error(exc, _byo_endpoint()) 1720 2321 1721 2322 assert classified.reason_code == "local_capacity_exhausted" 1722 2323 assert is_blocking_reason(classified.reason_code) is False ··· 1740 2341 exc = RuntimeError("outer") 1741 2342 exc.__cause__ = inner 1742 2343 1743 - classified = provider._classify_byo_generate_error(exc) 2344 + classified = provider._classify_byo_generate_error(exc, _byo_endpoint()) 1744 2345 1745 2346 assert classified.reason_code == "local_endpoint_unreachable" 1746 2347 assert str(classified) == provider.LOCAL_ENDPOINT_UNREACHABLE_COPY ··· 1756 2357 outer.__cause__ = unreachable 1757 2358 unreachable.__cause__ = capacity 1758 2359 1759 - classified = provider._classify_byo_generate_error(outer) 2360 + classified = provider._classify_byo_generate_error(outer, _byo_endpoint()) 1760 2361 1761 2362 assert classified.reason_code == "local_capacity_exhausted" 1762 2363 ··· 1768 2369 status_code = 500 1769 2370 1770 2371 classified = provider._classify_byo_generate_error( 1771 - InternalServerError("server failed") 2372 + InternalServerError("server failed"), 2373 + _byo_endpoint(), 1772 2374 ) 1773 2375 1774 2376 assert classified.reason_code == "local_endpoint_contract_failed" 1775 2377 assert str(classified) == provider.LOCAL_ENDPOINT_CONTRACT_COPY 1776 2378 1777 2379 2380 + @pytest.mark.parametrize( 2381 + "body", 2382 + [ 2383 + _FAKE_F1_COMPLETION_OVERFLOW_BODY, 2384 + _FAKE_F2_PROMPT_OVERFLOW_BODY, 2385 + ], 2386 + ) 2387 + def test_classify_byo_generate_error_context_body_maps_to_context_window(body): 2388 + provider = _provider() 2389 + endpoint = _byo_endpoint() 2390 + 2391 + classified = provider._classify_byo_generate_error( 2392 + _http_status_error(body), 2393 + endpoint, 2394 + ) 2395 + 2396 + assert classified.reason_code == "context_window_exceeded" 2397 + assert ( 2398 + str(classified) 2399 + == "The configured endpoint rejected the request: prompt and completion " 2400 + "exceed the served context window." 2401 + ) 2402 + 2403 + 2404 + @pytest.mark.parametrize( 2405 + "body", 2406 + [ 2407 + _FAKE_F1_COMPLETION_OVERFLOW_BODY, 2408 + _FAKE_F2_PROMPT_OVERFLOW_BODY, 2409 + ], 2410 + ) 2411 + def test_classify_byo_cogitate_error_context_body_maps_to_context_window(body): 2412 + from solstone.think.providers import local_endpoint 2413 + 2414 + class BadRequestError(RuntimeError): 2415 + status_code = 400 2416 + 2417 + def __init__(self, message: str, payload: str) -> None: 2418 + super().__init__(message) 2419 + self.message = message 2420 + self.body = payload 2421 + 2422 + exc = BadRequestError("litellm bad request wrapper", body) 2423 + 2424 + assert local_endpoint.classify_byo_cogitate_error(exc) == "context_window_exceeded" 2425 + 2426 + 1778 2427 def test_run_generate_byo_http_status_maps_to_contract_failed(monkeypatch): 1779 2428 provider = _provider() 1780 2429 monkeypatch.setattr(provider, "resolve_local_endpoint", _byo_endpoint) ··· 2278 2927 ], 2279 2928 ) 2280 2929 def test_openhands_local_byo_llm_kwargs(monkeypatch, credential, expected_key): 2281 - from solstone.think.providers import local_endpoint, openhands 2930 + from solstone.think.providers import openhands 2282 2931 2283 2932 captured = {} 2284 2933 ··· 2290 2939 sdk_module.LLM = FakeLLM 2291 2940 monkeypatch.setitem(sys.modules, "openhands.sdk", sdk_module) 2292 2941 monkeypatch.setattr( 2293 - local_endpoint, 2294 - "resolve_local_endpoint", 2295 - lambda: _byo_endpoint(credential), 2296 - ) 2297 - monkeypatch.setattr( 2298 2942 "solstone.think.providers.local_server.connect", 2299 2943 lambda: (_ for _ in ()).throw(AssertionError("connect not expected")), 2300 2944 ) 2301 2945 2302 - llm = openhands._build_llm("local", LOCAL_MODEL) 2946 + llm = openhands._build_llm( 2947 + "local", 2948 + LOCAL_MODEL, 2949 + endpoint=_byo_endpoint(credential), 2950 + served_window=None, 2951 + ) 2303 2952 2304 2953 assert isinstance(llm, FakeLLM) 2305 2954 assert captured == { ··· 2345 2994 sdk_module = types.ModuleType("openhands.sdk") 2346 2995 sdk_module.LLM = FakeLLM 2347 2996 monkeypatch.setitem(sys.modules, "openhands.sdk", sdk_module) 2348 - monkeypatch.setattr( 2349 - local_endpoint, 2350 - "resolve_local_endpoint", 2351 - lambda: local_endpoint.LocalEndpoint( 2352 - base_url=configured_endpoint, 2353 - served_model_id="confidential-model", 2354 - credential="confidential-token", 2355 - is_bundled=False, 2356 - ), 2997 + endpoint = local_endpoint.LocalEndpoint( 2998 + base_url=configured_endpoint, 2999 + served_model_id="confidential-model", 3000 + credential="confidential-token", 3001 + is_bundled=False, 3002 + is_confidential=True, 2357 3003 ) 2358 3004 monkeypatch.setattr( 2359 3005 "solstone.think.services.spp_transport.confidential_egress_base_url", ··· 2364 3010 lambda: (_ for _ in ()).throw(AssertionError("connect not expected")), 2365 3011 ) 2366 3012 2367 - llm = openhands._build_llm("local", LOCAL_MODEL) 3013 + llm = openhands._build_llm( 3014 + "local", 3015 + LOCAL_MODEL, 3016 + endpoint=endpoint, 3017 + served_window=None, 3018 + ) 2368 3019 2369 3020 assert isinstance(llm, FakeLLM) 2370 3021 assert captured["base_url"] == f"{forwarder}/v1" 2371 3022 assert configured_endpoint not in captured["base_url"] 3023 + 3024 + 3025 + def test_openhands_local_confidential_known_window_caps_and_extra_body(monkeypatch): 3026 + from solstone.think.providers import local_endpoint, openhands 3027 + 3028 + captured = {} 3029 + 3030 + class FakeLLM: 3031 + def __init__(self, **kwargs): 3032 + captured.update(kwargs) 3033 + 3034 + sdk_module = types.ModuleType("openhands.sdk") 3035 + sdk_module.LLM = FakeLLM 3036 + monkeypatch.setitem(sys.modules, "openhands.sdk", sdk_module) 3037 + endpoint = local_endpoint.LocalEndpoint( 3038 + base_url="https://spp.example.test", 3039 + served_model_id="confidential-model", 3040 + credential="confidential-token", 3041 + is_bundled=False, 3042 + is_confidential=True, 3043 + ) 3044 + monkeypatch.setattr( 3045 + "solstone.think.services.spp_transport.confidential_egress_base_url", 3046 + lambda _base_url: "http://127.0.0.1:4567", 3047 + ) 3048 + 3049 + openhands._build_llm( 3050 + "local", 3051 + LOCAL_MODEL, 3052 + endpoint=endpoint, 3053 + served_window=16384, 3054 + ) 3055 + 3056 + assert captured["max_input_tokens"] == 16384 3057 + assert captured["max_output_tokens"] == 4096 3058 + assert captured["litellm_extra_body"] == { 3059 + "chat_template_kwargs": {"enable_thinking": False} 3060 + } 3061 + 3062 + 3063 + def test_openhands_local_known_window_caps_without_extra_body(monkeypatch): 3064 + from solstone.think.providers import openhands 3065 + 3066 + captured = {} 3067 + 3068 + class FakeLLM: 3069 + def __init__(self, **kwargs): 3070 + captured.update(kwargs) 3071 + 3072 + sdk_module = types.ModuleType("openhands.sdk") 3073 + sdk_module.LLM = FakeLLM 3074 + monkeypatch.setitem(sys.modules, "openhands.sdk", sdk_module) 3075 + 3076 + openhands._build_llm( 3077 + "local", 3078 + LOCAL_MODEL, 3079 + endpoint=_qwen_byo_endpoint(), 3080 + served_window=32768, 3081 + ) 3082 + 3083 + assert captured["max_input_tokens"] == 32768 3084 + assert captured["max_output_tokens"] == 8192 3085 + assert "litellm_extra_body" not in captured 3086 + 3087 + 3088 + def test_openhands_local_small_window_omits_caps_and_extra_body(monkeypatch): 3089 + from solstone.think.providers import openhands 3090 + 3091 + captured = {} 3092 + 3093 + class FakeLLM: 3094 + def __init__(self, **kwargs): 3095 + captured.update(kwargs) 3096 + 3097 + sdk_module = types.ModuleType("openhands.sdk") 3098 + sdk_module.LLM = FakeLLM 3099 + monkeypatch.setitem(sys.modules, "openhands.sdk", sdk_module) 3100 + 3101 + openhands._build_llm( 3102 + "local", 3103 + LOCAL_MODEL, 3104 + endpoint=_qwen_byo_endpoint(), 3105 + served_window=8192, 3106 + ) 3107 + 3108 + assert "max_input_tokens" not in captured 3109 + assert "max_output_tokens" not in captured 3110 + assert "litellm_extra_body" not in captured 2372 3111 2373 3112 2374 3113 def test_local_context_window_split_floor_vs_tier():
+5 -1
tests/test_log_policy.py
··· 66 66 real_apply_http_logging_policy = apply_http_logging_policy 67 67 68 68 def fail_build_llm( 69 - provider: str, model: str, *, num_retries: int | None = None 69 + provider: str, 70 + model: str, 71 + *, 72 + num_retries: int | None = None, 73 + **_kwargs: Any, 70 74 ) -> Any: 71 75 raise RuntimeError("sentinel") 72 76
+109 -1
tests/test_openhands_errors.py
··· 165 165 ): 166 166 build_exc = RuntimeError("llm exploded") 167 167 168 - def fail_build(_provider, _model, *, num_retries=None): 168 + def fail_build(_provider, _model, *, num_retries=None, **_kwargs): 169 169 raise build_exc 170 170 171 171 monkeypatch.setattr(openhands, "_build_llm", fail_build) ··· 247 247 assert events[0]["error"] == LOCAL_ENDPOINT_CONTRACT_COPY 248 248 assert events[0]["reason_code"] == "local_endpoint_contract_failed" 249 249 assert token not in events[0]["trace"] 250 + 251 + 252 + def test_run_cogitate_local_byo_context_body_event_redacts( 253 + fake_openhands, 254 + run_env, 255 + monkeypatch, 256 + ): 257 + from tests.test_local import _FAKE_F2_PROMPT_OVERFLOW_BODY 258 + 259 + token = "SENTINEL-BYO-CONTEXT-CRED-219a" 260 + endpoint = LocalEndpoint( 261 + base_url="http://byo.example/openai", 262 + served_model_id="served-model", 263 + credential=token, 264 + is_bundled=False, 265 + ) 266 + 267 + class BadRequestError(RuntimeError): 268 + status_code = 400 269 + 270 + def __init__(self) -> None: 271 + super().__init__(f"bad request with {token}") 272 + self.message = f"bad request with {token}" 273 + self.body = _FAKE_F2_PROMPT_OVERFLOW_BODY + f" {token}" + ("x" * 6000) 274 + 275 + async def fail(_conversation): 276 + raise BadRequestError() 277 + 278 + monkeypatch.setattr( 279 + "solstone.think.providers.local_endpoint.resolve_local_endpoint", 280 + lambda: endpoint, 281 + ) 282 + fake_openhands.Conversation.arun_impl = fail 283 + events: list[dict] = [] 284 + local_env = {**run_env, "provider": "local", "model": LOCAL_MODEL} 285 + 286 + with pytest.raises(BadRequestError) as raised: 287 + asyncio.run(openhands.run_cogitate(local_env, events.append)) 288 + 289 + assert len(events) == 1 290 + assert events[0]["reason_code"] == "context_window_exceeded" 291 + assert token not in json.dumps(events) 292 + assert token not in str(raised.value) 293 + 294 + 295 + def test_run_cogitate_local_window_threaded_once( 296 + fake_openhands, 297 + run_env, 298 + monkeypatch, 299 + ): 300 + endpoint = LocalEndpoint( 301 + base_url="https://spp.example.test", 302 + served_model_id="confidential-model", 303 + credential="confidential-token", 304 + is_bundled=False, 305 + is_confidential=True, 306 + ) 307 + window_calls = [] 308 + llm_calls = [] 309 + agent_calls = [] 310 + original_build_llm = openhands._build_llm 311 + 312 + def resolve_window(resolved_endpoint): 313 + window_calls.append(resolved_endpoint) 314 + return 16384 315 + 316 + def spy_build_llm(*args, **kwargs): 317 + llm_calls.append(kwargs) 318 + return original_build_llm(*args, **kwargs) 319 + 320 + def spy_build_agent(**kwargs): 321 + agent_calls.append(kwargs) 322 + return SimpleNamespace( 323 + llm=kwargs["llm"], 324 + tools=kwargs["tool_specs"], 325 + include_default_tools=kwargs["include_default_tools"], 326 + system_prompt=kwargs["system_prompt"], 327 + condenser=SimpleNamespace(max_tokens=kwargs["condenser_max_tokens"]), 328 + ) 329 + 330 + monkeypatch.setattr( 331 + "solstone.think.providers.local_endpoint.resolve_local_endpoint", 332 + lambda: endpoint, 333 + ) 334 + monkeypatch.setattr( 335 + "solstone.think.providers.local_endpoint.resolve_endpoint_served_window", 336 + resolve_window, 337 + ) 338 + monkeypatch.setattr( 339 + "solstone.think.services.spp_transport.confidential_egress_base_url", 340 + lambda _base_url: "http://127.0.0.1:4567", 341 + ) 342 + monkeypatch.setattr(openhands, "_build_llm", spy_build_llm) 343 + monkeypatch.setattr(openhands, "_build_cogitate_agent", spy_build_agent) 344 + local_env = {**run_env, "provider": "local", "model": LOCAL_MODEL} 345 + 346 + asyncio.run(openhands.run_cogitate(local_env, lambda _event: None)) 347 + 348 + assert window_calls == [endpoint] 349 + assert llm_calls[0]["endpoint"] is endpoint 350 + assert llm_calls[0]["served_window"] == 16384 351 + assert agent_calls[0]["condenser_max_tokens"] == 11264 352 + llm = fake_openhands.LLM.instances[0] 353 + assert llm.max_input_tokens == 16384 354 + assert llm.max_output_tokens == 4096 355 + assert llm.litellm_extra_body == { 356 + "chat_template_kwargs": {"enable_thinking": False} 357 + } 250 358 251 359 252 360 @pytest.mark.parametrize("exc_type", [httpx.ConnectError, httpx.ConnectTimeout])