hotsocket.fyi
This repository has no description
1import OpenAI from "openai"
2import { logMessage } from "../db"
3import { buildCompletionMessages, rememberRecalledMemoryChunks } from "../memory"
4import { recordMetric } from "../metrics"
5import { emit } from "../stream"
6import type { LoopState } from "./types"
7import {
8 API_BASE,
9 ENABLE_THINKING,
10 FALLBACK_BASE,
11 FALLBACK_MODEL,
12 FALLBACK_TOOL_CHOICE,
13 MODEL,
14 PRIMARY_TOOL_CHOICE,
15 SUMMARY_MODEL,
16 TOOLS,
17 USE_FALLBACK,
18 apiErrorDetails,
19 client,
20 errorSummary,
21 estimatePromptTokens,
22 fallbackClient,
23 fallbackContextWindow,
24 findSummaryMessageIndex,
25 isContentFilterError,
26 isImageParseError,
27 isPromptTooLargeError,
28 retryDelayMs,
29 sanitizeMessages,
30 scrubImagesFromConversation,
31 shouldFallback,
32 summaryClient,
33 summarizeConversationViaLLM,
34} from "./util"
35import { assistantContentText } from "./loop-content"
36import type { CompletionRequest, CompletionTurnResult, ToolCallAssembly } from "./loop-shared"
37
38/**
39 * Resolves the configured summary client/model pair.
40 *
41 * @returns Active summary provider config.
42 */
43export function configuredSummaryProvider(): { client: OpenAI | null; model: string } {
44 if (summaryClient && SUMMARY_MODEL) return { client: summaryClient, model: SUMMARY_MODEL }
45 return {
46 client: USE_FALLBACK ? fallbackClient : client,
47 model: USE_FALLBACK ? FALLBACK_MODEL : MODEL,
48 }
49}
50
51function logApiError(err: unknown, context: string): void {
52 if (!(err instanceof OpenAI.APIError)) return
53 console.error(`[api] ${err.status} ${err.message} - ${context}`)
54 for (const line of apiErrorDetails(err)) console.error(line)
55}
56
57/**
58 * Appends an assistant message to state and persists it to conversation logs.
59 *
60 * @param convId - Active conversation id.
61 * @param state - Mutable loop state.
62 * @param msg - Assistant message to append.
63 */
64export function addAssistantMessage(convId: number, state: LoopState, msg: OpenAI.Chat.ChatCompletionMessage): void {
65 state.conversation.push(msg)
66 logMessage(convId, msg.role, msg.content ?? "", msg.tool_calls ?? undefined)
67}
68
69function recordPromptResponse(request: CompletionRequest, result: CompletionTurnResult, promptMetricId: number | null): void {
70 recordMetric({
71 type: "prompt_response",
72 promptMetricId: promptMetricId ?? undefined,
73 model: request.model,
74 toolChoice: request.tool_choice,
75 messages: request.messages,
76 response: result.message,
77 usage: result.usage,
78 })
79}
80
81/**
82 * Applies token usage from a completion response to loop state counters.
83 *
84 * @param state - Mutable loop state.
85 * @param usage - Completion usage payload (if provided by the API).
86 */
87export function applyUsage(state: LoopState, usage: OpenAI.Completions.CompletionUsage | undefined): void {
88 if (!usage) return
89 state.tokenCount += usage.total_tokens
90 if (usage.prompt_tokens) state.contextSize = usage.prompt_tokens
91 console.log(`[tokens] +${usage.total_tokens} total=${state.tokenCount}`)
92 recordMetric({ type: "usage", usage })
93}
94
95/**
96 * Emits model reasoning text when exposed by the provider.
97 *
98 * Supports both `reasoning_content` and `<think>...</think>` wrappers.
99 *
100 * @param msg - Assistant message to inspect for reasoning traces.
101 */
102export function emitThinking(msg: OpenAI.Chat.ChatCompletionMessage): void {
103 const rawMsg = msg as unknown as Record<string, unknown>
104 let thinkingText: string | null = null
105
106 if (typeof rawMsg.reasoning_content === "string" && rawMsg.reasoning_content.trim()) {
107 thinkingText = rawMsg.reasoning_content.trim()
108 } else if (typeof msg.content === "string") {
109 const match = msg.content.match(/^<think>([\s\S]*?)<\/think>\s*/i)
110 if (match) {
111 thinkingText = match[1]!.trim()
112 ;(msg as unknown as Record<string, unknown>).content = msg.content.slice(match[0].length)
113 }
114 }
115
116 if (thinkingText) emit({ type: "thinking", text: thinkingText })
117}
118
119function sleep(ms: number): Promise<void> {
120 return new Promise((resolve) => setTimeout(resolve, ms))
121}
122
123function formatRetryAt(retryAfterMs: number): string {
124 const retryAt = new Date(Date.now() + retryAfterMs)
125 const local = retryAt.toLocaleString(undefined, {
126 hour12: false,
127 timeZoneName: "short",
128 })
129 return `${local} (${retryAt.toISOString()})`
130}
131
132function apiErrorSearchText(err: { message: string; error?: unknown }): string {
133 const parts = [err.message]
134 if (err.error !== undefined) {
135 try {
136 parts.push(JSON.stringify(err.error))
137 } catch {
138 parts.push(String(err.error))
139 }
140 }
141 return parts.join("\n")
142}
143
144function shouldRetryWithAutoToolChoice(err: unknown): boolean {
145 if (!(err instanceof OpenAI.APIError)) return false
146 const text = apiErrorSearchText(err)
147 return /no endpoints found that support the provided 'tool_choice' value|does not support this tool_choice/i.test(text)
148}
149
150function shouldRetryWithoutReasoningForTools(err: unknown): boolean {
151 if (!(err instanceof OpenAI.APIError)) return false
152 return /function call should not be used with prefix/i.test(apiErrorSearchText(err))
153}
154
155function shouldRetryWithReasoningEnabled(err: unknown): boolean {
156 if (!(err instanceof OpenAI.APIError)) return false
157 return /reasoning is mandatory|reasoning cannot be disabled|reasoning is required/i.test(apiErrorSearchText(err))
158}
159
160function enableReasoningForRequest(request: CompletionRequest): CompletionRequest {
161 const next: CompletionRequest = {
162 ...request,
163 include_reasoning: true,
164 reasoning: { enabled: true, effort: "low" },
165 }
166 delete next.enable_thinking
167 if (next.chat_template_kwargs) {
168 const { enable_thinking: _ignored, ...rest } = next.chat_template_kwargs
169 next.chat_template_kwargs = Object.keys(rest).length > 0 ? rest : undefined
170 if (!next.chat_template_kwargs) delete next.chat_template_kwargs
171 }
172 return next
173}
174
175function toolCompatibleReasoningExtras(
176 request?: Pick<CompletionRequest, "provider" | "chat_template_kwargs">,
177): Partial<CompletionRequest> {
178 return {
179 include_reasoning: false,
180 reasoning: { enabled: false, exclude: true, effort: "none" },
181 provider: { ...request?.provider, require_parameters: true },
182 }
183}
184
185function prefixModeToolCallExtras(request?: Pick<CompletionRequest, "provider" | "chat_template_kwargs">): Partial<CompletionRequest> {
186 return {
187 ...toolCompatibleReasoningExtras(request),
188 enable_thinking: false,
189 chat_template_kwargs: {
190 ...request?.chat_template_kwargs,
191 enable_thinking: false,
192 },
193 }
194}
195
196function disableReasoningForToolCalls(request: CompletionRequest): CompletionRequest {
197 return {
198 ...request,
199 ...prefixModeToolCallExtras(request),
200 }
201}
202
203function configuredThinkingRequestExtras(
204 request?: Pick<CompletionRequest, "chat_template_kwargs">,
205): Partial<CompletionRequest> {
206 if (ENABLE_THINKING) return {}
207 return {
208 include_reasoning: false,
209 reasoning: { enabled: false, exclude: true, effort: "none" },
210 enable_thinking: false,
211 chat_template_kwargs: {
212 ...request?.chat_template_kwargs,
213 enable_thinking: false,
214 },
215 }
216}
217
218function openRouterToolRequestExtras(baseUrl: string): Partial<CompletionRequest> {
219 if (!baseUrl.includes("openrouter.ai")) return {}
220 return toolCompatibleReasoningExtras()
221}
222
223function shouldRetryWithoutStreamUsage(err: unknown): boolean {
224 if (!(err instanceof OpenAI.APIError)) return false
225 if (err.status !== 400) return false
226 return /stream_options|include_usage/i.test(err.message)
227}
228
229function coerceReasoningToolArgument(rawValue: string): unknown {
230 const value = rawValue.trim()
231 if (!value) return ""
232 if (/^true$/i.test(value)) return true
233 if (/^false$/i.test(value)) return false
234 if (/^null$/i.test(value)) return null
235
236 if (/^-?\d+(?:\.\d+)?$/.test(value)) {
237 const parsed = Number(value)
238 if (Number.isFinite(parsed)) return parsed
239 }
240
241 if (
242 (value.startsWith("{") && value.endsWith("}")) ||
243 (value.startsWith("[") && value.endsWith("]")) ||
244 (value.startsWith('"') && value.endsWith('"'))
245 ) {
246 try {
247 return JSON.parse(value)
248 } catch {
249 // keep raw string fallback
250 }
251 }
252
253 return value
254}
255
256function parseReasoningToolCallBlock(rawBlock: string): ToolCallAssembly | null {
257 const functionMatch = rawBlock.match(/<function(?:=|\s+name\s*=\s*["']?)([^>"'\s/]+)["']?\s*>/i)
258 if (!functionMatch || functionMatch.index === undefined) return null
259
260 const functionName = functionMatch[1]?.trim()
261 if (!functionName) return null
262
263 const functionBodyStart = functionMatch.index + functionMatch[0].length
264 const functionBodyEnd = rawBlock.indexOf("</function>", functionBodyStart)
265 if (functionBodyEnd < 0) return null
266
267 const functionBody = rawBlock.slice(functionBodyStart, functionBodyEnd)
268 const args: Record<string, unknown> = {}
269
270 const parameterRegex = /<parameter(?:=|\s+name\s*=\s*["']?)([^>"'\s/]+)["']?\s*>([\s\S]*?)<\/parameter>/gi
271 for (const match of functionBody.matchAll(parameterRegex)) {
272 const key = match[1]?.trim()
273 if (!key) continue
274 args[key] = coerceReasoningToolArgument(match[2] ?? "")
275 }
276
277 return {
278 id: "",
279 type: "function",
280 function: {
281 name: functionName,
282 arguments: JSON.stringify(args),
283 },
284 }
285}
286
287function drainReasoningToolCallBlocks(buffer: string): { blocks: string[]; remainder: string } {
288 const blocks: string[] = []
289 let remaining = buffer
290
291 while (true) {
292 const openMatch = remaining.match(/<tool_call(?:\s[^>]*)?>/i)
293 if (!openMatch || openMatch.index === undefined) {
294 const partialStart = remaining.lastIndexOf("<tool_call")
295 return {
296 blocks,
297 remainder: partialStart >= 0 ? remaining.slice(partialStart) : "",
298 }
299 }
300
301 const openStart = openMatch.index
302 const openEnd = openStart + openMatch[0].length
303 const closeStart = remaining.indexOf("</tool_call>", openEnd)
304 if (closeStart < 0) {
305 return {
306 blocks,
307 remainder: remaining.slice(openStart),
308 }
309 }
310
311 blocks.push(remaining.slice(openEnd, closeStart))
312 remaining = remaining.slice(closeStart + "</tool_call>".length)
313 }
314}
315
316async function consumeCompletionStream(
317 stream: AsyncIterable<OpenAI.Chat.ChatCompletionChunk>,
318): Promise<CompletionTurnResult> {
319 const contentParts: string[] = []
320 const streamedToolCalls = new Map<number, ToolCallAssembly>()
321 const reasoningToolCalls: ToolCallAssembly[] = []
322 let reasoningToolBuffer = ""
323
324 let usage: OpenAI.Completions.CompletionUsage | undefined
325 let emittedText = false
326 let emittedThinking = false
327 const reasoningParts: string[] = []
328
329 for await (const chunk of stream) {
330 if (chunk.usage) usage = chunk.usage
331
332 const choice = chunk.choices[0]
333 if (!choice) continue
334
335 const delta = choice.delta as OpenAI.Chat.ChatCompletionChunk.Choice.Delta & {
336 reasoning_content?: string
337 }
338
339 if (typeof delta.reasoning_content === "string" && delta.reasoning_content.length > 0) {
340 if (ENABLE_THINKING) reasoningParts.push(delta.reasoning_content)
341
342 reasoningToolBuffer += delta.reasoning_content
343 const { blocks, remainder } = drainReasoningToolCallBlocks(reasoningToolBuffer)
344 reasoningToolBuffer = remainder
345
346 for (const block of blocks) {
347 const parsedCall = parseReasoningToolCallBlock(block)
348 if (!parsedCall) continue
349 parsedCall.id = `call_reasoning_${reasoningToolCalls.length}`
350 reasoningToolCalls.push(parsedCall)
351 }
352 }
353
354 if (typeof delta.content === "string" && delta.content.length > 0) {
355 if (ENABLE_THINKING && !emittedThinking && reasoningParts.length > 0) {
356 emit({ type: "thinking", text: reasoningParts.join("") })
357 emittedThinking = true
358 }
359 contentParts.push(delta.content)
360 emit({ type: "text", text: delta.content })
361 emittedText = true
362 }
363
364 if (!Array.isArray(delta.tool_calls)) continue
365
366 for (const partial of delta.tool_calls) {
367 const index = partial.index ?? 0
368 const existing = streamedToolCalls.get(index) ?? {
369 id: partial.id ?? `call_${index}`,
370 type: "function" as const,
371 function: { name: "", arguments: "" },
372 }
373
374 if (partial.id) existing.id = partial.id
375 if (partial.type === "function") existing.type = "function"
376 if (partial.function?.name) existing.function.name += partial.function.name
377 if (partial.function?.arguments) existing.function.arguments += partial.function.arguments
378 streamedToolCalls.set(index, existing)
379 }
380 }
381
382 if (streamedToolCalls.size === 0) {
383 const trailingReasoningCall = parseReasoningToolCallBlock(reasoningToolBuffer)
384 if (trailingReasoningCall) {
385 trailingReasoningCall.id = `call_reasoning_${reasoningToolCalls.length}`
386 reasoningToolCalls.push(trailingReasoningCall)
387 }
388 }
389
390 const finalToolCalls =
391 streamedToolCalls.size > 0
392 ? [...streamedToolCalls.entries()]
393 .sort((a, b) => a[0] - b[0])
394 .map(([, toolCall]) => toolCall)
395 : reasoningToolCalls
396
397 const message: OpenAI.Chat.ChatCompletionMessage = {
398 role: "assistant",
399 content: contentParts.length > 0 ? contentParts.join("") : null,
400 refusal: null,
401 ...(finalToolCalls.length > 0
402 ? {
403 tool_calls: finalToolCalls,
404 }
405 : {}),
406 }
407
408 if (reasoningParts.length > 0) {
409 ;(message as OpenAI.Chat.ChatCompletionMessage & { reasoning_content?: string }).reasoning_content =
410 reasoningParts.join("")
411 }
412
413 return {
414 message,
415 usage,
416 emittedText,
417 emittedThinking,
418 bufferedThinking: reasoningParts.join(""),
419 }
420}
421
422async function createStreamedCompletion(
423 apiClient: OpenAI,
424 request: CompletionRequest,
425): Promise<CompletionTurnResult> {
426 const streamedRequest = {
427 ...request,
428 stream: true,
429 stream_options: { include_usage: true },
430 } as const
431
432 const promptMetricId = recordMetric({ type: "prompt", messages: request.messages })
433
434 try {
435 const stream = await apiClient.chat.completions.create(streamedRequest)
436 const result = await consumeCompletionStream(stream as AsyncIterable<OpenAI.Chat.ChatCompletionChunk>)
437 recordPromptResponse(request, result, promptMetricId)
438 return result
439 } catch (err) {
440 if (shouldRetryWithoutStreamUsage(err)) {
441 const stream = await apiClient.chat.completions.create({
442 ...request,
443 stream: true,
444 } as const)
445 const result = await consumeCompletionStream(stream as AsyncIterable<OpenAI.Chat.ChatCompletionChunk>)
446 recordPromptResponse(request, result, promptMetricId)
447 return result
448 }
449 throw err
450 }
451}
452
453async function createFallbackCompletion(messages: OpenAI.Chat.ChatCompletionMessageParam[]): Promise<CompletionTurnResult> {
454 const request: CompletionRequest = {
455 model: FALLBACK_MODEL,
456 messages,
457 tools: TOOLS,
458 tool_choice: FALLBACK_TOOL_CHOICE,
459 ...openRouterToolRequestExtras(FALLBACK_BASE),
460 ...configuredThinkingRequestExtras(),
461 }
462
463 let currentRequest = request
464 let retriedAutoToolChoice = false
465 let retriedWithoutReasoning = false
466 let retriedWithReasoning = false
467 while (true) {
468 try {
469 return await createStreamedCompletion(fallbackClient, currentRequest)
470 } catch (err) {
471 if (currentRequest.tool_choice !== "auto" && !retriedAutoToolChoice && shouldRetryWithAutoToolChoice(err)) {
472 retriedAutoToolChoice = true
473 console.warn(
474 `[fallback] provider rejected tool_choice=${currentRequest.tool_choice}; retrying with tool_choice=auto`,
475 )
476 currentRequest = {
477 ...currentRequest,
478 tool_choice: "auto",
479 }
480 continue
481 }
482 if (!retriedWithoutReasoning && shouldRetryWithoutReasoningForTools(err)) {
483 retriedWithoutReasoning = true
484 console.warn("[fallback] provider rejected function calling in reasoning/prefix mode; retrying fallback with tool-compatible reasoning disabled")
485 currentRequest = disableReasoningForToolCalls(currentRequest)
486 continue
487 }
488 if (!retriedWithReasoning && shouldRetryWithReasoningEnabled(err)) {
489 retriedWithReasoning = true
490 console.warn(`[fallback] model ${currentRequest.model} requires reasoning; retrying fallback with reasoning enabled`)
491 currentRequest = enableReasoningForRequest(currentRequest)
492 continue
493 }
494 throw err
495 }
496 }
497}
498
499async function createPrimaryCompletion(messages: OpenAI.Chat.ChatCompletionMessageParam[]): Promise<CompletionTurnResult> {
500 const request: CompletionRequest = {
501 model: MODEL,
502 messages,
503 tools: TOOLS,
504 tool_choice: PRIMARY_TOOL_CHOICE,
505 ...openRouterToolRequestExtras(API_BASE),
506 ...configuredThinkingRequestExtras(),
507 }
508
509 let currentRequest = request
510 let retriedAutoToolChoice = false
511 let retriedWithoutReasoning = false
512 let retriedWithReasoning = false
513 while (true) {
514 try {
515 return await createStreamedCompletion(client!, currentRequest)
516 } catch (err) {
517 if (currentRequest.tool_choice !== "auto" && !retriedAutoToolChoice && shouldRetryWithAutoToolChoice(err)) {
518 retriedAutoToolChoice = true
519 console.warn(`[api] provider rejected tool_choice=${currentRequest.tool_choice}; retrying primary with tool_choice=auto`)
520 currentRequest = {
521 ...currentRequest,
522 tool_choice: "auto",
523 }
524 continue
525 }
526 if (!retriedWithoutReasoning && shouldRetryWithoutReasoningForTools(err)) {
527 retriedWithoutReasoning = true
528 console.warn("[api] provider rejected function calling in reasoning/prefix mode; retrying primary with tool-compatible reasoning disabled")
529 currentRequest = disableReasoningForToolCalls(currentRequest)
530 continue
531 }
532 if (!retriedWithReasoning && shouldRetryWithReasoningEnabled(err)) {
533 retriedWithReasoning = true
534 console.warn(`[api] model ${currentRequest.model} requires reasoning; retrying primary with reasoning enabled`)
535 currentRequest = enableReasoningForRequest(currentRequest)
536 continue
537 }
538 throw err
539 }
540 }
541}
542
543function logPromptSizeDebug(state: LoopState, err: unknown, label: string): void {
544 const messageCount = state.conversation.length
545 const roleCounts = state.conversation.reduce<Record<string, number>>((acc, m) => {
546 const role = (m as { role?: string }).role ?? "unknown"
547 acc[role] = (acc[role] ?? 0) + 1
548 return acc
549 }, {})
550 const estimate = estimatePromptTokens(state.conversation)
551 const charLength = JSON.stringify(state.conversation).length
552 const summary = err instanceof OpenAI.APIError ? `${err.status} ${err.message}` : errorSummary(err)
553 console.warn(
554 `[api] ${label}: ${summary} - messages=${messageCount} est_tokens=${estimate} chars=${charLength} roles=${JSON.stringify(roleCounts)} observedPromptTokens=${state.contextSize}`,
555 )
556}
557
558async function recoverFromPromptTooLarge(state: LoopState, attempt: number): Promise<boolean> {
559 const beforeCount = state.conversation.length
560 const beforeEstimate = estimatePromptTokens(state.conversation)
561
562 const summaryProvider = configuredSummaryProvider()
563 if (!summaryProvider.client || !summaryProvider.model) {
564 console.warn(`[context] recovery: no summary client available; cannot llm-summarize`)
565 return false
566 }
567
568 console.warn(`[context] recovery: attempting llm summarization via ${summaryProvider.model} (attempt=${attempt + 1})`)
569 const summarized = await summarizeConversationViaLLM(state.conversation, summaryProvider.client, summaryProvider.model)
570 if (!summarized) {
571 console.warn(`[context] recovery: llm summarization returned no changes`)
572 return false
573 }
574
575 const afterEstimate = estimatePromptTokens(summarized)
576 if (afterEstimate >= beforeEstimate) {
577 console.warn(`[context] recovery: llm summary not smaller (${beforeEstimate} -> ${afterEstimate}); keeping original`)
578 return false
579 }
580
581 state.conversation = summarized
582 state.contextSize = afterEstimate
583
584 const summaryIdx = findSummaryMessageIndex(state.conversation)
585 const summary = summaryIdx >= 0 ? (state.conversation[summaryIdx]?.content as string) : undefined
586
587 console.warn(
588 `[context] recovery: llm-summarized conversation (${beforeCount} -> ${summarized.length} msgs, ${beforeEstimate} -> ${afterEstimate} tokens)`,
589 )
590
591 recordMetric({
592 type: "compaction",
593 before: beforeEstimate,
594 after: afterEstimate,
595 method: "force-llm",
596 summary,
597 })
598 return true
599}
600
601/**
602 * Fetches the next assistant completion, including fallback and backoff behavior.
603 *
604 * @param state - Mutable loop state containing current conversation/context.
605 * @param baseConversation - Optional alternate base conversation for retries.
606 * @returns The next chat completion response.
607 * @throws If the primary request fails with a non-fallback error condition.
608 */
609export async function fetchCompletion(
610 state: LoopState,
611 baseConversation: OpenAI.Chat.ChatCompletionMessageParam[] = state.conversation,
612): Promise<CompletionTurnResult> {
613 let promptTooLargeAttempts = 0
614 let imagesScrubbed = false
615
616 // Both content-filter rejections and image parse/format rejections (e.g.
617 // z.ai/GLM code 1210) stick across turns when the offending image lives in
618 // the persisted conversation, which crash-loops the runner on restart.
619 // Scrub the image parts (replacing them with a placeholder the model sees)
620 // and retry so the loop survives instead of aborting.
621 const recoverByScrubbingImages = (err: unknown, label: string): boolean => {
622 if (imagesScrubbed) return false
623 if (!isContentFilterError(err) && !isImageParseError(err)) return false
624 const reason = isContentFilterError(err) ? "content-filter rejection" : "image parse/format rejection"
625 const scrubbed = scrubImagesFromConversation(state.conversation)
626 imagesScrubbed = true
627 if (scrubbed > 0) {
628 console.warn(
629 `[api] ${label} ${reason}; scrubbed ${scrubbed} image attachment(s) from conversation and retrying`,
630 )
631 return true
632 }
633 console.warn(`[api] ${label} ${reason} but no images found to scrub`)
634 return false
635 }
636
637 while (true) {
638 if (baseConversation === state.conversation) {
639 state.conversation = sanitizeMessages(state.conversation)
640 baseConversation = state.conversation
641 } else {
642 baseConversation = sanitizeMessages(baseConversation)
643 }
644
645 const requestContext = await buildCompletionMessages(
646 baseConversation,
647 state.memoryRecallCooldowns,
648 state.memoryRecallTurn,
649 )
650 const requestMessages = requestContext.messages
651
652 if (USE_FALLBACK) {
653 const fallbackWindow = fallbackContextWindow(requestMessages)
654 if (fallbackWindow.nearLimit) {
655 console.warn(
656 `[fallback] prompt estimate ${fallbackWindow.estimate} nearing fallback limit ${fallbackWindow.softLimit} (${FALLBACK_MODEL})`,
657 )
658 }
659
660 try {
661 const completion = await createFallbackCompletion(requestMessages)
662 state.memoryRecallCooldowns = rememberRecalledMemoryChunks(
663 state.memoryRecallCooldowns,
664 requestContext.recalledChunkIds,
665 state.memoryRecallTurn,
666 )
667 return completion
668 } catch (fallbackErr) {
669 if (isPromptTooLargeError(fallbackErr) && promptTooLargeAttempts < 2) {
670 logPromptSizeDebug(state, fallbackErr, `fallback rejected prompt (attempt ${promptTooLargeAttempts + 1}/2)`)
671 const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts)
672 promptTooLargeAttempts++
673 if (recovered) continue
674 }
675 if (recoverByScrubbingImages(fallbackErr, "fallback")) continue
676 if (shouldFallback(fallbackErr)) {
677 const retryAfter = retryDelayMs(fallbackErr)
678 console.warn(
679 `[fallback] transient failure (${errorSummary(fallbackErr)}); retrying after ${Math.ceil(retryAfter / 1000)}s`,
680 )
681 console.log(
682 `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying fallback...`,
683 )
684 await sleep(retryAfter)
685 continue
686 }
687 logApiError(fallbackErr, `model=${FALLBACK_MODEL} api=${FALLBACK_BASE}`)
688 throw fallbackErr
689 }
690 }
691
692 try {
693 const completion = await createPrimaryCompletion(requestMessages)
694 state.memoryRecallCooldowns = rememberRecalledMemoryChunks(
695 state.memoryRecallCooldowns,
696 requestContext.recalledChunkIds,
697 state.memoryRecallTurn,
698 )
699 return completion
700 } catch (primaryErr) {
701 if (isPromptTooLargeError(primaryErr) && promptTooLargeAttempts < 2) {
702 logPromptSizeDebug(state, primaryErr, `primary rejected prompt (attempt ${promptTooLargeAttempts + 1}/2)`)
703 const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts)
704 promptTooLargeAttempts++
705 if (recovered) continue
706 logApiError(primaryErr, `model=${MODEL} api=${API_BASE}`)
707 throw primaryErr
708 }
709
710 if (recoverByScrubbingImages(primaryErr, "primary")) continue
711
712 if (!shouldFallback(primaryErr)) {
713 logApiError(primaryErr, `model=${MODEL} api=${API_BASE}`)
714 throw primaryErr
715 }
716
717 const fallbackWindow = fallbackContextWindow(requestMessages)
718 if (fallbackWindow.skip) {
719 console.warn(
720 `[api] primary down (${errorSummary(primaryErr)}) and fallback context estimate ${fallbackWindow.estimate} exceeds hard limit ${fallbackWindow.hardLimit}; retrying primary after backoff`,
721 )
722 const retryAfter = retryDelayMs(primaryErr)
723 console.log(
724 `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying primary...`,
725 )
726 await sleep(retryAfter)
727 continue
728 }
729
730 if (fallbackWindow.nearLimit) {
731 console.warn(
732 `[fallback] prompt estimate ${fallbackWindow.estimate} nearing fallback limit ${fallbackWindow.softLimit} (${FALLBACK_MODEL})`,
733 )
734 }
735
736 console.warn(`[api] primary down (${errorSummary(primaryErr)}) - switching to fallback`)
737 try {
738 const completion = await createFallbackCompletion(requestMessages)
739 state.memoryRecallCooldowns = rememberRecalledMemoryChunks(
740 state.memoryRecallCooldowns,
741 requestContext.recalledChunkIds,
742 state.memoryRecallTurn,
743 )
744 return completion
745 } catch (fallbackErr) {
746 if (isPromptTooLargeError(fallbackErr) && promptTooLargeAttempts < 2) {
747 logPromptSizeDebug(state, fallbackErr, `fallback rejected prompt during failover (attempt ${promptTooLargeAttempts + 1}/2)`)
748 const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts)
749 promptTooLargeAttempts++
750 if (recovered) continue
751 }
752 if (recoverByScrubbingImages(fallbackErr, "fallback-failover")) continue
753 console.warn(
754 `[api] fallback failed (${errorSummary(fallbackErr)}) after primary failure (${errorSummary(primaryErr)}); retrying primary after backoff`,
755 )
756 const retryAfter = retryDelayMs(primaryErr)
757 console.log(
758 `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying primary...`,
759 )
760 await sleep(retryAfter)
761 }
762 }
763 }
764}
765
766export const __completionTest = {
767 consumeCompletionStream,
768}