hotsocket.fyi
This repository has no description
1import OpenAI from "openai"
2import { logMessage } from "../db"
3import { buildCompletionMessages, rememberRecalledMemoryChunks } from "../memory"
4import { recordMetric } from "../metrics"
5import { emit } from "../stream"
6import type { LoopState } from "./types"
7import {
8 API_BASE,
9 ENABLE_THINKING,
10 FALLBACK_BASE,
11 FALLBACK_MODEL,
12 FALLBACK_TOOL_CHOICE,
13 MODEL,
14 PRIMARY_TOOL_CHOICE,
15 SUMMARY_MODEL,
16 TOOLS,
17 USE_FALLBACK,
18 apiErrorDetails,
19 client,
20 errorSummary,
21 estimatePromptTokens,
22 fallbackClient,
23 fallbackContextWindow,
24 findSummaryMessageIndex,
25 isContentFilterError,
26 isPromptTooLargeError,
27 retryDelayMs,
28 sanitizeMessages,
29 scrubImagesFromConversation,
30 shouldFallback,
31 summaryClient,
32 summarizeConversationViaLLM,
33} from "./util"
34import { assistantContentText } from "./loop-content"
35import type { CompletionRequest, CompletionTurnResult, ToolCallAssembly } from "./loop-shared"
36
37/**
38 * Resolves the configured summary client/model pair.
39 *
40 * @returns Active summary provider config.
41 */
42export function configuredSummaryProvider(): { client: OpenAI | null; model: string } {
43 if (summaryClient && SUMMARY_MODEL) return { client: summaryClient, model: SUMMARY_MODEL }
44 return {
45 client: USE_FALLBACK ? fallbackClient : client,
46 model: USE_FALLBACK ? FALLBACK_MODEL : MODEL,
47 }
48}
49
50function logApiError(err: unknown, context: string): void {
51 if (!(err instanceof OpenAI.APIError)) return
52 console.error(`[api] ${err.status} ${err.message} - ${context}`)
53 for (const line of apiErrorDetails(err)) console.error(line)
54}
55
56/**
57 * Appends an assistant message to state and persists it to conversation logs.
58 *
59 * @param convId - Active conversation id.
60 * @param state - Mutable loop state.
61 * @param msg - Assistant message to append.
62 */
63export function addAssistantMessage(convId: number, state: LoopState, msg: OpenAI.Chat.ChatCompletionMessage): void {
64 state.conversation.push(msg)
65 logMessage(convId, msg.role, msg.content ?? "", msg.tool_calls ?? undefined)
66}
67
68function recordPromptResponse(request: CompletionRequest, result: CompletionTurnResult, promptMetricId: number | null): void {
69 recordMetric({
70 type: "prompt_response",
71 promptMetricId: promptMetricId ?? undefined,
72 model: request.model,
73 toolChoice: request.tool_choice,
74 messages: request.messages,
75 response: result.message,
76 usage: result.usage,
77 })
78}
79
80/**
81 * Applies token usage from a completion response to loop state counters.
82 *
83 * @param state - Mutable loop state.
84 * @param usage - Completion usage payload (if provided by the API).
85 */
86export function applyUsage(state: LoopState, usage: OpenAI.Completions.CompletionUsage | undefined): void {
87 if (!usage) return
88 state.tokenCount += usage.total_tokens
89 if (usage.prompt_tokens) state.contextSize = usage.prompt_tokens
90 console.log(`[tokens] +${usage.total_tokens} total=${state.tokenCount}`)
91 recordMetric({ type: "usage", usage })
92}
93
94/**
95 * Emits model reasoning text when exposed by the provider.
96 *
97 * Supports both `reasoning_content` and `<think>...</think>` wrappers.
98 *
99 * @param msg - Assistant message to inspect for reasoning traces.
100 */
101export function emitThinking(msg: OpenAI.Chat.ChatCompletionMessage): void {
102 const rawMsg = msg as unknown as Record<string, unknown>
103 let thinkingText: string | null = null
104
105 if (typeof rawMsg.reasoning_content === "string" && rawMsg.reasoning_content.trim()) {
106 thinkingText = rawMsg.reasoning_content.trim()
107 } else if (typeof msg.content === "string") {
108 const match = msg.content.match(/^<think>([\s\S]*?)<\/think>\s*/i)
109 if (match) {
110 thinkingText = match[1]!.trim()
111 ;(msg as unknown as Record<string, unknown>).content = msg.content.slice(match[0].length)
112 }
113 }
114
115 if (thinkingText) emit({ type: "thinking", text: thinkingText })
116}
117
118function sleep(ms: number): Promise<void> {
119 return new Promise((resolve) => setTimeout(resolve, ms))
120}
121
122function formatRetryAt(retryAfterMs: number): string {
123 const retryAt = new Date(Date.now() + retryAfterMs)
124 const local = retryAt.toLocaleString(undefined, {
125 hour12: false,
126 timeZoneName: "short",
127 })
128 return `${local} (${retryAt.toISOString()})`
129}
130
131function apiErrorSearchText(err: { message: string; error?: unknown }): string {
132 const parts = [err.message]
133 if (err.error !== undefined) {
134 try {
135 parts.push(JSON.stringify(err.error))
136 } catch {
137 parts.push(String(err.error))
138 }
139 }
140 return parts.join("\n")
141}
142
143function shouldRetryWithAutoToolChoice(err: unknown): boolean {
144 if (!(err instanceof OpenAI.APIError)) return false
145 const text = apiErrorSearchText(err)
146 return /no endpoints found that support the provided 'tool_choice' value|does not support this tool_choice/i.test(text)
147}
148
149function shouldRetryWithoutReasoningForTools(err: unknown): boolean {
150 if (!(err instanceof OpenAI.APIError)) return false
151 return /function call should not be used with prefix/i.test(apiErrorSearchText(err))
152}
153
154function toolCompatibleReasoningExtras(
155 request?: Pick<CompletionRequest, "provider" | "chat_template_kwargs">,
156): Partial<CompletionRequest> {
157 return {
158 include_reasoning: false,
159 reasoning: { enabled: false, exclude: true, effort: "none" },
160 provider: { ...request?.provider, require_parameters: true },
161 }
162}
163
164function prefixModeToolCallExtras(request?: Pick<CompletionRequest, "provider" | "chat_template_kwargs">): Partial<CompletionRequest> {
165 return {
166 ...toolCompatibleReasoningExtras(request),
167 enable_thinking: false,
168 chat_template_kwargs: {
169 ...request?.chat_template_kwargs,
170 enable_thinking: false,
171 },
172 }
173}
174
175function disableReasoningForToolCalls(request: CompletionRequest): CompletionRequest {
176 return {
177 ...request,
178 ...prefixModeToolCallExtras(request),
179 }
180}
181
182function configuredThinkingRequestExtras(
183 request?: Pick<CompletionRequest, "chat_template_kwargs">,
184): Partial<CompletionRequest> {
185 if (ENABLE_THINKING) return {}
186 return {
187 include_reasoning: false,
188 reasoning: { enabled: false, exclude: true, effort: "none" },
189 enable_thinking: false,
190 chat_template_kwargs: {
191 ...request?.chat_template_kwargs,
192 enable_thinking: false,
193 },
194 }
195}
196
197function openRouterToolRequestExtras(baseUrl: string): Partial<CompletionRequest> {
198 if (!baseUrl.includes("openrouter.ai")) return {}
199 return toolCompatibleReasoningExtras()
200}
201
202function shouldRetryWithoutStreamUsage(err: unknown): boolean {
203 if (!(err instanceof OpenAI.APIError)) return false
204 if (err.status !== 400) return false
205 return /stream_options|include_usage/i.test(err.message)
206}
207
208function coerceReasoningToolArgument(rawValue: string): unknown {
209 const value = rawValue.trim()
210 if (!value) return ""
211 if (/^true$/i.test(value)) return true
212 if (/^false$/i.test(value)) return false
213 if (/^null$/i.test(value)) return null
214
215 if (/^-?\d+(?:\.\d+)?$/.test(value)) {
216 const parsed = Number(value)
217 if (Number.isFinite(parsed)) return parsed
218 }
219
220 if (
221 (value.startsWith("{") && value.endsWith("}")) ||
222 (value.startsWith("[") && value.endsWith("]")) ||
223 (value.startsWith('"') && value.endsWith('"'))
224 ) {
225 try {
226 return JSON.parse(value)
227 } catch {
228 // keep raw string fallback
229 }
230 }
231
232 return value
233}
234
235function parseReasoningToolCallBlock(rawBlock: string): ToolCallAssembly | null {
236 const functionMatch = rawBlock.match(/<function(?:=|\s+name\s*=\s*["']?)([^>"'\s/]+)["']?\s*>/i)
237 if (!functionMatch || functionMatch.index === undefined) return null
238
239 const functionName = functionMatch[1]?.trim()
240 if (!functionName) return null
241
242 const functionBodyStart = functionMatch.index + functionMatch[0].length
243 const functionBodyEnd = rawBlock.indexOf("</function>", functionBodyStart)
244 if (functionBodyEnd < 0) return null
245
246 const functionBody = rawBlock.slice(functionBodyStart, functionBodyEnd)
247 const args: Record<string, unknown> = {}
248
249 const parameterRegex = /<parameter(?:=|\s+name\s*=\s*["']?)([^>"'\s/]+)["']?\s*>([\s\S]*?)<\/parameter>/gi
250 for (const match of functionBody.matchAll(parameterRegex)) {
251 const key = match[1]?.trim()
252 if (!key) continue
253 args[key] = coerceReasoningToolArgument(match[2] ?? "")
254 }
255
256 return {
257 id: "",
258 type: "function",
259 function: {
260 name: functionName,
261 arguments: JSON.stringify(args),
262 },
263 }
264}
265
266function drainReasoningToolCallBlocks(buffer: string): { blocks: string[]; remainder: string } {
267 const blocks: string[] = []
268 let remaining = buffer
269
270 while (true) {
271 const openMatch = remaining.match(/<tool_call(?:\s[^>]*)?>/i)
272 if (!openMatch || openMatch.index === undefined) {
273 const partialStart = remaining.lastIndexOf("<tool_call")
274 return {
275 blocks,
276 remainder: partialStart >= 0 ? remaining.slice(partialStart) : "",
277 }
278 }
279
280 const openStart = openMatch.index
281 const openEnd = openStart + openMatch[0].length
282 const closeStart = remaining.indexOf("</tool_call>", openEnd)
283 if (closeStart < 0) {
284 return {
285 blocks,
286 remainder: remaining.slice(openStart),
287 }
288 }
289
290 blocks.push(remaining.slice(openEnd, closeStart))
291 remaining = remaining.slice(closeStart + "</tool_call>".length)
292 }
293}
294
295async function consumeCompletionStream(
296 stream: AsyncIterable<OpenAI.Chat.ChatCompletionChunk>,
297): Promise<CompletionTurnResult> {
298 const contentParts: string[] = []
299 const streamedToolCalls = new Map<number, ToolCallAssembly>()
300 const reasoningToolCalls: ToolCallAssembly[] = []
301 let reasoningToolBuffer = ""
302
303 let usage: OpenAI.Completions.CompletionUsage | undefined
304 let emittedText = false
305 let emittedThinking = false
306 const reasoningParts: string[] = []
307
308 for await (const chunk of stream) {
309 if (chunk.usage) usage = chunk.usage
310
311 const choice = chunk.choices[0]
312 if (!choice) continue
313
314 const delta = choice.delta as OpenAI.Chat.ChatCompletionChunk.Choice.Delta & {
315 reasoning_content?: string
316 }
317
318 if (typeof delta.reasoning_content === "string" && delta.reasoning_content.length > 0) {
319 if (ENABLE_THINKING) reasoningParts.push(delta.reasoning_content)
320
321 reasoningToolBuffer += delta.reasoning_content
322 const { blocks, remainder } = drainReasoningToolCallBlocks(reasoningToolBuffer)
323 reasoningToolBuffer = remainder
324
325 for (const block of blocks) {
326 const parsedCall = parseReasoningToolCallBlock(block)
327 if (!parsedCall) continue
328 parsedCall.id = `call_reasoning_${reasoningToolCalls.length}`
329 reasoningToolCalls.push(parsedCall)
330 }
331 }
332
333 if (typeof delta.content === "string" && delta.content.length > 0) {
334 if (ENABLE_THINKING && !emittedThinking && reasoningParts.length > 0) {
335 emit({ type: "thinking", text: reasoningParts.join("") })
336 emittedThinking = true
337 }
338 contentParts.push(delta.content)
339 emit({ type: "text", text: delta.content })
340 emittedText = true
341 }
342
343 if (!Array.isArray(delta.tool_calls)) continue
344
345 for (const partial of delta.tool_calls) {
346 const index = partial.index ?? 0
347 const existing = streamedToolCalls.get(index) ?? {
348 id: partial.id ?? `call_${index}`,
349 type: "function" as const,
350 function: { name: "", arguments: "" },
351 }
352
353 if (partial.id) existing.id = partial.id
354 if (partial.type === "function") existing.type = "function"
355 if (partial.function?.name) existing.function.name += partial.function.name
356 if (partial.function?.arguments) existing.function.arguments += partial.function.arguments
357 streamedToolCalls.set(index, existing)
358 }
359 }
360
361 if (streamedToolCalls.size === 0) {
362 const trailingReasoningCall = parseReasoningToolCallBlock(reasoningToolBuffer)
363 if (trailingReasoningCall) {
364 trailingReasoningCall.id = `call_reasoning_${reasoningToolCalls.length}`
365 reasoningToolCalls.push(trailingReasoningCall)
366 }
367 }
368
369 const finalToolCalls =
370 streamedToolCalls.size > 0
371 ? [...streamedToolCalls.entries()]
372 .sort((a, b) => a[0] - b[0])
373 .map(([, toolCall]) => toolCall)
374 : reasoningToolCalls
375
376 const message: OpenAI.Chat.ChatCompletionMessage = {
377 role: "assistant",
378 content: contentParts.length > 0 ? contentParts.join("") : null,
379 refusal: null,
380 ...(finalToolCalls.length > 0
381 ? {
382 tool_calls: finalToolCalls,
383 }
384 : {}),
385 }
386
387 if (reasoningParts.length > 0) {
388 ;(message as OpenAI.Chat.ChatCompletionMessage & { reasoning_content?: string }).reasoning_content =
389 reasoningParts.join("")
390 }
391
392 return {
393 message,
394 usage,
395 emittedText,
396 emittedThinking,
397 bufferedThinking: reasoningParts.join(""),
398 }
399}
400
401async function createStreamedCompletion(
402 apiClient: OpenAI,
403 request: CompletionRequest,
404): Promise<CompletionTurnResult> {
405 const streamedRequest = {
406 ...request,
407 stream: true,
408 stream_options: { include_usage: true },
409 } as const
410
411 const promptMetricId = recordMetric({ type: "prompt", messages: request.messages })
412
413 try {
414 const stream = await apiClient.chat.completions.create(streamedRequest)
415 const result = await consumeCompletionStream(stream as AsyncIterable<OpenAI.Chat.ChatCompletionChunk>)
416 recordPromptResponse(request, result, promptMetricId)
417 return result
418 } catch (err) {
419 if (shouldRetryWithoutStreamUsage(err)) {
420 const stream = await apiClient.chat.completions.create({
421 ...request,
422 stream: true,
423 } as const)
424 const result = await consumeCompletionStream(stream as AsyncIterable<OpenAI.Chat.ChatCompletionChunk>)
425 recordPromptResponse(request, result, promptMetricId)
426 return result
427 }
428 throw err
429 }
430}
431
432async function createFallbackCompletion(messages: OpenAI.Chat.ChatCompletionMessageParam[]): Promise<CompletionTurnResult> {
433 const request: CompletionRequest = {
434 model: FALLBACK_MODEL,
435 messages,
436 tools: TOOLS,
437 tool_choice: FALLBACK_TOOL_CHOICE,
438 ...openRouterToolRequestExtras(FALLBACK_BASE),
439 ...configuredThinkingRequestExtras(),
440 }
441
442 let currentRequest = request
443 let retriedAutoToolChoice = false
444 let retriedWithoutReasoning = false
445 while (true) {
446 try {
447 return await createStreamedCompletion(fallbackClient, currentRequest)
448 } catch (err) {
449 if (currentRequest.tool_choice !== "auto" && !retriedAutoToolChoice && shouldRetryWithAutoToolChoice(err)) {
450 retriedAutoToolChoice = true
451 console.warn(
452 `[fallback] provider rejected tool_choice=${currentRequest.tool_choice}; retrying with tool_choice=auto`,
453 )
454 currentRequest = {
455 ...currentRequest,
456 tool_choice: "auto",
457 }
458 continue
459 }
460 if (!retriedWithoutReasoning && shouldRetryWithoutReasoningForTools(err)) {
461 retriedWithoutReasoning = true
462 console.warn("[fallback] provider rejected function calling in reasoning/prefix mode; retrying fallback with tool-compatible reasoning disabled")
463 currentRequest = disableReasoningForToolCalls(currentRequest)
464 continue
465 }
466 throw err
467 }
468 }
469}
470
471async function createPrimaryCompletion(messages: OpenAI.Chat.ChatCompletionMessageParam[]): Promise<CompletionTurnResult> {
472 const request: CompletionRequest = {
473 model: MODEL,
474 messages,
475 tools: TOOLS,
476 tool_choice: PRIMARY_TOOL_CHOICE,
477 ...openRouterToolRequestExtras(API_BASE),
478 ...configuredThinkingRequestExtras(),
479 }
480
481 let currentRequest = request
482 let retriedAutoToolChoice = false
483 let retriedWithoutReasoning = false
484 while (true) {
485 try {
486 return await createStreamedCompletion(client!, currentRequest)
487 } catch (err) {
488 if (currentRequest.tool_choice !== "auto" && !retriedAutoToolChoice && shouldRetryWithAutoToolChoice(err)) {
489 retriedAutoToolChoice = true
490 console.warn(`[api] provider rejected tool_choice=${currentRequest.tool_choice}; retrying primary with tool_choice=auto`)
491 currentRequest = {
492 ...currentRequest,
493 tool_choice: "auto",
494 }
495 continue
496 }
497 if (!retriedWithoutReasoning && shouldRetryWithoutReasoningForTools(err)) {
498 retriedWithoutReasoning = true
499 console.warn("[api] provider rejected function calling in reasoning/prefix mode; retrying primary with tool-compatible reasoning disabled")
500 currentRequest = disableReasoningForToolCalls(currentRequest)
501 continue
502 }
503 throw err
504 }
505 }
506}
507
508function logPromptSizeDebug(state: LoopState, err: unknown, label: string): void {
509 const messageCount = state.conversation.length
510 const roleCounts = state.conversation.reduce<Record<string, number>>((acc, m) => {
511 const role = (m as { role?: string }).role ?? "unknown"
512 acc[role] = (acc[role] ?? 0) + 1
513 return acc
514 }, {})
515 const estimate = estimatePromptTokens(state.conversation)
516 const charLength = JSON.stringify(state.conversation).length
517 const summary = err instanceof OpenAI.APIError ? `${err.status} ${err.message}` : errorSummary(err)
518 console.warn(
519 `[api] ${label}: ${summary} - messages=${messageCount} est_tokens=${estimate} chars=${charLength} roles=${JSON.stringify(roleCounts)} observedPromptTokens=${state.contextSize}`,
520 )
521}
522
523async function recoverFromPromptTooLarge(state: LoopState, attempt: number): Promise<boolean> {
524 const beforeCount = state.conversation.length
525 const beforeEstimate = estimatePromptTokens(state.conversation)
526
527 const summaryProvider = configuredSummaryProvider()
528 if (!summaryProvider.client || !summaryProvider.model) {
529 console.warn(`[context] recovery: no summary client available; cannot llm-summarize`)
530 return false
531 }
532
533 console.warn(`[context] recovery: attempting llm summarization via ${summaryProvider.model} (attempt=${attempt + 1})`)
534 const summarized = await summarizeConversationViaLLM(state.conversation, summaryProvider.client, summaryProvider.model)
535 if (!summarized) {
536 console.warn(`[context] recovery: llm summarization returned no changes`)
537 return false
538 }
539
540 const afterEstimate = estimatePromptTokens(summarized)
541 if (afterEstimate >= beforeEstimate) {
542 console.warn(`[context] recovery: llm summary not smaller (${beforeEstimate} -> ${afterEstimate}); keeping original`)
543 return false
544 }
545
546 state.conversation = summarized
547 state.contextSize = afterEstimate
548
549 const summaryIdx = findSummaryMessageIndex(state.conversation)
550 const summary = summaryIdx >= 0 ? (state.conversation[summaryIdx]?.content as string) : undefined
551
552 console.warn(
553 `[context] recovery: llm-summarized conversation (${beforeCount} -> ${summarized.length} msgs, ${beforeEstimate} -> ${afterEstimate} tokens)`,
554 )
555
556 recordMetric({
557 type: "compaction",
558 before: beforeEstimate,
559 after: afterEstimate,
560 method: "force-llm",
561 summary,
562 })
563 return true
564}
565
566/**
567 * Fetches the next assistant completion, including fallback and backoff behavior.
568 *
569 * @param state - Mutable loop state containing current conversation/context.
570 * @param baseConversation - Optional alternate base conversation for retries.
571 * @returns The next chat completion response.
572 * @throws If the primary request fails with a non-fallback error condition.
573 */
574export async function fetchCompletion(
575 state: LoopState,
576 baseConversation: OpenAI.Chat.ChatCompletionMessageParam[] = state.conversation,
577): Promise<CompletionTurnResult> {
578 let promptTooLargeAttempts = 0
579 let contentFilterScrubbed = false
580
581 const recoverFromContentFilter = (err: unknown, label: string): boolean => {
582 if (contentFilterScrubbed) return false
583 if (!isContentFilterError(err)) return false
584 const scrubbed = scrubImagesFromConversation(state.conversation)
585 contentFilterScrubbed = true
586 if (scrubbed > 0) {
587 console.warn(
588 `[api] ${label} content-filter rejection; scrubbed ${scrubbed} image attachment(s) from conversation and retrying`,
589 )
590 return true
591 }
592 console.warn(`[api] ${label} content-filter rejection but no images found to scrub`)
593 return false
594 }
595
596 while (true) {
597 if (baseConversation === state.conversation) {
598 state.conversation = sanitizeMessages(state.conversation)
599 baseConversation = state.conversation
600 } else {
601 baseConversation = sanitizeMessages(baseConversation)
602 }
603
604 const requestContext = await buildCompletionMessages(
605 baseConversation,
606 state.memoryRecallCooldowns,
607 state.memoryRecallTurn,
608 )
609 const requestMessages = requestContext.messages
610
611 if (USE_FALLBACK) {
612 const fallbackWindow = fallbackContextWindow(requestMessages)
613 if (fallbackWindow.nearLimit) {
614 console.warn(
615 `[fallback] prompt estimate ${fallbackWindow.estimate} nearing fallback limit ${fallbackWindow.softLimit} (${FALLBACK_MODEL})`,
616 )
617 }
618
619 try {
620 const completion = await createFallbackCompletion(requestMessages)
621 state.memoryRecallCooldowns = rememberRecalledMemoryChunks(
622 state.memoryRecallCooldowns,
623 requestContext.recalledChunkIds,
624 state.memoryRecallTurn,
625 )
626 return completion
627 } catch (fallbackErr) {
628 if (isPromptTooLargeError(fallbackErr) && promptTooLargeAttempts < 2) {
629 logPromptSizeDebug(state, fallbackErr, `fallback rejected prompt (attempt ${promptTooLargeAttempts + 1}/2)`)
630 const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts)
631 promptTooLargeAttempts++
632 if (recovered) continue
633 }
634 if (recoverFromContentFilter(fallbackErr, "fallback")) continue
635 if (shouldFallback(fallbackErr)) {
636 const retryAfter = retryDelayMs(fallbackErr)
637 console.warn(
638 `[fallback] transient failure (${errorSummary(fallbackErr)}); retrying after ${Math.ceil(retryAfter / 1000)}s`,
639 )
640 console.log(
641 `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying fallback...`,
642 )
643 await sleep(retryAfter)
644 continue
645 }
646 logApiError(fallbackErr, `model=${FALLBACK_MODEL} api=${FALLBACK_BASE}`)
647 throw fallbackErr
648 }
649 }
650
651 try {
652 const completion = await createPrimaryCompletion(requestMessages)
653 state.memoryRecallCooldowns = rememberRecalledMemoryChunks(
654 state.memoryRecallCooldowns,
655 requestContext.recalledChunkIds,
656 state.memoryRecallTurn,
657 )
658 return completion
659 } catch (primaryErr) {
660 if (isPromptTooLargeError(primaryErr) && promptTooLargeAttempts < 2) {
661 logPromptSizeDebug(state, primaryErr, `primary rejected prompt (attempt ${promptTooLargeAttempts + 1}/2)`)
662 const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts)
663 promptTooLargeAttempts++
664 if (recovered) continue
665 logApiError(primaryErr, `model=${MODEL} api=${API_BASE}`)
666 throw primaryErr
667 }
668
669 if (recoverFromContentFilter(primaryErr, "primary")) continue
670
671 if (!shouldFallback(primaryErr)) {
672 logApiError(primaryErr, `model=${MODEL} api=${API_BASE}`)
673 throw primaryErr
674 }
675
676 const fallbackWindow = fallbackContextWindow(requestMessages)
677 if (fallbackWindow.skip) {
678 console.warn(
679 `[api] primary down (${errorSummary(primaryErr)}) and fallback context estimate ${fallbackWindow.estimate} exceeds hard limit ${fallbackWindow.hardLimit}; retrying primary after backoff`,
680 )
681 const retryAfter = retryDelayMs(primaryErr)
682 console.log(
683 `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying primary...`,
684 )
685 await sleep(retryAfter)
686 continue
687 }
688
689 if (fallbackWindow.nearLimit) {
690 console.warn(
691 `[fallback] prompt estimate ${fallbackWindow.estimate} nearing fallback limit ${fallbackWindow.softLimit} (${FALLBACK_MODEL})`,
692 )
693 }
694
695 console.warn(`[api] primary down (${errorSummary(primaryErr)}) - switching to fallback`)
696 try {
697 const completion = await createFallbackCompletion(requestMessages)
698 state.memoryRecallCooldowns = rememberRecalledMemoryChunks(
699 state.memoryRecallCooldowns,
700 requestContext.recalledChunkIds,
701 state.memoryRecallTurn,
702 )
703 return completion
704 } catch (fallbackErr) {
705 if (isPromptTooLargeError(fallbackErr) && promptTooLargeAttempts < 2) {
706 logPromptSizeDebug(state, fallbackErr, `fallback rejected prompt during failover (attempt ${promptTooLargeAttempts + 1}/2)`)
707 const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts)
708 promptTooLargeAttempts++
709 if (recovered) continue
710 }
711 if (recoverFromContentFilter(fallbackErr, "fallback-failover")) continue
712 console.warn(
713 `[api] fallback failed (${errorSummary(fallbackErr)}) after primary failure (${errorSummary(primaryErr)}); retrying primary after backoff`,
714 )
715 const retryAfter = retryDelayMs(primaryErr)
716 console.log(
717 `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying primary...`,
718 )
719 await sleep(retryAfter)
720 }
721 }
722 }
723}
724
725export const __completionTest = {
726 consumeCompletionStream,
727}