hotsocket.fyi
This repository has no description
1import OpenAI from "openai"
2import { logMessage } from "../db"
3import { buildCompletionMessages, rememberRecalledMemoryChunks } from "../memory"
4import { recordMetric } from "../metrics"
5import { emit } from "../stream"
6import type { LoopState } from "./types"
7import {
8 API_BASE,
9 ENABLE_THINKING,
10 FALLBACK_BASE,
11 FALLBACK_MODEL,
12 FALLBACK_TOOL_CHOICE,
13 MODEL,
14 PRIMARY_TOOL_CHOICE,
15 SUMMARY_MODEL,
16 TOOLS,
17 USE_FALLBACK,
18 apiErrorDetails,
19 client,
20 errorSummary,
21 estimatePromptTokens,
22 fallbackClient,
23 fallbackContextWindow,
24 findSummaryMessageIndex,
25 isContentFilterError,
26 isImageParseError,
27 isPromptTooLargeError,
28 loadAgentSummaryContext,
29 retryDelayMs,
30 sanitizeMessages,
31 scrubImagesFromConversation,
32 shouldFallback,
33 summaryClient,
34 summarizeConversationViaLLM,
35} from "./util"
36import { assistantContentText } from "./loop-content"
37import type { CompletionRequest, CompletionTurnResult, ToolCallAssembly } from "./loop-shared"
38
39/**
40 * Resolves the configured summary client/model pair.
41 *
42 * @returns Active summary provider config.
43 */
44export function configuredSummaryProvider(): { client: OpenAI | null; model: string } {
45 if (summaryClient && SUMMARY_MODEL) return { client: summaryClient, model: SUMMARY_MODEL }
46 return {
47 client: USE_FALLBACK ? fallbackClient : client,
48 model: USE_FALLBACK ? FALLBACK_MODEL : MODEL,
49 }
50}
51
52function logApiError(err: unknown, context: string): void {
53 if (!(err instanceof OpenAI.APIError)) return
54 console.error(`[api] ${err.status} ${err.message} - ${context}`)
55 for (const line of apiErrorDetails(err)) console.error(line)
56}
57
58/**
59 * Appends an assistant message to state and persists it to conversation logs.
60 *
61 * @param convId - Active conversation id.
62 * @param state - Mutable loop state.
63 * @param msg - Assistant message to append.
64 */
65export function addAssistantMessage(convId: number, state: LoopState, msg: OpenAI.Chat.ChatCompletionMessage): void {
66 state.conversation.push(msg)
67 logMessage(convId, msg.role, msg.content ?? "", msg.tool_calls ?? undefined)
68}
69
70function recordPromptResponse(request: CompletionRequest, result: CompletionTurnResult, promptMetricId: number | null): void {
71 recordMetric({
72 type: "prompt_response",
73 promptMetricId: promptMetricId ?? undefined,
74 model: request.model,
75 toolChoice: request.tool_choice,
76 messages: request.messages,
77 response: result.message,
78 usage: result.usage,
79 })
80}
81
82/**
83 * Applies token usage from a completion response to loop state counters.
84 *
85 * @param state Mutable loop state.
86 * @param usage Completion usage payload (if provided by the API).
87 * @param timing Runner-measured stream timing for throughput display.
88 */
89export function applyUsage(
90 state: LoopState,
91 usage: OpenAI.Completions.CompletionUsage | undefined,
92 timing: Pick<CompletionTurnResult, "elapsedMs" | "tokensPerSecond"> = {},
93): void {
94 if (!usage) return
95 state.tokenCount += usage.total_tokens
96 if (usage.prompt_tokens) state.contextSize = usage.prompt_tokens
97 const rate = typeof timing.tokensPerSecond === "number" ? ` ${timing.tokensPerSecond.toFixed(1)} tok/s` : ""
98 console.log(`[tokens] +${usage.total_tokens} total=${state.tokenCount}${rate}`)
99 emit({
100 type: "usage",
101 promptTokens: usage.prompt_tokens,
102 completionTokens: usage.completion_tokens,
103 totalTokens: usage.total_tokens,
104 elapsedMs: timing.elapsedMs,
105 tokensPerSecond: timing.tokensPerSecond,
106 })
107 recordMetric({ type: "usage", usage })
108}
109
110/**
111 * Emits model reasoning text when exposed by the provider.
112 *
113 * Supports both `reasoning_content` and `<think>...</think>` wrappers.
114 *
115 * @param msg - Assistant message to inspect for reasoning traces.
116 */
117export function emitThinking(msg: OpenAI.Chat.ChatCompletionMessage): void {
118 const rawMsg = msg as unknown as Record<string, unknown>
119 let thinkingText: string | null = null
120
121 if (typeof rawMsg.reasoning_content === "string" && rawMsg.reasoning_content.trim()) {
122 thinkingText = rawMsg.reasoning_content.trim()
123 } else if (typeof msg.content === "string") {
124 const match = msg.content.match(/^<think>([\s\S]*?)<\/think>\s*/i)
125 if (match) {
126 thinkingText = match[1]!.trim()
127 ;(msg as unknown as Record<string, unknown>).content = msg.content.slice(match[0].length)
128 }
129 }
130
131 if (thinkingText) emit({ type: "thinking", text: thinkingText })
132}
133
134function sleep(ms: number): Promise<void> {
135 return new Promise((resolve) => setTimeout(resolve, ms))
136}
137
138function formatRetryAt(retryAfterMs: number): string {
139 const retryAt = new Date(Date.now() + retryAfterMs)
140 const local = retryAt.toLocaleString(undefined, {
141 hour12: false,
142 timeZoneName: "short",
143 })
144 return `${local} (${retryAt.toISOString()})`
145}
146
147function apiErrorSearchText(err: { message: string; error?: unknown }): string {
148 const parts = [err.message]
149 if (err.error !== undefined) {
150 try {
151 parts.push(JSON.stringify(err.error))
152 } catch {
153 parts.push(String(err.error))
154 }
155 }
156 return parts.join("\n")
157}
158
159function shouldRetryWithAutoToolChoice(err: unknown): boolean {
160 if (!(err instanceof OpenAI.APIError)) return false
161 const text = apiErrorSearchText(err)
162 return /no endpoints found that support the provided 'tool_choice' value|does not support this tool_choice/i.test(text)
163}
164
165function shouldRetryWithoutReasoningForTools(err: unknown): boolean {
166 if (!(err instanceof OpenAI.APIError)) return false
167 return /function call should not be used with prefix/i.test(apiErrorSearchText(err))
168}
169
170function shouldRetryWithReasoningEnabled(err: unknown): boolean {
171 if (!(err instanceof OpenAI.APIError)) return false
172 return /reasoning is mandatory|reasoning cannot be disabled|reasoning is required/i.test(apiErrorSearchText(err))
173}
174
175function enableReasoningForRequest(request: CompletionRequest): CompletionRequest {
176 const next: CompletionRequest = {
177 ...request,
178 include_reasoning: true,
179 reasoning: { enabled: true, effort: "low" },
180 }
181 delete next.enable_thinking
182 if (next.chat_template_kwargs) {
183 const { enable_thinking: _ignored, ...rest } = next.chat_template_kwargs
184 next.chat_template_kwargs = Object.keys(rest).length > 0 ? rest : undefined
185 if (!next.chat_template_kwargs) delete next.chat_template_kwargs
186 }
187 return next
188}
189
190function toolCompatibleReasoningExtras(
191 request?: Pick<CompletionRequest, "provider" | "chat_template_kwargs">,
192): Partial<CompletionRequest> {
193 return {
194 include_reasoning: false,
195 reasoning: { enabled: false, exclude: true, effort: "none" },
196 provider: { ...request?.provider, require_parameters: true },
197 }
198}
199
200function prefixModeToolCallExtras(request?: Pick<CompletionRequest, "provider" | "chat_template_kwargs">): Partial<CompletionRequest> {
201 return {
202 ...toolCompatibleReasoningExtras(request),
203 enable_thinking: false,
204 chat_template_kwargs: {
205 ...request?.chat_template_kwargs,
206 enable_thinking: false,
207 },
208 }
209}
210
211function disableReasoningForToolCalls(request: CompletionRequest): CompletionRequest {
212 return {
213 ...request,
214 ...prefixModeToolCallExtras(request),
215 }
216}
217
218function configuredThinkingRequestExtras(
219 request?: Pick<CompletionRequest, "chat_template_kwargs">,
220): Partial<CompletionRequest> {
221 if (ENABLE_THINKING) return {}
222 return {
223 include_reasoning: false,
224 reasoning: { enabled: false, exclude: true, effort: "none" },
225 enable_thinking: false,
226 chat_template_kwargs: {
227 ...request?.chat_template_kwargs,
228 enable_thinking: false,
229 },
230 }
231}
232
233function openRouterToolRequestExtras(baseUrl: string): Partial<CompletionRequest> {
234 if (!baseUrl.includes("openrouter.ai")) return {}
235 return toolCompatibleReasoningExtras()
236}
237
238function shouldRetryWithoutStreamUsage(err: unknown): boolean {
239 if (!(err instanceof OpenAI.APIError)) return false
240 if (err.status !== 400) return false
241 return /stream_options|include_usage/i.test(err.message)
242}
243
244function coerceReasoningToolArgument(rawValue: string): unknown {
245 const value = rawValue.trim()
246 if (!value) return ""
247 if (/^true$/i.test(value)) return true
248 if (/^false$/i.test(value)) return false
249 if (/^null$/i.test(value)) return null
250
251 if (/^-?\d+(?:\.\d+)?$/.test(value)) {
252 const parsed = Number(value)
253 if (Number.isFinite(parsed)) return parsed
254 }
255
256 if (
257 (value.startsWith("{") && value.endsWith("}")) ||
258 (value.startsWith("[") && value.endsWith("]")) ||
259 (value.startsWith('"') && value.endsWith('"'))
260 ) {
261 try {
262 return JSON.parse(value)
263 } catch {
264 // keep raw string fallback
265 }
266 }
267
268 return value
269}
270
271function parseReasoningToolCallBlock(rawBlock: string): ToolCallAssembly | null {
272 const functionMatch = rawBlock.match(/<function(?:=|\s+name\s*=\s*["']?)([^>"'\s/]+)["']?\s*>/i)
273 if (!functionMatch || functionMatch.index === undefined) return null
274
275 const functionName = functionMatch[1]?.trim()
276 if (!functionName) return null
277
278 const functionBodyStart = functionMatch.index + functionMatch[0].length
279 const functionBodyEnd = rawBlock.indexOf("</function>", functionBodyStart)
280 if (functionBodyEnd < 0) return null
281
282 const functionBody = rawBlock.slice(functionBodyStart, functionBodyEnd)
283 const args: Record<string, unknown> = {}
284
285 const parameterRegex = /<parameter(?:=|\s+name\s*=\s*["']?)([^>"'\s/]+)["']?\s*>([\s\S]*?)<\/parameter>/gi
286 for (const match of functionBody.matchAll(parameterRegex)) {
287 const key = match[1]?.trim()
288 if (!key) continue
289 args[key] = coerceReasoningToolArgument(match[2] ?? "")
290 }
291
292 return {
293 id: "",
294 type: "function",
295 function: {
296 name: functionName,
297 arguments: JSON.stringify(args),
298 },
299 }
300}
301
302function drainReasoningToolCallBlocks(buffer: string): { blocks: string[]; remainder: string } {
303 const blocks: string[] = []
304 let remaining = buffer
305
306 while (true) {
307 const openMatch = remaining.match(/<tool_call(?:\s[^>]*)?>/i)
308 if (!openMatch || openMatch.index === undefined) {
309 const partialStart = remaining.lastIndexOf("<tool_call")
310 return {
311 blocks,
312 remainder: partialStart >= 0 ? remaining.slice(partialStart) : "",
313 }
314 }
315
316 const openStart = openMatch.index
317 const openEnd = openStart + openMatch[0].length
318 const closeStart = remaining.indexOf("</tool_call>", openEnd)
319 if (closeStart < 0) {
320 return {
321 blocks,
322 remainder: remaining.slice(openStart),
323 }
324 }
325
326 blocks.push(remaining.slice(openEnd, closeStart))
327 remaining = remaining.slice(closeStart + "</tool_call>".length)
328 }
329}
330
331async function consumeCompletionStream(
332 stream: AsyncIterable<OpenAI.Chat.ChatCompletionChunk>,
333): Promise<CompletionTurnResult> {
334 const startedAt = Date.now()
335 const contentParts: string[] = []
336 const streamedToolCalls = new Map<number, ToolCallAssembly>()
337 const reasoningToolCalls: ToolCallAssembly[] = []
338 let reasoningToolBuffer = ""
339
340 let usage: OpenAI.Completions.CompletionUsage | undefined
341 let emittedText = false
342 let emittedThinking = false
343 const reasoningParts: string[] = []
344
345 for await (const chunk of stream) {
346 if (chunk.usage) usage = chunk.usage
347
348 const choice = chunk.choices[0]
349 if (!choice) continue
350
351 const delta = choice.delta as OpenAI.Chat.ChatCompletionChunk.Choice.Delta & {
352 reasoning_content?: string
353 }
354
355 if (typeof delta.reasoning_content === "string" && delta.reasoning_content.length > 0) {
356 if (ENABLE_THINKING) reasoningParts.push(delta.reasoning_content)
357
358 reasoningToolBuffer += delta.reasoning_content
359 const { blocks, remainder } = drainReasoningToolCallBlocks(reasoningToolBuffer)
360 reasoningToolBuffer = remainder
361
362 for (const block of blocks) {
363 const parsedCall = parseReasoningToolCallBlock(block)
364 if (!parsedCall) continue
365 parsedCall.id = `call_reasoning_${reasoningToolCalls.length}`
366 reasoningToolCalls.push(parsedCall)
367 }
368 }
369
370 if (typeof delta.content === "string" && delta.content.length > 0) {
371 if (ENABLE_THINKING && !emittedThinking && reasoningParts.length > 0) {
372 emit({ type: "thinking", text: reasoningParts.join("") })
373 emittedThinking = true
374 }
375 contentParts.push(delta.content)
376 emit({ type: "text", text: delta.content })
377 emittedText = true
378 }
379
380 if (!Array.isArray(delta.tool_calls)) continue
381
382 for (const partial of delta.tool_calls) {
383 const index = partial.index ?? 0
384 const existing = streamedToolCalls.get(index) ?? {
385 id: partial.id ?? `call_${index}`,
386 type: "function" as const,
387 function: { name: "", arguments: "" },
388 }
389
390 if (partial.id) existing.id = partial.id
391 if (partial.type === "function") existing.type = "function"
392 if (partial.function?.name) existing.function.name += partial.function.name
393 if (partial.function?.arguments) existing.function.arguments += partial.function.arguments
394 streamedToolCalls.set(index, existing)
395 }
396 }
397
398 if (streamedToolCalls.size === 0) {
399 const trailingReasoningCall = parseReasoningToolCallBlock(reasoningToolBuffer)
400 if (trailingReasoningCall) {
401 trailingReasoningCall.id = `call_reasoning_${reasoningToolCalls.length}`
402 reasoningToolCalls.push(trailingReasoningCall)
403 }
404 }
405
406 const finalToolCalls =
407 streamedToolCalls.size > 0
408 ? [...streamedToolCalls.entries()]
409 .sort((a, b) => a[0] - b[0])
410 .map(([, toolCall]) => toolCall)
411 : reasoningToolCalls
412
413 const message: OpenAI.Chat.ChatCompletionMessage = {
414 role: "assistant",
415 content: contentParts.length > 0 ? contentParts.join("") : null,
416 refusal: null,
417 ...(finalToolCalls.length > 0
418 ? {
419 tool_calls: finalToolCalls,
420 }
421 : {}),
422 }
423
424 if (reasoningParts.length > 0) {
425 ;(message as OpenAI.Chat.ChatCompletionMessage & { reasoning_content?: string }).reasoning_content =
426 reasoningParts.join("")
427 }
428
429 const elapsedMs = Math.max(0, Date.now() - startedAt)
430 const tokensPerSecond =
431 usage && elapsedMs > 0 ? usage.completion_tokens / (elapsedMs / 1000) : undefined
432
433 return {
434 message,
435 usage,
436 emittedText,
437 emittedThinking,
438 bufferedThinking: reasoningParts.join(""),
439 elapsedMs,
440 tokensPerSecond,
441 }
442}
443
444async function createStreamedCompletion(
445 apiClient: OpenAI,
446 request: CompletionRequest,
447): Promise<CompletionTurnResult> {
448 const streamedRequest = {
449 ...request,
450 stream: true,
451 stream_options: { include_usage: true },
452 } as const
453
454 const promptMetricId = recordMetric({ type: "prompt", messages: request.messages })
455
456 try {
457 const stream = await apiClient.chat.completions.create(streamedRequest)
458 const result = await consumeCompletionStream(stream as AsyncIterable<OpenAI.Chat.ChatCompletionChunk>)
459 recordPromptResponse(request, result, promptMetricId)
460 return result
461 } catch (err) {
462 if (shouldRetryWithoutStreamUsage(err)) {
463 const stream = await apiClient.chat.completions.create({
464 ...request,
465 stream: true,
466 } as const)
467 const result = await consumeCompletionStream(stream as AsyncIterable<OpenAI.Chat.ChatCompletionChunk>)
468 recordPromptResponse(request, result, promptMetricId)
469 return result
470 }
471 throw err
472 }
473}
474
475async function createFallbackCompletion(messages: OpenAI.Chat.ChatCompletionMessageParam[]): Promise<CompletionTurnResult> {
476 const request: CompletionRequest = {
477 model: FALLBACK_MODEL,
478 messages,
479 tools: TOOLS,
480 tool_choice: FALLBACK_TOOL_CHOICE,
481 ...openRouterToolRequestExtras(FALLBACK_BASE),
482 ...configuredThinkingRequestExtras(),
483 }
484
485 let currentRequest = request
486 let retriedAutoToolChoice = false
487 let retriedWithoutReasoning = false
488 let retriedWithReasoning = false
489 while (true) {
490 try {
491 return await createStreamedCompletion(fallbackClient, currentRequest)
492 } catch (err) {
493 if (currentRequest.tool_choice !== "auto" && !retriedAutoToolChoice && shouldRetryWithAutoToolChoice(err)) {
494 retriedAutoToolChoice = true
495 console.warn(
496 `[fallback] provider rejected tool_choice=${currentRequest.tool_choice}; retrying with tool_choice=auto`,
497 )
498 currentRequest = {
499 ...currentRequest,
500 tool_choice: "auto",
501 }
502 continue
503 }
504 if (!retriedWithoutReasoning && shouldRetryWithoutReasoningForTools(err)) {
505 retriedWithoutReasoning = true
506 console.warn("[fallback] provider rejected function calling in reasoning/prefix mode; retrying fallback with tool-compatible reasoning disabled")
507 currentRequest = disableReasoningForToolCalls(currentRequest)
508 continue
509 }
510 if (!retriedWithReasoning && shouldRetryWithReasoningEnabled(err)) {
511 retriedWithReasoning = true
512 console.warn(`[fallback] model ${currentRequest.model} requires reasoning; retrying fallback with reasoning enabled`)
513 currentRequest = enableReasoningForRequest(currentRequest)
514 continue
515 }
516 throw err
517 }
518 }
519}
520
521async function createPrimaryCompletion(messages: OpenAI.Chat.ChatCompletionMessageParam[]): Promise<CompletionTurnResult> {
522 const request: CompletionRequest = {
523 model: MODEL,
524 messages,
525 tools: TOOLS,
526 tool_choice: PRIMARY_TOOL_CHOICE,
527 ...openRouterToolRequestExtras(API_BASE),
528 ...configuredThinkingRequestExtras(),
529 }
530
531 let currentRequest = request
532 let retriedAutoToolChoice = false
533 let retriedWithoutReasoning = false
534 let retriedWithReasoning = false
535 while (true) {
536 try {
537 return await createStreamedCompletion(client!, currentRequest)
538 } catch (err) {
539 if (currentRequest.tool_choice !== "auto" && !retriedAutoToolChoice && shouldRetryWithAutoToolChoice(err)) {
540 retriedAutoToolChoice = true
541 console.warn(`[api] provider rejected tool_choice=${currentRequest.tool_choice}; retrying primary with tool_choice=auto`)
542 currentRequest = {
543 ...currentRequest,
544 tool_choice: "auto",
545 }
546 continue
547 }
548 if (!retriedWithoutReasoning && shouldRetryWithoutReasoningForTools(err)) {
549 retriedWithoutReasoning = true
550 console.warn("[api] provider rejected function calling in reasoning/prefix mode; retrying primary with tool-compatible reasoning disabled")
551 currentRequest = disableReasoningForToolCalls(currentRequest)
552 continue
553 }
554 if (!retriedWithReasoning && shouldRetryWithReasoningEnabled(err)) {
555 retriedWithReasoning = true
556 console.warn(`[api] model ${currentRequest.model} requires reasoning; retrying primary with reasoning enabled`)
557 currentRequest = enableReasoningForRequest(currentRequest)
558 continue
559 }
560 throw err
561 }
562 }
563}
564
565function logPromptSizeDebug(state: LoopState, err: unknown, label: string): void {
566 const messageCount = state.conversation.length
567 const roleCounts = state.conversation.reduce<Record<string, number>>((acc, m) => {
568 const role = (m as { role?: string }).role ?? "unknown"
569 acc[role] = (acc[role] ?? 0) + 1
570 return acc
571 }, {})
572 const estimate = estimatePromptTokens(state.conversation)
573 const charLength = JSON.stringify(state.conversation).length
574 const summary = err instanceof OpenAI.APIError ? `${err.status} ${err.message}` : errorSummary(err)
575 console.warn(
576 `[api] ${label}: ${summary} - messages=${messageCount} est_tokens=${estimate} chars=${charLength} roles=${JSON.stringify(roleCounts)} observedPromptTokens=${state.contextSize}`,
577 )
578}
579
580async function recoverFromPromptTooLarge(state: LoopState, attempt: number): Promise<boolean> {
581 const beforeCount = state.conversation.length
582 const beforeEstimate = estimatePromptTokens(state.conversation)
583
584 const summaryProvider = configuredSummaryProvider()
585 if (!summaryProvider.client || !summaryProvider.model) {
586 console.warn(`[context] recovery: no summary client available; cannot llm-summarize`)
587 return false
588 }
589
590 console.warn(`[context] recovery: attempting llm summarization via ${summaryProvider.model} (attempt=${attempt + 1})`)
591 const agentContext = await loadAgentSummaryContext()
592 const summarized = await summarizeConversationViaLLM(state.conversation, summaryProvider.client, summaryProvider.model, {
593 agentContext,
594 })
595 if (!summarized) {
596 console.warn(`[context] recovery: llm summarization returned no changes`)
597 return false
598 }
599
600 const afterEstimate = estimatePromptTokens(summarized)
601 if (afterEstimate >= beforeEstimate) {
602 console.warn(`[context] recovery: llm summary not smaller (${beforeEstimate} -> ${afterEstimate}); keeping original`)
603 return false
604 }
605
606 state.conversation = summarized
607 state.contextSize = afterEstimate
608
609 const summaryIdx = findSummaryMessageIndex(state.conversation)
610 const summary = summaryIdx >= 0 ? (state.conversation[summaryIdx]?.content as string) : undefined
611
612 console.warn(
613 `[context] recovery: llm-summarized conversation (${beforeCount} -> ${summarized.length} msgs, ${beforeEstimate} -> ${afterEstimate} tokens)`,
614 )
615
616 recordMetric({
617 type: "compaction",
618 before: beforeEstimate,
619 after: afterEstimate,
620 method: "force-llm",
621 summary,
622 })
623 return true
624}
625
626/**
627 * Fetches the next assistant completion, including fallback and backoff behavior.
628 *
629 * @param state - Mutable loop state containing current conversation/context.
630 * @param baseConversation - Optional alternate base conversation for retries.
631 * @returns The next chat completion response.
632 * @throws If the primary request fails with a non-fallback error condition.
633 */
634export async function fetchCompletion(
635 state: LoopState,
636 baseConversation: OpenAI.Chat.ChatCompletionMessageParam[] = state.conversation,
637): Promise<CompletionTurnResult> {
638 let promptTooLargeAttempts = 0
639 let imagesScrubbed = false
640
641 // Both content-filter rejections and image parse/format rejections (e.g.
642 // z.ai/GLM code 1210) stick across turns when the offending image lives in
643 // the persisted conversation, which crash-loops the runner on restart.
644 // Scrub the image parts (replacing them with a placeholder the model sees)
645 // and retry so the loop survives instead of aborting.
646 const recoverByScrubbingImages = (err: unknown, label: string): boolean => {
647 if (imagesScrubbed) return false
648 if (!isContentFilterError(err) && !isImageParseError(err)) return false
649 const reason = isContentFilterError(err) ? "content-filter rejection" : "image parse/format rejection"
650 const scrubbed = scrubImagesFromConversation(state.conversation)
651 imagesScrubbed = true
652 if (scrubbed > 0) {
653 console.warn(
654 `[api] ${label} ${reason}; scrubbed ${scrubbed} image attachment(s) from conversation and retrying`,
655 )
656 return true
657 }
658 console.warn(`[api] ${label} ${reason} but no images found to scrub`)
659 return false
660 }
661
662 while (true) {
663 if (baseConversation === state.conversation) {
664 state.conversation = sanitizeMessages(state.conversation)
665 baseConversation = state.conversation
666 } else {
667 baseConversation = sanitizeMessages(baseConversation)
668 }
669
670 // Only run memory recall on the first step of a new turn. Re-running it on
671 // every agentic iteration floods context with the same recalled chunks for
672 // the same (unchanged) user message.
673 const requestContext = state.memoryRecallPending
674 ? await buildCompletionMessages(
675 baseConversation,
676 state.memoryRecallCooldowns,
677 state.memoryRecallTurn,
678 )
679 : { messages: baseConversation, recalledChunkIds: [] as number[] }
680 const requestMessages = requestContext.messages
681
682 if (USE_FALLBACK) {
683 const fallbackWindow = fallbackContextWindow(requestMessages)
684 if (fallbackWindow.nearLimit) {
685 console.warn(
686 `[fallback] prompt estimate ${fallbackWindow.estimate} nearing fallback limit ${fallbackWindow.softLimit} (${FALLBACK_MODEL})`,
687 )
688 }
689
690 try {
691 const completion = await createFallbackCompletion(requestMessages)
692 state.memoryRecallCooldowns = rememberRecalledMemoryChunks(
693 state.memoryRecallCooldowns,
694 requestContext.recalledChunkIds,
695 state.memoryRecallTurn,
696 )
697 return completion
698 } catch (fallbackErr) {
699 if (isPromptTooLargeError(fallbackErr) && promptTooLargeAttempts < 2) {
700 logPromptSizeDebug(state, fallbackErr, `fallback rejected prompt (attempt ${promptTooLargeAttempts + 1}/2)`)
701 const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts)
702 promptTooLargeAttempts++
703 if (recovered) continue
704 }
705 if (recoverByScrubbingImages(fallbackErr, "fallback")) continue
706 if (shouldFallback(fallbackErr)) {
707 const retryAfter = retryDelayMs(fallbackErr)
708 console.warn(
709 `[fallback] transient failure (${errorSummary(fallbackErr)}); retrying after ${Math.ceil(retryAfter / 1000)}s`,
710 )
711 console.log(
712 `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying fallback...`,
713 )
714 await sleep(retryAfter)
715 continue
716 }
717 logApiError(fallbackErr, `model=${FALLBACK_MODEL} api=${FALLBACK_BASE}`)
718 throw fallbackErr
719 }
720 }
721
722 try {
723 const completion = await createPrimaryCompletion(requestMessages)
724 state.memoryRecallCooldowns = rememberRecalledMemoryChunks(
725 state.memoryRecallCooldowns,
726 requestContext.recalledChunkIds,
727 state.memoryRecallTurn,
728 )
729 return completion
730 } catch (primaryErr) {
731 if (isPromptTooLargeError(primaryErr) && promptTooLargeAttempts < 2) {
732 logPromptSizeDebug(state, primaryErr, `primary rejected prompt (attempt ${promptTooLargeAttempts + 1}/2)`)
733 const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts)
734 promptTooLargeAttempts++
735 if (recovered) continue
736 logApiError(primaryErr, `model=${MODEL} api=${API_BASE}`)
737 throw primaryErr
738 }
739
740 if (recoverByScrubbingImages(primaryErr, "primary")) continue
741
742 if (!shouldFallback(primaryErr)) {
743 logApiError(primaryErr, `model=${MODEL} api=${API_BASE}`)
744 throw primaryErr
745 }
746
747 const fallbackWindow = fallbackContextWindow(requestMessages)
748 if (fallbackWindow.skip) {
749 console.warn(
750 `[api] primary down (${errorSummary(primaryErr)}) and fallback context estimate ${fallbackWindow.estimate} exceeds hard limit ${fallbackWindow.hardLimit}; retrying primary after backoff`,
751 )
752 const retryAfter = retryDelayMs(primaryErr)
753 console.log(
754 `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying primary...`,
755 )
756 await sleep(retryAfter)
757 continue
758 }
759
760 if (fallbackWindow.nearLimit) {
761 console.warn(
762 `[fallback] prompt estimate ${fallbackWindow.estimate} nearing fallback limit ${fallbackWindow.softLimit} (${FALLBACK_MODEL})`,
763 )
764 }
765
766 console.warn(`[api] primary down (${errorSummary(primaryErr)}) - switching to fallback`)
767 try {
768 const completion = await createFallbackCompletion(requestMessages)
769 state.memoryRecallCooldowns = rememberRecalledMemoryChunks(
770 state.memoryRecallCooldowns,
771 requestContext.recalledChunkIds,
772 state.memoryRecallTurn,
773 )
774 return completion
775 } catch (fallbackErr) {
776 if (isPromptTooLargeError(fallbackErr) && promptTooLargeAttempts < 2) {
777 logPromptSizeDebug(state, fallbackErr, `fallback rejected prompt during failover (attempt ${promptTooLargeAttempts + 1}/2)`)
778 const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts)
779 promptTooLargeAttempts++
780 if (recovered) continue
781 }
782 if (recoverByScrubbingImages(fallbackErr, "fallback-failover")) continue
783 console.warn(
784 `[api] fallback failed (${errorSummary(fallbackErr)}) after primary failure (${errorSummary(primaryErr)}); retrying primary after backoff`,
785 )
786 const retryAfter = retryDelayMs(primaryErr)
787 console.log(
788 `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying primary...`,
789 )
790 await sleep(retryAfter)
791 }
792 }
793 }
794}
795
796export const __completionTest = {
797 consumeCompletionStream,
798}