hotsocket.fyi / niri
This repository has no description
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

fallover on all errors

author
nekomimi.pet date (May 31, 2026, 11:01 PM -0400) commit d95d87cf parent a9a84619
+388 -20
+3 -1
.gitignore
··· 9 9 10 10 # runtime state 11 11 session.json 12 + primary-failover.json 13 + rest-snapshot.json 12 14 13 15 # agent home — ignore everything except the example file 14 16 home/* 15 17 !home/soul.example.md 16 18 17 - .codex 19 + .codex
+1 -1
src/container/config.ts
··· 27 27 return path.posix.normalize(root) 28 28 })() 29 29 30 - const DEFAULT_IMAGE_MAX_BYTES = 150_000 30 + const DEFAULT_IMAGE_MAX_BYTES = 1_000_000 31 31 /** Maximum image size (bytes) accepted by `image_tool`. */ 32 32 export const IMAGE_MAX_BYTES = (() => { 33 33 const parsed = parseInt(process.env.IMAGE_TOOL_MAX_BYTES ?? `${DEFAULT_IMAGE_MAX_BYTES}`, 10)
+56
src/container/tools.test.ts
··· 1 + import test from "node:test" 2 + import assert from "node:assert/strict" 3 + import fs from "node:fs/promises" 4 + import path from "node:path" 5 + import { IMAGE_MAX_BYTES, IMAGE_ROOT, USE_DOCKER_SHELL } from "./config" 6 + import { readImageForModel } from "./tools" 7 + 8 + const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) 9 + 10 + function pngLikeBytes(size: number): Buffer { 11 + const data = Buffer.alloc(size) 12 + PNG_SIGNATURE.copy(data) 13 + return data 14 + } 15 + 16 + test("default image tool limit is 1MB", (t) => { 17 + if (process.env.IMAGE_TOOL_MAX_BYTES) { 18 + t.skip("IMAGE_TOOL_MAX_BYTES overrides the default") 19 + return 20 + } 21 + 22 + assert.equal(IMAGE_MAX_BYTES, 1_000_000) 23 + }) 24 + 25 + test("readImageForModel accepts files up to IMAGE_MAX_BYTES and rejects larger files", async (t) => { 26 + if (USE_DOCKER_SHELL) { 27 + t.skip("local filesystem image limit test does not run through Docker shell") 28 + return 29 + } 30 + if (IMAGE_MAX_BYTES > 5_000_000) { 31 + t.skip("configured image limit is too large for a focused unit test") 32 + return 33 + } 34 + 35 + await fs.mkdir(IMAGE_ROOT, { recursive: true }) 36 + const dir = await fs.mkdtemp(path.join(IMAGE_ROOT, "image-limit-")) 37 + 38 + try { 39 + const acceptedPath = path.join(dir, "accepted.png") 40 + await fs.writeFile(acceptedPath, pngLikeBytes(IMAGE_MAX_BYTES)) 41 + 42 + const accepted = await readImageForModel(acceptedPath, 5_000) 43 + assert.equal(accepted.bytes, IMAGE_MAX_BYTES) 44 + assert.equal(accepted.mime, "image/png") 45 + 46 + const oversizedPath = path.join(dir, "oversized.png") 47 + await fs.writeFile(oversizedPath, pngLikeBytes(IMAGE_MAX_BYTES + 1)) 48 + 49 + await assert.rejects( 50 + readImageForModel(oversizedPath, 5_000), 51 + new RegExp(`file too large: ${IMAGE_MAX_BYTES + 1} bytes \\(max ${IMAGE_MAX_BYTES}\\)`), 52 + ) 53 + } finally { 54 + await fs.rm(dir, { recursive: true, force: true }) 55 + } 56 + })
+107 -13
src/runner/loop-completion.ts
··· 10 10 FALLBACK_BASE, 11 11 FALLBACK_MODEL, 12 12 FALLBACK_TOOL_CHOICE, 13 + ANTHROPIC_BASE_URL, 13 14 ANTHROPIC_MODEL, 14 15 MODEL, 15 16 PRIMARY_TOOL_CHOICE, ··· 19 20 USE_FALLBACK, 20 21 apiErrorDetails, 21 22 client, 23 + clearPrimaryFailover, 22 24 errorSummary, 23 25 estimatePromptTokens, 24 26 fallbackClient, ··· 27 29 isContentFilterError, 28 30 isImageParseError, 29 31 isPromptTooLargeError, 32 + isQuotaExhaustedError, 30 33 loadAgentSummaryContext, 34 + primaryFailoverStatus, 35 + recordPrimaryQuotaFailover, 31 36 retryDelayMs, 32 37 sanitizeMessages, 33 38 scrubImagesFromConversation, 34 39 shouldFallback, 40 + shouldRetryProvider, 35 41 summaryClient, 36 42 summarizeConversationViaLLM, 37 43 } from "./util" ··· 44 50 * 45 51 * @returns Active summary provider config. 46 52 */ 47 - export function configuredSummaryProvider(): { client: OpenAI | null; model: string } { 53 + export async function configuredSummaryProvider(): Promise<{ client: OpenAI | null; model: string }> { 48 54 if (summaryClient && SUMMARY_MODEL) return { client: summaryClient, model: SUMMARY_MODEL } 55 + const failover = USE_FALLBACK ? null : await primaryFailoverStatus() 56 + if (failover?.active) return { client: fallbackClient, model: FALLBACK_MODEL } 49 57 return { 50 58 client: USE_FALLBACK ? fallbackClient : client, 51 59 model: USE_FALLBACK ? FALLBACK_MODEL : MODEL, ··· 56 64 if (!(err instanceof OpenAI.APIError)) return 57 65 console.error(`[api] ${err.status} ${err.message} - ${context}`) 58 66 for (const line of apiErrorDetails(err)) console.error(line) 67 + } 68 + 69 + function primaryApiContext(): string { 70 + const model = USE_ANTHROPIC ? ANTHROPIC_MODEL : MODEL 71 + const api = USE_ANTHROPIC ? ANTHROPIC_BASE_URL : API_BASE 72 + return `model=${model} api=${api}` 59 73 } 60 74 61 75 /** ··· 138 152 return new Promise((resolve) => setTimeout(resolve, ms)) 139 153 } 140 154 141 - function formatRetryAt(retryAfterMs: number): string { 142 - const retryAt = new Date(Date.now() + retryAfterMs) 155 + function formatAbsoluteTime(timeMs: number): string { 156 + const retryAt = new Date(timeMs) 143 157 const local = retryAt.toLocaleString(undefined, { 144 158 hour12: false, 145 159 timeZoneName: "short", 146 160 }) 147 161 return `${local} (${retryAt.toISOString()})` 162 + } 163 + 164 + function formatRetryAt(retryAfterMs: number): string { 165 + return formatAbsoluteTime(Date.now() + retryAfterMs) 166 + } 167 + 168 + const PRIMARY_FAILOVER_NOTICE_INTERVAL_MS = 60 * 60 * 1000 169 + let lastPrimaryFailoverNoticeAtMs = 0 170 + 171 + function shouldLogPrimaryFailoverNotice(nowMs = Date.now()): boolean { 172 + if (nowMs - lastPrimaryFailoverNoticeAtMs < PRIMARY_FAILOVER_NOTICE_INTERVAL_MS) return false 173 + lastPrimaryFailoverNoticeAtMs = nowMs 174 + return true 148 175 } 149 176 150 177 function apiErrorSearchText(err: { message: string; error?: unknown }): string { ··· 593 620 const beforeCount = state.conversation.length 594 621 const beforeEstimate = estimatePromptTokens(state.conversation) 595 622 596 - const summaryProvider = configuredSummaryProvider() 623 + const summaryProvider = await configuredSummaryProvider() 597 624 if (!summaryProvider.client || !summaryProvider.model) { 598 625 console.warn(`[context] recovery: no summary client available; cannot llm-summarize`) 599 626 return false ··· 691 718 : { messages: baseConversation, recalledChunkIds: [] as number[] } 692 719 const requestMessages = requestContext.messages 693 720 721 + const primaryFailoverBefore = USE_FALLBACK ? null : await primaryFailoverStatus() 722 + if (primaryFailoverBefore?.active) { 723 + if (shouldLogPrimaryFailoverNotice()) { 724 + console.warn( 725 + `[api] primary quota cooldown active; using fallback until ${formatAbsoluteTime(primaryFailoverBefore.retryAtMs)} (${Math.ceil(primaryFailoverBefore.remainingMs / 1000)}s remaining)`, 726 + ) 727 + } 728 + 729 + const fallbackWindow = fallbackContextWindow(requestMessages) 730 + if (fallbackWindow.nearLimit) { 731 + console.warn( 732 + `[fallback] prompt estimate ${fallbackWindow.estimate} nearing fallback limit ${fallbackWindow.softLimit} (${FALLBACK_MODEL})`, 733 + ) 734 + } 735 + 736 + try { 737 + const completion = await createFallbackCompletion(requestMessages) 738 + state.memoryRecallCooldowns = rememberRecalledMemoryChunks( 739 + state.memoryRecallCooldowns, 740 + requestContext.recalledChunkIds, 741 + state.memoryRecallTurn, 742 + ) 743 + return completion 744 + } catch (fallbackErr) { 745 + if (isPromptTooLargeError(fallbackErr) && promptTooLargeAttempts < 2) { 746 + logPromptSizeDebug(state, fallbackErr, `fallback rejected prompt during primary quota cooldown (attempt ${promptTooLargeAttempts + 1}/2)`) 747 + const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts) 748 + promptTooLargeAttempts++ 749 + if (recovered) continue 750 + } 751 + if (recoverByScrubbingImages(fallbackErr, "fallback-primary-quota-cooldown")) continue 752 + if (shouldRetryProvider(fallbackErr)) { 753 + const retryAfter = retryDelayMs(fallbackErr) 754 + console.warn( 755 + `[fallback] transient failure (${errorSummary(fallbackErr)}); retrying after ${Math.ceil(retryAfter / 1000)}s`, 756 + ) 757 + console.log( 758 + `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying fallback...`, 759 + ) 760 + await sleep(retryAfter) 761 + continue 762 + } 763 + logApiError(fallbackErr, `model=${FALLBACK_MODEL} api=${FALLBACK_BASE}`) 764 + throw fallbackErr 765 + } 766 + } else if (primaryFailoverBefore?.retryAtMs) { 767 + console.warn(`[api] primary quota cooldown expired; probing primary before fallback`) 768 + } 769 + 694 770 if (USE_FALLBACK) { 695 771 const fallbackWindow = fallbackContextWindow(requestMessages) 696 772 if (fallbackWindow.nearLimit) { ··· 715 791 if (recovered) continue 716 792 } 717 793 if (recoverByScrubbingImages(fallbackErr, "fallback")) continue 718 - if (shouldFallback(fallbackErr)) { 794 + if (shouldRetryProvider(fallbackErr)) { 719 795 const retryAfter = retryDelayMs(fallbackErr) 720 796 console.warn( 721 797 `[fallback] transient failure (${errorSummary(fallbackErr)}); retrying after ${Math.ceil(retryAfter / 1000)}s`, ··· 733 809 734 810 try { 735 811 const completion = await createPrimaryCompletion(requestMessages) 812 + if (primaryFailoverBefore?.retryAtMs && (await clearPrimaryFailover())) { 813 + console.warn(`[api] primary probe succeeded; restored primary routing`) 814 + } 736 815 state.memoryRecallCooldowns = rememberRecalledMemoryChunks( 737 816 state.memoryRecallCooldowns, 738 817 requestContext.recalledChunkIds, ··· 745 824 const recovered = await recoverFromPromptTooLarge(state, promptTooLargeAttempts) 746 825 promptTooLargeAttempts++ 747 826 if (recovered) continue 748 - logApiError(primaryErr, `model=${MODEL} api=${API_BASE}`) 827 + logApiError(primaryErr, primaryApiContext()) 749 828 throw primaryErr 750 829 } 751 830 752 831 if (recoverByScrubbingImages(primaryErr, "primary")) continue 753 832 754 - if (!shouldFallback(primaryErr)) { 755 - logApiError(primaryErr, `model=${MODEL} api=${API_BASE}`) 833 + const primaryQuotaExhausted = isQuotaExhaustedError(primaryErr) 834 + 835 + if (!primaryQuotaExhausted && !shouldFallback(primaryErr)) { 836 + logApiError(primaryErr, primaryApiContext()) 756 837 throw primaryErr 757 838 } 758 839 840 + if (primaryQuotaExhausted) { 841 + logApiError(primaryErr, `primary quota exhausted; ${primaryApiContext()}`) 842 + const failover = await recordPrimaryQuotaFailover(primaryErr) 843 + console.warn( 844 + `[api] primary quota exhausted; using fallback until ${formatAbsoluteTime(failover.retryAtMs)}`, 845 + ) 846 + } 847 + 759 848 const fallbackWindow = fallbackContextWindow(requestMessages) 760 - if (fallbackWindow.skip) { 849 + if (!primaryQuotaExhausted && fallbackWindow.skip) { 761 850 console.warn( 762 851 `[api] primary down (${errorSummary(primaryErr)}) and fallback context estimate ${fallbackWindow.estimate} exceeds hard limit ${fallbackWindow.hardLimit}; retrying primary after backoff`, 763 852 ) ··· 775 864 ) 776 865 } 777 866 778 - console.warn(`[api] primary down (${errorSummary(primaryErr)}) - switching to fallback`) 867 + console.warn( 868 + primaryQuotaExhausted 869 + ? `[api] primary quota cooldown set - switching to fallback` 870 + : `[api] primary down (${errorSummary(primaryErr)}) - switching to fallback`, 871 + ) 779 872 try { 780 873 const completion = await createFallbackCompletion(requestMessages) 781 874 state.memoryRecallCooldowns = rememberRecalledMemoryChunks( ··· 792 885 if (recovered) continue 793 886 } 794 887 if (recoverByScrubbingImages(fallbackErr, "fallback-failover")) continue 888 + const retryTarget = primaryQuotaExhausted ? "fallback" : "primary" 795 889 console.warn( 796 - `[api] fallback failed (${errorSummary(fallbackErr)}) after primary failure (${errorSummary(primaryErr)}); retrying primary after backoff`, 890 + `[api] fallback failed (${errorSummary(fallbackErr)}) after primary failure (${errorSummary(primaryErr)}); retrying ${retryTarget} after backoff`, 797 891 ) 798 - const retryAfter = retryDelayMs(primaryErr) 892 + const retryAfter = primaryQuotaExhausted ? retryDelayMs(fallbackErr) : retryDelayMs(primaryErr) 799 893 console.log( 800 - `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying primary...`, 894 + `[runner] backing off ${Math.ceil(retryAfter / 1000)}s (until ${formatRetryAt(retryAfter)}) before retrying ${retryTarget}...`, 801 895 ) 802 896 await sleep(retryAfter) 803 897 }
+1 -1
src/runner/loop.ts
··· 176 176 if (state.contextSize < CONTEXT_COMPACT_TRIGGER_TOKENS) return false 177 177 const beforeEstimate = estimatePromptTokens(state.conversation) 178 178 179 - const summaryProvider = configuredSummaryProvider() 179 + const summaryProvider = await configuredSummaryProvider() 180 180 if (!summaryProvider.client || !summaryProvider.model) { 181 181 console.warn(`[context] ${phase}: no summary client available; skipping llm compaction`) 182 182 return false
+81
src/runner/util.test.ts
··· 1 1 import assert from "node:assert/strict" 2 + import fs from "node:fs/promises" 2 3 import test from "node:test" 3 4 import OpenAI from "openai" 4 5 import type { Message } from "../types" 5 6 import { 6 7 AGENT_NAME, 8 + PRIMARY_QUOTA_RETRY_MS, 9 + clearPrimaryFailover, 7 10 isContentFilterError, 8 11 isImageParseError, 12 + isQuotaExhaustedError, 9 13 isTransientTransportError, 14 + primaryFailoverStatus, 15 + recordPrimaryQuotaFailover, 10 16 restForestFromMessages, 11 17 sanitizeMessages, 12 18 scrubImagesFromConversation, 13 19 shouldFallback, 20 + shouldRetryProvider, 14 21 summarizeConversationViaLLM, 15 22 } from "./util" 23 + 24 + const PRIMARY_FAILOVER_FILE = new URL("../../primary-failover.json", import.meta.url) 16 25 17 26 type AssistantMessageWithReasoning = OpenAI.Chat.ChatCompletionAssistantMessageParam & { 18 27 reasoning_content?: string 19 28 } 20 29 30 + async function withPreservedPrimaryFailoverFile(fn: () => Promise<void>): Promise<void> { 31 + let original: string | null = null 32 + try { 33 + original = await fs.readFile(PRIMARY_FAILOVER_FILE, "utf-8") 34 + } catch { 35 + original = null 36 + } 37 + 38 + try { 39 + await clearPrimaryFailover() 40 + await fn() 41 + } finally { 42 + await clearPrimaryFailover() 43 + if (original !== null) { 44 + await fs.writeFile(PRIMARY_FAILOVER_FILE, original, "utf-8") 45 + } 46 + } 47 + } 48 + 21 49 test("sanitizeMessages backfills empty reasoning_content for assistant history", () => { 22 50 const messages = sanitizeMessages([ 23 51 { ··· 52 80 53 81 assert.equal(isTransientTransportError(nested), true) 54 82 assert.equal(shouldFallback(nested), true) 83 + }) 84 + 85 + test("quota-exhausted 403 errors trigger failover but not same-provider retry", () => { 86 + const body = { 87 + error: { 88 + type: "permission_error", 89 + message: "You've reached your usage limit for this billing cycle.", 90 + }, 91 + type: "error", 92 + } 93 + const err = new OpenAI.APIError(403, body, `403 ${JSON.stringify(body)}`, undefined) 94 + 95 + assert.equal(isQuotaExhaustedError(err), true) 96 + assert.equal(shouldFallback(err), true) 97 + assert.equal(shouldRetryProvider(err), false) 98 + }) 99 + 100 + test("plain 403 permission errors do not trigger failover", () => { 101 + const err = new OpenAI.APIError( 102 + 403, 103 + { error: { type: "permission_error", message: "missing required workspace permission" } }, 104 + "403 missing required workspace permission", 105 + undefined, 106 + ) 107 + 108 + assert.equal(isQuotaExhaustedError(err), false) 109 + assert.equal(shouldFallback(err), false) 110 + assert.equal(shouldRetryProvider(err), false) 111 + }) 112 + 113 + test("recordPrimaryQuotaFailover sets a daily primary retry cooldown", async () => { 114 + await withPreservedPrimaryFailoverFile(async () => { 115 + const nowMs = Date.parse("2026-06-01T00:00:00.000Z") 116 + const err = new OpenAI.APIError( 117 + 403, 118 + { error: { type: "permission_error", message: "You've reached your usage limit for this billing cycle." } }, 119 + "403 usage limit", 120 + undefined, 121 + ) 122 + 123 + const recorded = await recordPrimaryQuotaFailover(err, nowMs) 124 + assert.equal(recorded.active, true) 125 + assert.equal(recorded.remainingMs, PRIMARY_QUOTA_RETRY_MS) 126 + assert.equal(recorded.retryAt, new Date(nowMs + PRIMARY_QUOTA_RETRY_MS).toISOString()) 127 + assert.match(recorded.reason ?? "", /usage limit/i) 128 + 129 + const beforeRetry = await primaryFailoverStatus(nowMs + PRIMARY_QUOTA_RETRY_MS - 1) 130 + assert.equal(beforeRetry.active, true) 131 + 132 + const atRetry = await primaryFailoverStatus(nowMs + PRIMARY_QUOTA_RETRY_MS) 133 + assert.equal(atRetry.active, false) 134 + assert.equal(atRetry.remainingMs, 0) 135 + }) 55 136 }) 56 137 57 138 test("z.ai/GLM image parse rejection (code 1210) is detected as an image parse error", () => {
+139 -4
src/runner/util.ts
··· 12 12 const PROJECT_ROOT = path.resolve(fileURLToPath(import.meta.url), "../../..") 13 13 const SESSION_FILE = path.join(PROJECT_ROOT, "session.json") 14 14 const REST_SNAPSHOT_FILE = path.join(PROJECT_ROOT, "rest-snapshot.json") 15 + const PRIMARY_FAILOVER_FILE = path.join(PROJECT_ROOT, "primary-failover.json") 15 16 16 17 export const TOKEN_NUDGE_THRESHOLD = parseInt(process.env.TOKEN_NUDGE_THRESHOLD ?? "120000") 17 18 export const FALLBACK_TOKEN_NUDGE_THRESHOLD = parseInt(process.env.FALLBACK_TOKEN_NUDGE_THRESHOLD ?? "50000") 18 19 export const CONTEXT_COMPACT_TRIGGER_TOKENS = parseInt(process.env.CONTEXT_COMPACT_TRIGGER_TOKENS ?? "90000") 20 + export const PRIMARY_QUOTA_RETRY_MS = Math.max( 21 + 60_000, 22 + Number.parseInt(process.env.PRIMARY_QUOTA_RETRY_MS ?? `${24 * 60 * 60 * 1000}`, 10) || 24 * 60 * 60 * 1000, 23 + ) 19 24 20 25 const NIRI_ENV = (process.env.NIRI_ENV ?? "default").trim().toLowerCase() 21 26 export const USE_FALLBACK = NIRI_ENV === "local" ··· 541 546 await fs.unlink(SESSION_FILE).catch(() => {}) 542 547 } 543 548 549 + type PrimaryFailoverSnapshot = { 550 + failedAt: string 551 + retryAt: string 552 + reason: string 553 + } 554 + 555 + export type PrimaryFailoverStatus = { 556 + active: boolean 557 + retryAtMs: number 558 + retryAt: string | null 559 + remainingMs: number 560 + reason: string | null 561 + } 562 + 563 + let primaryFailoverLoaded = false 564 + let primaryFailoverRetryAtMs = 0 565 + let primaryFailoverReason: string | null = null 566 + 567 + function primaryFailoverStatusFromMemory(nowMs: number): PrimaryFailoverStatus { 568 + const retryAtMs = primaryFailoverRetryAtMs 569 + const remainingMs = Math.max(0, retryAtMs - nowMs) 570 + return { 571 + active: remainingMs > 0, 572 + retryAtMs, 573 + retryAt: retryAtMs > 0 ? new Date(retryAtMs).toISOString() : null, 574 + remainingMs, 575 + reason: primaryFailoverReason, 576 + } 577 + } 578 + 579 + async function loadPrimaryFailoverState(): Promise<void> { 580 + if (primaryFailoverLoaded) return 581 + primaryFailoverLoaded = true 582 + 583 + try { 584 + const raw = await fs.readFile(PRIMARY_FAILOVER_FILE, "utf-8") 585 + const parsed = JSON.parse(raw) as Partial<PrimaryFailoverSnapshot> 586 + const retryAtMs = typeof parsed.retryAt === "string" ? Date.parse(parsed.retryAt) : NaN 587 + primaryFailoverRetryAtMs = Number.isFinite(retryAtMs) && retryAtMs > 0 ? retryAtMs : 0 588 + primaryFailoverReason = typeof parsed.reason === "string" && parsed.reason.trim() ? parsed.reason : null 589 + } catch { 590 + primaryFailoverRetryAtMs = 0 591 + primaryFailoverReason = null 592 + } 593 + } 594 + 595 + export async function primaryFailoverStatus(nowMs = Date.now()): Promise<PrimaryFailoverStatus> { 596 + await loadPrimaryFailoverState() 597 + return primaryFailoverStatusFromMemory(nowMs) 598 + } 599 + 600 + export async function recordPrimaryQuotaFailover(err: unknown, nowMs = Date.now()): Promise<PrimaryFailoverStatus> { 601 + await loadPrimaryFailoverState() 602 + 603 + primaryFailoverRetryAtMs = nowMs + PRIMARY_QUOTA_RETRY_MS 604 + primaryFailoverReason = errorSummary(err) 605 + 606 + const snapshot: PrimaryFailoverSnapshot = { 607 + failedAt: new Date(nowMs).toISOString(), 608 + retryAt: new Date(primaryFailoverRetryAtMs).toISOString(), 609 + reason: primaryFailoverReason, 610 + } 611 + await fs.writeFile(PRIMARY_FAILOVER_FILE, `${JSON.stringify(snapshot, null, 2)}\n`, { encoding: "utf-8", mode: 0o666 }) 612 + return primaryFailoverStatusFromMemory(nowMs) 613 + } 614 + 615 + export async function clearPrimaryFailover(): Promise<boolean> { 616 + await loadPrimaryFailoverState() 617 + const hadFailover = primaryFailoverRetryAtMs > 0 || primaryFailoverReason !== null 618 + primaryFailoverRetryAtMs = 0 619 + primaryFailoverReason = null 620 + if (hadFailover) await fs.unlink(PRIMARY_FAILOVER_FILE).catch(() => {}) 621 + return hadFailover 622 + } 623 + 544 624 type RestSnapshot = { 545 625 restedAt: string 546 626 note?: string ··· 688 768 } 689 769 690 770 /** 691 - * Determines whether an error should trigger fallback model routing. 771 + * Detects provider/account quota exhaustion errors that should fail over to 772 + * the configured fallback provider instead of aborting the runner. 692 773 * 693 - * @param err - Error thrown by the primary API call. 694 - * @returns `true` when fallback should be attempted. 774 + * Some OpenAI-compatible providers return quota exhaustion as a 403 775 + * `permission_error` rather than 429. Treat only quota/billing-shaped 4xx 776 + * errors this way; a plain auth or permission failure should still surface. 695 777 */ 696 - export function shouldFallback(err: unknown): boolean { 778 + export function isQuotaExhaustedError(err: unknown): boolean { 779 + if (!(err instanceof OpenAI.APIError)) return false 780 + if (!err.status || err.status < 400 || err.status >= 500) return false 781 + 782 + const text = apiErrorText(err) 783 + return /usage limit|quota|insufficient[_\s-]*quota|billing cycle|billing hard limit|spending limit|monthly limit|out of credits|no credits|credit balance|insufficient balance|balance (?:is )?(?:not enough|too low)|account balance/i.test( 784 + text, 785 + ) 786 + } 787 + 788 + /** 789 + * Determines whether an error should be retried against the same provider. 790 + * 791 + * @param err - Error thrown by the active API call. 792 + * @returns `true` when a same-provider retry/backoff should be attempted. 793 + */ 794 + export function shouldRetryProvider(err: unknown): boolean { 697 795 if (err instanceof OpenAI.APIError) { 698 796 // 429 + 5xx = overloaded or down; 0/undefined = network-level failure 699 797 if (!err.status || err.status === 429 || err.status >= 500) return true 700 798 return false 701 799 } 702 800 return isTransientTransportError(err) 801 + } 802 + 803 + /** 804 + * Determines whether an error should trigger fallback model routing. 805 + * 806 + * @param err - Error thrown by the primary API call. 807 + * @returns `true` when fallback should be attempted. 808 + */ 809 + export function shouldFallback(err: unknown): boolean { 810 + return shouldRetryProvider(err) || isQuotaExhaustedError(err) 811 + } 812 + 813 + function apiErrorText(err: InstanceType<typeof OpenAI.APIError>): string { 814 + const parts: string[] = [err.message] 815 + if (typeof err.code === "string") parts.push(err.code) 816 + if (typeof err.type === "string") parts.push(err.type) 817 + if (typeof err.param === "string") parts.push(err.param) 818 + collectApiErrorText(err.error, parts) 819 + return parts.join("\n") 820 + } 821 + 822 + function collectApiErrorText(value: unknown, parts: string[], depth = 0): void { 823 + if (depth > 4 || value == null) return 824 + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { 825 + parts.push(String(value)) 826 + return 827 + } 828 + if (Array.isArray(value)) { 829 + for (const item of value.slice(0, 20)) collectApiErrorText(item, parts, depth + 1) 830 + return 831 + } 832 + if (typeof value !== "object") return 833 + 834 + for (const [key, nested] of Object.entries(value as Record<string, unknown>)) { 835 + parts.push(key) 836 + collectApiErrorText(nested, parts, depth + 1) 837 + } 703 838 } 704 839 705 840 function errorCauseChainText(err: unknown): string {