retry capacity-style upstream errors

scripts/admin-lib.mjs

@@ -177,16 +177,27 @@ export function normalizeIntArray(values, fallback = [516]) {
   return normalized.length > 0 ? [...new Set(normalized)] : [...fallback];
 }
 
-export function normalizeStringArray(values, fallback = []) {
-  const source = values === undefined || values === null ? fallback : values;
-  const queue = Array.isArray(source) ? source.flat(Infinity) : [source];
-  const normalized = queue
+export function normalizeStringArray(values, fallback = []) {
+  const source = values === undefined || values === null ? fallback : values;
+  const queue = Array.isArray(source) ? source.flat(Infinity) : [source];
+  const normalized = queue
     .flatMap((value) => `${value ?? ""}`.split(/[\s,]+/))
     .map((value) => value.trim())
     .filter(Boolean);
-
-  return normalized.length > 0 ? [...new Set(normalized)] : [...fallback];
-}
+
+  return normalized.length > 0 ? [...new Set(normalized)] : [...fallback];
+}
+
+export function normalizePhraseArray(values, fallback = []) {
+  const source = values === undefined || values === null ? fallback : values;
+  const queue = Array.isArray(source) ? source.flat(Infinity) : [source];
+  const normalized = queue
+    .flatMap((value) => (typeof value === "string" ? value.split(/\r?\n/) : [value]))
+    .map((value) => `${value ?? ""}`.trim())
+    .filter(Boolean);
+
+  return normalized.length > 0 ? [...new Set(normalized)] : [...fallback];
+}
 
 export function isProcessAlive(processId) {
   try {
@@ -404,19 +415,23 @@ export async function installForCurrentProvider({
     }
   }
 
-  const gatewayConfig = {
-    listen_host: listenHost,
-    listen_port: listenPort,
-    upstream_base_url: originalBaseUrl,
+  const gatewayConfig = {
+    listen_host: listenHost,
+    listen_port: listenPort,
+    upstream_base_url: originalBaseUrl,
     request_body_limit_bytes:
       existingGatewayConfig?.request_body_limit_bytes === undefined || existingGatewayConfig?.request_body_limit_bytes === null
         ? 10485760
         : Number.parseInt(`${existingGatewayConfig.request_body_limit_bytes}`, 10),
-    endpoints: mergedEndpoints,
-    reasoning_equals: normalizeIntArray(existingGatewayConfig?.reasoning_equals, [516]),
-    non_stream_status_code:
-      existingGatewayConfig?.non_stream_status_code === undefined || existingGatewayConfig?.non_stream_status_code === null
-        ? 502
+    endpoints: mergedEndpoints,
+    reasoning_equals: normalizeIntArray(existingGatewayConfig?.reasoning_equals, [516]),
+    retryable_status_codes: normalizeIntArray(existingGatewayConfig?.retryable_status_codes, [429, 503]),
+    retryable_error_messages: normalizePhraseArray(existingGatewayConfig?.retryable_error_messages, [
+      "Selected model is at capacity. Please try a different model.",
+    ]),
+    non_stream_status_code:
+      existingGatewayConfig?.non_stream_status_code === undefined || existingGatewayConfig?.non_stream_status_code === null
+        ? 502
         : Number.parseInt(`${existingGatewayConfig.non_stream_status_code}`, 10),
     stream_action: existingGatewayConfig?.stream_action || "strict_502",
     log_match: existingGatewayConfig?.log_match === undefined ? true : Boolean(existingGatewayConfig.log_match),

scripts/run-profile.mjs

@@ -17,6 +17,7 @@ import {
   getGatewayBaseUrl,
   getGatewayStatePaths,
   normalizeIntArray,
+  normalizePhraseArray,
   normalizeStringArray,
   parseOptions,
   readJsonFile,
@@ -185,6 +186,14 @@ function buildProfileConfig({ profileName, profileEnv, existingGatewayConfig, pr
       profileEnv.CODEX_RETRY_GATEWAY_REASONING_EQUALS || existingGatewayConfig?.reasoning_equals,
       [516],
     ),
+    retryable_status_codes: normalizeIntArray(
+      profileEnv.CODEX_RETRY_GATEWAY_RETRYABLE_STATUS_CODES || existingGatewayConfig?.retryable_status_codes,
+      [429, 503],
+    ),
+    retryable_error_messages: normalizePhraseArray(
+      profileEnv.CODEX_RETRY_GATEWAY_RETRYABLE_ERROR_MESSAGES || existingGatewayConfig?.retryable_error_messages,
+      ["Selected model is at capacity. Please try a different model."],
+    ),
     non_stream_status_code: profileEnv.CODEX_RETRY_GATEWAY_NON_STREAM_STATUS_CODE
       ? Number.parseInt(`${profileEnv.CODEX_RETRY_GATEWAY_NON_STREAM_STATUS_CODE}`, 10)
       : Number.parseInt(`${existingGatewayConfig?.non_stream_status_code || 502}`, 10),

scripts/test-gateway-e2e.mjs

@@ -123,6 +123,20 @@ function startFakeUpstream(port) {
           ]);
           return;
         }
+        if (parsed.test_capacity_error) {
+          createJsonResponse(
+            res,
+            parsed.test_capacity_status ?? 503,
+            {
+              error: {
+                message: parsed.test_capacity_message || "Selected model is at capacity. Please try a different model.",
+                type: "server_error",
+              },
+            },
+            { "x-upstream-test": "capacity-error" },
+          );
+          return;
+        }
         if (parsed.stream) {
           createSseResponse(res, [
             'data: {"type":"response.output_text.delta","delta":"hello"}\n\n',
@@ -273,11 +287,13 @@ async function run() {
     listen_port: gatewayPort,
     upstream_base_url: `http://127.0.0.1:${upstreamPort}`,
     request_body_limit_bytes: 10 * 1024 * 1024,
-    endpoints: ["/responses", "/chat/completions", "/v1/responses", "/v1/chat/completions"],
-    reasoning_equals: [516],
-    non_stream_status_code: 502,
+    endpoints: ["/responses", "/chat/completions", "/v1/responses", "/v1/chat/completions"],
+    reasoning_equals: [516],
+    retryable_status_codes: [429, 503],
+    retryable_error_messages: ["Selected model is at capacity. Please try a different model."],
+    non_stream_status_code: 502,
     stream_action: "strict_502",
-    log_match: true,
+    log_match: true,
     health_path: "/__codex_retry_gateway/health",
   };
 
@@ -349,6 +365,34 @@ async function run() {
       `请求体大小记录异常: ${recoveredEntry?.request_body_bytes}`,
     );
 
+    const capacityResponse = await fetch(`http://127.0.0.1:${gatewayPort}/responses`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ test_capacity_error: true }),
+    });
+    const capacityBody = await capacityResponse.json();
+    assert(capacityResponse.status === 502, `capacity error 未返回 502: ${capacityResponse.status}`);
+    assert(
+      capacityBody?.error?.code === "upstream_error_retry_triggered",
+      "capacity error 返回体未标记 retry trigger",
+    );
+    assert(
+      capacityBody?.error?.upstream_status_code === 503,
+      "capacity error 返回体未保留 upstream status",
+    );
+
+    const streamCapacityResponse = await fetch(`http://127.0.0.1:${gatewayPort}/responses`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ stream: true, test_capacity_error: true }),
+    });
+    const streamCapacityBody = await streamCapacityResponse.json();
+    assert(streamCapacityResponse.status === 502, `stream+capacity error 未返回 502: ${streamCapacityResponse.status}`);
+    assert(
+      streamCapacityBody?.error?.code === "upstream_error_retry_triggered",
+      "stream+capacity error 返回体未标记 retry trigger",
+    );
+
     for (const streamPath of [
       "/responses",
       "/v1/responses",

scripts/test-install-restore.mjs

@@ -221,6 +221,8 @@ async function run() {
       headers: { "content-type": "application/json" },
       body: JSON.stringify({
         reasoning_equals: [1024],
+        retryable_status_codes: [429, 503, 529],
+        retryable_error_messages: ["Selected model is at capacity. Please try a different model."],
         endpoints: ["/responses", "/v1/responses"],
         non_stream_status_code: 503,
         log_match: false,
@@ -237,6 +239,10 @@ async function run() {
       JSON.stringify(updatedGatewayConfig.reasoning_equals) === JSON.stringify([1024]),
       "Saved config file did not persist reasoning_equals",
     );
+    assert(
+      JSON.stringify(updatedGatewayConfig.retryable_status_codes) === JSON.stringify([429, 503, 529]),
+      "Saved config file did not persist retryable_status_codes",
+    );
 
     const incrementalLogsResponse = await fetch(
       `http://127.0.0.1:${gatewayPort}/__codex_retry_gateway/api/logs?since_seq=${logsPayload.latest_seq}`,