harden upstream retry handling

scripts/test-gateway-e2e.mjs

@@ -39,24 +39,24 @@ function createJsonResponse(res, statusCode, body, extraHeaders = {}) {
   res.end(JSON.stringify(body));
 }
 
-function createSseResponse(res, chunks) {
-  res.writeHead(200, {
-    "content-type": "text/event-stream; charset=utf-8",
-    "cache-control": "no-cache",
-    connection: "keep-alive",
+function createSseResponse(res, chunks, intervalMs = 20) {
+  res.writeHead(200, {
+    "content-type": "text/event-stream; charset=utf-8",
+    "cache-control": "no-cache",
+    connection: "keep-alive",
     "x-upstream-test": "sse",
   });
 
   let index = 0;
   const timer = setInterval(() => {
-    if (index >= chunks.length) {
-      clearInterval(timer);
-      res.end();
-      return;
-    }
-    res.write(chunks[index]);
-    index += 1;
-  }, 20);
+    if (index >= chunks.length) {
+      clearInterval(timer);
+      res.end();
+      return;
+    }
+    res.write(chunks[index]);
+    index += 1;
+  }, intervalMs);
 
   res.on("close", () => {
     clearInterval(timer);
@@ -82,6 +82,7 @@ function createTerminatedSseResponse(res, chunks, destroyDelayMs = 20) {
 
 function startFakeUpstream(port) {
   const failBeforeResponseCounts = new Map();
+  const capacityBeforeSuccessCounts = new Map();
   const server = http.createServer((req, res) => {
     const responsePaths = new Set(["/responses", "/v1/responses"]);
     const chatCompletionPaths = new Set(["/chat/completions", "/v1/chat/completions"]);
@@ -137,12 +138,31 @@ function startFakeUpstream(port) {
           );
           return;
         }
+        if (parsed.test_capacity_before_success_times) {
+          const capacityKey = `${req.url}:capacity-before-success:${parsed.test_capacity_before_success_times}`;
+          const capacityCount = (capacityBeforeSuccessCounts.get(capacityKey) || 0) + 1;
+          capacityBeforeSuccessCounts.set(capacityKey, capacityCount);
+          if (capacityCount <= parsed.test_capacity_before_success_times) {
+            createJsonResponse(
+              res,
+              parsed.test_capacity_status ?? 503,
+              {
+                error: {
+                  message: parsed.test_capacity_message || "Selected model is at capacity. Please try a different model.",
+                  type: "server_error",
+                },
+              },
+              { "x-upstream-test": "capacity-error" },
+            );
+            return;
+          }
+        }
         if (parsed.stream) {
           createSseResponse(res, [
             'data: {"type":"response.output_text.delta","delta":"hello"}\n\n',
             `data: {"response":{"usage":{"output_tokens_details":{"reasoning_tokens":${reasoning}}}}}\n\n`,
             "data: [DONE]\n\n",
-          ]);
+          ], parsed.test_stream_chunk_delay_ms ?? 20);
           return;
         }
         createJsonResponse(
@@ -283,19 +303,21 @@ async function run() {
   const logPath = path.join(tempRoot, "gateway.log");
 
   const config = {
-    listen_host: "127.0.0.1",
-    listen_port: gatewayPort,
-    upstream_base_url: `http://127.0.0.1:${upstreamPort}`,
-    request_body_limit_bytes: 10 * 1024 * 1024,
+    listen_host: "127.0.0.1",
+    listen_port: gatewayPort,
+    upstream_base_url: `http://127.0.0.1:${upstreamPort}`,
+    request_body_limit_bytes: 10 * 1024 * 1024,
     endpoints: ["/responses", "/chat/completions", "/v1/responses", "/v1/chat/completions"],
     reasoning_equals: [516],
     retryable_status_codes: [429, 503],
     retryable_error_messages: ["Selected model is at capacity. Please try a different model."],
+    upstream_fetch_retry_attempts: 5,
+    upstream_fetch_retry_backoff_ms: 25,
     non_stream_status_code: 502,
     stream_action: "strict_502",
     log_match: true,
-    health_path: "/__codex_retry_gateway/health",
-  };
+    health_path: "/__codex_retry_gateway/health",
+  };
 
   await writeFile(configPath, JSON.stringify(config, null, 2), "utf8");
 
@@ -381,6 +403,25 @@ async function run() {
       "capacity error 返回体未保留 upstream status",
     );
 
+    const capacityRecoveredResponse = await fetch(`http://127.0.0.1:${gatewayPort}/responses`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ test_capacity_before_success_times: 2, test_reasoning_tokens: 128 }),
+    });
+    const capacityRecoveredBody = await capacityRecoveredResponse.json();
+    assert(capacityRecoveredResponse.status === 200, `capacity 抖动后未自动恢复: ${capacityRecoveredResponse.status}`);
+    assert(
+      capacityRecoveredBody?.usage?.output_tokens_details?.reasoning_tokens === 128,
+      "capacity 抖动恢复后的返回体异常",
+    );
+
+    const requestsAfterCapacityRecoveryResponse = await fetch(`http://127.0.0.1:${gatewayPort}/__codex_retry_gateway/api/requests?limit=20`);
+    const requestsAfterCapacityRecovery = await requestsAfterCapacityRecoveryResponse.json();
+    const capacityRecoveredEntry = requestsAfterCapacityRecovery?.entries?.find(
+      (entry) => entry.path === "/responses" && entry.status_code === 200 && entry.upstream_attempt_count >= 3,
+    );
+    assert(capacityRecoveredEntry, "capacity 抖动恢复后的请求记录未保留重试次数");
+
     const streamCapacityResponse = await fetch(`http://127.0.0.1:${gatewayPort}/responses`, {
       method: "POST",
       headers: { "content-type": "application/json" },
@@ -421,6 +462,36 @@ async function run() {
       assert(!okStream.closedByError, `${streamPath} 流式 128 不应异常断开`);
     }
 
+    const streamProgressPromise = fetch(`http://127.0.0.1:${gatewayPort}/responses`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ stream: true, test_reasoning_tokens: 128, test_stream_chunk_delay_ms: 180 }),
+    });
+    await new Promise((resolve) => setTimeout(resolve, 260));
+    const midRequestsResponse = await fetch(`http://127.0.0.1:${gatewayPort}/__codex_retry_gateway/api/requests?limit=20`);
+    const midRequestsPayload = await midRequestsResponse.json();
+    const inFlightStreamEntry = midRequestsPayload?.entries?.find(
+      (entry) =>
+        entry.path === "/responses" &&
+        entry.response_stream === true &&
+        (entry.lifecycle_state === "streaming" || entry.lifecycle_state === "receive_first") &&
+        (entry.stream_chunk_count || 0) >= 1,
+    );
+    assert(inFlightStreamEntry, "流式请求过程中未暴露进行中状态");
+    assert(
+      (inFlightStreamEntry?.response_bytes_received || 0) > 0,
+      "流式请求过程中未累计接收字节数",
+    );
+    const streamProgressResponse = await streamProgressPromise;
+    assert(streamProgressResponse.status === 200, `stream progress 响应状态异常: ${streamProgressResponse.status}`);
+    const streamReader = streamProgressResponse.body.getReader();
+    while (true) {
+      const { done } = await streamReader.read();
+      if (done) {
+        break;
+      }
+    }
+
     const terminatedStream = await readSseUntilClose(
       `http://127.0.0.1:${gatewayPort}/responses`,
       { stream: true, test_force_terminate: true },