harden upstream retry handling

2026-06-29 18:04:53 +08:00
parent 94bece3cf2
commit 57c7264c7b
5 changed files with 465 additions and 33 deletions
@@ -17,6 +17,8 @@ type GatewayConfig = {
  reasoning_equals?: number[];
  retryable_status_codes?: number[];
  retryable_error_messages?: string[];
+  upstream_fetch_retry_attempts?: number;
+  upstream_fetch_retry_backoff_ms?: number;
  non_stream_status_code?: number;
  log_match?: boolean;
 };
@@ -71,15 +73,20 @@ type RequestEntry = {
  first_response_at?: string | null;
  first_response_delay_ms?: number | null;
  finished_at?: string | null;
+  last_activity_at?: string | null;
  duration_ms?: number | null;
  profile_name?: string;
  method?: string;
  path?: string;
  request_body_bytes?: number | null;
+  response_bytes_received?: number | null;
  model?: string | null;
  requested_model?: string | null;
  forwarded_model?: string | null;
  response_stream?: boolean;
+  stream_chunk_count?: number | null;
+  usage_last_updated_at?: string | null;
+  upstream_attempt_count?: number | null;
  matched?: boolean;
  status_code?: number | null;
  upstream_status_code?: number | null;
@@ -119,6 +126,8 @@ type ProfileFormModel = {
  reasoning_equals?: string;
  retryable_status_codes?: string;
  retryable_error_messages?: string[];
+  upstream_fetch_retry_attempts?: string;
+  upstream_fetch_retry_backoff_ms?: string;
  endpoints?: string[];
 };

@@ -186,6 +195,8 @@ type ProfileFormState = {
  reasoning_equals: string;
  retryable_status_codes: string;
  retryable_error_messages: string;
+  upstream_fetch_retry_attempts: string;
+  upstream_fetch_retry_backoff_ms: string;
  endpoints: string;
 };

@@ -193,6 +204,8 @@ type RuleFormState = {
  reasoning_equals: string;
  retryable_status_codes: string;
  retryable_error_messages: string;
+  upstream_fetch_retry_attempts: string;
+  upstream_fetch_retry_backoff_ms: string;
  endpoints: string;
  non_stream_status_code: string;
  log_match: boolean;
@@ -274,6 +287,8 @@ const defaultProfileForm: ProfileFormState = {
  reasoning_equals: "516,1034,1552",
  retryable_status_codes: "429,503",
  retryable_error_messages: "Selected model is at capacity. Please try a different model.",
+  upstream_fetch_retry_attempts: "5",
+  upstream_fetch_retry_backoff_ms: "350",
  endpoints: "/responses\n/chat/completions\n/v1/responses\n/v1/chat/completions",
 };

@@ -385,6 +400,12 @@ function profileFormFromStatus(status: StatusPayload | null): ProfileFormState {
    retryable_error_messages: Array.isArray(config.retryable_error_messages)
      ? config.retryable_error_messages.join("\n")
      : defaultProfileForm.retryable_error_messages,
+    upstream_fetch_retry_attempts: String(
+      config.upstream_fetch_retry_attempts ?? defaultProfileForm.upstream_fetch_retry_attempts,
+    ),
+    upstream_fetch_retry_backoff_ms: String(
+      config.upstream_fetch_retry_backoff_ms ?? defaultProfileForm.upstream_fetch_retry_backoff_ms,
+    ),
    endpoints: Array.isArray(config.endpoints) ? config.endpoints.join("\n") : defaultProfileForm.endpoints,
  };
 }
@@ -412,6 +433,10 @@ function profileFormFromProfile(profile: Profile): ProfileFormState {
    retryable_error_messages: Array.isArray(form.retryable_error_messages)
      ? form.retryable_error_messages.join("\n")
      : defaultProfileForm.retryable_error_messages,
+    upstream_fetch_retry_attempts:
+      form.upstream_fetch_retry_attempts || defaultProfileForm.upstream_fetch_retry_attempts,
+    upstream_fetch_retry_backoff_ms:
+      form.upstream_fetch_retry_backoff_ms || defaultProfileForm.upstream_fetch_retry_backoff_ms,
    endpoints: Array.isArray(form.endpoints) ? form.endpoints.join("\n") : "",
  };
 }
@@ -426,6 +451,12 @@ function ruleFormFromStatus(status: StatusPayload | null): RuleFormState {
    retryable_error_messages: Array.isArray(config.retryable_error_messages)
      ? config.retryable_error_messages.join("\n")
      : defaultProfileForm.retryable_error_messages,
+    upstream_fetch_retry_attempts: String(
+      config.upstream_fetch_retry_attempts ?? defaultProfileForm.upstream_fetch_retry_attempts,
+    ),
+    upstream_fetch_retry_backoff_ms: String(
+      config.upstream_fetch_retry_backoff_ms ?? defaultProfileForm.upstream_fetch_retry_backoff_ms,
+    ),
    endpoints: Array.isArray(config.endpoints) ? config.endpoints.join("\n") : "",
    non_stream_status_code: String(config.non_stream_status_code || 502),
    log_match: Boolean(config.log_match),
@@ -591,6 +622,8 @@ export default function App() {
            .map((value) => Number.parseInt(value, 10))
            .filter((value) => Number.isInteger(value)),
          retryable_error_messages: splitLines(ruleForm.retryable_error_messages),
+          upstream_fetch_retry_attempts: Number.parseInt(ruleForm.upstream_fetch_retry_attempts, 10),
+          upstream_fetch_retry_backoff_ms: Number.parseInt(ruleForm.upstream_fetch_retry_backoff_ms, 10),
          endpoints: splitLines(ruleForm.endpoints),
          non_stream_status_code: Number.parseInt(ruleForm.non_stream_status_code, 10),
          log_match: ruleForm.log_match,
@@ -639,6 +672,8 @@ export default function App() {
          reasoning_equals: splitList(profileForm.reasoning_equals),
          retryable_status_codes: splitList(profileForm.retryable_status_codes),
          retryable_error_messages: splitLines(profileForm.retryable_error_messages),
+          upstream_fetch_retry_attempts: Number.parseInt(profileForm.upstream_fetch_retry_attempts, 10),
+          upstream_fetch_retry_backoff_ms: Number.parseInt(profileForm.upstream_fetch_retry_backoff_ms, 10),
          endpoints: splitLines(profileForm.endpoints),
        }),
      });
@@ -944,10 +979,20 @@ export default function App() {
                                  <span className="meta-key">总</span>
                                  {durationSeconds(entry.duration_ms)}
                                </span>
+                                <span className="meta-pill meta-progress">
+                                  <span className="meta-key">更</span>
+                                  {timestamp(entry.last_activity_at || entry.usage_last_updated_at)}
+                                </span>
                                <span className="meta-pill meta-body">
                                  <span className="meta-key">体</span>
                                  {bytesFormat(entry.request_body_bytes)}
                                </span>
+                                <span className="meta-pill meta-stream">
+                                  <span className="meta-key">流</span>
+                                  {entry.response_stream
+                                    ? `${numberFormat(entry.stream_chunk_count)} chunk / ${bytesFormat(entry.response_bytes_received)}`
+                                    : "-"}
+                                </span>
                                <span className="meta-pill meta-received">
                                  <span className="meta-key">收</span>
                                  {timestamp(entry.finished_at)}
@@ -982,6 +1027,10 @@ export default function App() {
                                {upstream.auth_mode || "-"} / {upstream.auth_source || "-"}
                                {upstream.authorization_configured === false ? " / no auth" : ""}
                              </span>
+                              <span className="hint">
+                                重试 {numberFormat(entry.upstream_attempt_count)} 次
+                                {entry.upstream_status_code ? ` / upstream ${entry.upstream_status_code}` : ""}
+                              </span>
                            </div>

                            <div className="request-block">
@@ -1144,6 +1193,14 @@ export default function App() {
                    <Field label="retryable_error_messages" hint="每行一条错误文案；上游 JSON 错误包含任一行时会转成本地重试状态码。">
                      <textarea value={profileForm.retryable_error_messages} onChange={(event) => setProfileForm({ ...profileForm, retryable_error_messages: event.target.value })} />
                    </Field>
+                    <div className="field-row">
+                      <Field label="upstream_fetch_retry_attempts">
+                        <input type="number" min={1} value={profileForm.upstream_fetch_retry_attempts} onChange={(event) => setProfileForm({ ...profileForm, upstream_fetch_retry_attempts: event.target.value })} />
+                      </Field>
+                      <Field label="upstream_fetch_retry_backoff_ms">
+                        <input type="number" min={0} step={50} value={profileForm.upstream_fetch_retry_backoff_ms} onChange={(event) => setProfileForm({ ...profileForm, upstream_fetch_retry_backoff_ms: event.target.value })} />
+                      </Field>
+                    </div>
                    <Field label="endpoints">
                      <textarea value={profileForm.endpoints} onChange={(event) => setProfileForm({ ...profileForm, endpoints: event.target.value })} />
                    </Field>
@@ -1209,6 +1266,14 @@ export default function App() {
                  <Field label="retryable_error_messages" hint="每行一条错误文案；命中后会转成本地 non_stream_status_code。">
                    <textarea value={ruleForm.retryable_error_messages} onChange={(event) => setRuleForm({ ...ruleForm, retryable_error_messages: event.target.value })} />
                  </Field>
+                  <div className="field-row">
+                    <Field label="upstream_fetch_retry_attempts">
+                      <input type="number" min={1} value={ruleForm.upstream_fetch_retry_attempts} onChange={(event) => setRuleForm({ ...ruleForm, upstream_fetch_retry_attempts: event.target.value })} />
+                    </Field>
+                    <Field label="upstream_fetch_retry_backoff_ms">
+                      <input type="number" min={0} step={50} value={ruleForm.upstream_fetch_retry_backoff_ms} onChange={(event) => setRuleForm({ ...ruleForm, upstream_fetch_retry_backoff_ms: event.target.value })} />
+                    </Field>
+                  </div>
                  <Field label="endpoints">
                    <textarea value={ruleForm.endpoints} onChange={(event) => setRuleForm({ ...ruleForm, endpoints: event.target.value })} />
                  </Field>
@@ -602,11 +602,21 @@ a {
  background: #f6e3bd;
 }

+.meta-progress {
+  color: #4a4a86;
+  background: #e4e1fb;
+}
+
 .meta-body {
  color: #773f33;
  background: #f8ddd5;
 }

+.meta-stream {
+  color: #5d4f1f;
+  background: #f2ecbf;
+}
+
 .meta-received {
  color: #425268;
  background: #e5eaef;