persist gateway metrics across restarts

README.md

@@ -202,7 +202,7 @@ gateway 运行时只负责 API 与静态文件服务，不再把复杂 UI 硬写
 页面里可以直接做这几件事：
 
 - 看当前监听地址、真实上游、当前 provider、当前 Codex base URL
-- 看本次 gateway 启动以来的实时统计
+- 看基于持久请求记录累计的实时统计
   - 代理请求总数
   - 被检查响应总数
   - 累计 input / output / total / reasoning tokens
@@ -229,11 +229,11 @@ gateway 运行时只负责 API 与静态文件服务，不再把复杂 UI 硬写
 - 页面点“恢复 Codex 原设置并关闭网关”后，当前页面会失联，这是预期行为
 - 日常恢复优先用 UI；`restore-codex-config.ps1` 作为脚本级应急回滚入口保留
 - UI 恢复不会再额外拉起恢复子进程，而是由当前 gateway 直接完成恢复并退出
-- 统计口径默认按“本次 gateway 启动以来”累计
+- 统计口径默认按持久请求记录累计，重启后会从 `requests.sqlite` / `requests.jsonl` 回灌
 - `516` 占比 = `reasoning_tokens = 516` 的响应次数 / 被检查响应总数
 - 请求历史只记录元数据、请求体字节数和 token usage，不保存请求正文或响应正文；默认展示最近 200 条
 - gateway 日志持久化到 `~/.codex-retry-gateway/logs/gateway.log`
-- 请求记录持久化到 `~/.codex-retry-gateway/logs/requests.jsonl`，重启后仍可用于 UI 请求页和 token totals
+- 请求记录持久化到 `~/.codex-retry-gateway/logs/requests.jsonl` 与 `~/.codex-retry-gateway/logs/requests.sqlite`，重启后会用于 UI 请求页以及 overview 的累计 token / reasoning 统计
 - `manual_bearer` 的手动 token/password 只写入系统 secret 文件，API/UI 不读回明文；profile env 只保存 secret 文件路径
 - 其他 profile env 不保存明文 `sk-...`；固定密钥请使用 env/file 引用，或用 `auth_json` 指向 `auth.json` 字段名
 

build.md

@@ -81,7 +81,7 @@ http://127.0.0.1:4610/__codex_retry_gateway/ui
 页面支持：
 
 - 查看当前接管状态
-- 查看本次启动以来的实时日志
+- 查看跨重启保留的请求累计统计，以及当前进程的实时日志
 - 查看 `516` 命中次数与 `516` 占比
 - 热更新 `reasoning_equals` / `endpoints` / `non_stream_status_code` / `log_match`
 - 一键恢复 Codex 原设置并关闭 gateway

gateway.mjs

@@ -360,6 +360,7 @@ function buildGatewayErrorBody(message) {
 function createMonitor() {
   return {
     started_at: new Date().toISOString(),
+    persistent_since: null,
     next_log_seq: 1,
     next_request_seq: 1,
     log_entries: [],
@@ -613,15 +614,70 @@ function buildRequestQueryFilters({ query, filter }) {
   };
 }
 
-async function hydrateMonitorFromDisk(monitor, paths, requestHistoryLimit) {
+async function hydrateMonitorFromDisk(monitor, paths, requestHistoryLimit, requestsDb = null) {
-  const requestEntries = await readJsonlFile(paths.requestsPath);
   monitor.request_entries = [];
+
+  if (requestsDb) {
+    const totalsRow = requestsDb.prepare(`
+      SELECT
+        COUNT(*) AS total_proxy_request_count,
+        COALESCE(SUM(CASE WHEN inspected = 1 THEN 1 ELSE 0 END), 0) AS inspected_response_count,
+        COALESCE(SUM(CASE WHEN matched = 1 THEN 1 ELSE 0 END), 0) AS matched_response_count,
+        COALESCE(SUM(COALESCE(input_tokens, 0)), 0) AS input_tokens,
+        COALESCE(SUM(COALESCE(output_tokens, 0)), 0) AS output_tokens,
+        COALESCE(SUM(COALESCE(total_tokens, 0)), 0) AS total_tokens,
+        COALESCE(SUM(COALESCE(reasoning_tokens, 0)), 0) AS reasoning_tokens,
+        COALESCE(SUM(COALESCE(cached_tokens, 0)), 0) AS cached_tokens,
+        MIN(NULLIF(started_at, '')) AS persistent_since,
+        MAX(seq) AS max_seq
+      FROM requests
+    `).get();
+
+    monitor.total_proxy_request_count = totalsRow?.total_proxy_request_count || 0;
+    monitor.inspected_response_count = totalsRow?.inspected_response_count || 0;
+    monitor.matched_response_count = totalsRow?.matched_response_count || 0;
+    monitor.token_totals = {
+      input_tokens: totalsRow?.input_tokens || 0,
+      output_tokens: totalsRow?.output_tokens || 0,
+      total_tokens: totalsRow?.total_tokens || 0,
+      reasoning_tokens: totalsRow?.reasoning_tokens || 0,
+      cached_tokens: totalsRow?.cached_tokens || 0,
+    };
+    monitor.persistent_since = totalsRow?.persistent_since || null;
+    monitor.next_request_seq = Number.isInteger(totalsRow?.max_seq) ? totalsRow.max_seq + 1 : 1;
+    monitor.observed_reasoning_counts = {};
+
+    const reasoningRows = requestsDb.prepare(`
+      SELECT reasoning_tokens, COUNT(*) AS count
+      FROM requests
+      WHERE reasoning_tokens IS NOT NULL
+      GROUP BY reasoning_tokens
+    `).all();
+    for (const row of reasoningRows) {
+      if (!Number.isInteger(row?.reasoning_tokens)) {
+        continue;
+      }
+      monitor.observed_reasoning_counts[`${row.reasoning_tokens}`] = row.count || 0;
+    }
+    return;
+  }
+
+  const requestEntries = await readJsonlFile(paths.requestsPath);
   monitor.next_request_seq = requestEntries.reduce((maxSeq, entry) => {
     return Math.max(maxSeq, Number.isInteger(entry.seq) ? entry.seq + 1 : maxSeq);
   }, 1);
   for (const entry of requestEntries) {
+    monitor.total_proxy_request_count += 1;
+    if (entry.inspected) {
+      recordInspectedResponse(monitor, entry.reasoning_tokens ?? entry.usage?.reasoning_tokens ?? null, Boolean(entry.matched));
+    }
     addTokenTotals(monitor, entry.usage);
   }
+  const firstStartedAt = requestEntries
+    .map((entry) => `${entry?.started_at || ""}`.trim())
+    .filter(Boolean)
+    .sort()[0];
+  monitor.persistent_since = firstStartedAt || null;
 }
 
 function createMonitorRecorder(monitor) {
@@ -980,6 +1036,7 @@ function buildMetricsSnapshot(monitor) {
   const inspectedResponseCount = monitor.inspected_response_count;
   return {
     started_at: monitor.started_at,
+    persistent_since: monitor.persistent_since,
     total_proxy_request_count: monitor.total_proxy_request_count,
     inspected_response_count: inspectedResponseCount,
     matched_response_count: monitor.matched_response_count,
@@ -2684,9 +2741,9 @@ async function main() {
     requestsDb,
     server: null,
   };
-  await hydrateMonitorFromDisk(monitor, runtime.paths, config.request_history_limit);
   const importedCount = await importRequestsJsonlToDb(requestsDb, runtime.paths.requestsPath);
-  logger(`[start] hydrated token totals from jsonl path=${runtime.paths.requestsPath}`);
+  await hydrateMonitorFromDisk(monitor, runtime.paths, config.request_history_limit, requestsDb);
+  logger(`[start] hydrated persistent request metrics from db path=${runtime.paths.requestsDbPath}`);
   logger(`[start] requests db ready path=${runtime.paths.requestsDbPath} imported_jsonl_rows=${importedCount}`);
 
   const server = http.createServer(async (req, res) => {

scripts/test-gateway-e2e.mjs

@@ -299,8 +299,8 @@ async function run() {
 
   await writeFile(configPath, JSON.stringify(config, null, 2), "utf8");
 
-  const upstream = await startFakeUpstream(upstreamPort);
+  const upstream = await startFakeUpstream(upstreamPort);
-  const gateway = startGateway(configPath, logPath);
+  let gateway = startGateway(configPath, logPath);
 
   try {
     try {
@@ -427,6 +427,26 @@ async function run() {
     );
     assert(terminatedStream.status === 502, `/responses 上游半路断流未返回 502: ${terminatedStream.status}`);
 
+    const metricsBeforeRestartResponse = await fetch(`http://127.0.0.1:${gatewayPort}/__codex_retry_gateway/api/status`);
+    const metricsBeforeRestart = await metricsBeforeRestartResponse.json();
+    assert(metricsBeforeRestartResponse.status === 200, `status API 状态异常: ${metricsBeforeRestartResponse.status}`);
+    assert(metricsBeforeRestart?.metrics?.reasoning_516_count >= 1, "重启前 reasoning_516_count 未累计");
+    assert(metricsBeforeRestart?.metrics?.observed_reasoning_counts?.["128"] >= 1, "重启前 reasoning 128 未累计");
+    assert(metricsBeforeRestart?.metrics?.total_proxy_request_count >= 1, "重启前 total_proxy_request_count 未累计");
+
+    gateway.child.kill();
+    await once(gateway.child, "exit");
+    gateway = startGateway(configPath, logPath);
+    await waitForHealth(`http://127.0.0.1:${gatewayPort}${config.health_path}`);
+
+    const metricsAfterRestartResponse = await fetch(`http://127.0.0.1:${gatewayPort}/__codex_retry_gateway/api/status`);
+    const metricsAfterRestart = await metricsAfterRestartResponse.json();
+    assert(metricsAfterRestartResponse.status === 200, `重启后 status API 状态异常: ${metricsAfterRestartResponse.status}`);
+    assert(metricsAfterRestart?.metrics?.reasoning_516_count >= metricsBeforeRestart?.metrics?.reasoning_516_count, "重启后 reasoning_516_count 未保留");
+    assert(metricsAfterRestart?.metrics?.observed_reasoning_counts?.["128"] >= metricsBeforeRestart?.metrics?.observed_reasoning_counts?.["128"], "重启后 reasoning 128 计数未保留");
+    assert(metricsAfterRestart?.metrics?.total_proxy_request_count >= metricsBeforeRestart?.metrics?.total_proxy_request_count, "重启后 total_proxy_request_count 未保留");
+    assert(metricsAfterRestart?.metrics?.persistent_since, "重启后未返回 persistent_since");
+
     await new Promise((resolve) => setTimeout(resolve, 120));
     const logText = await readFile(logPath, "utf8");
     assert(
@@ -435,9 +455,9 @@ async function run() {
     );
 
     process.stdout.write("PASS codex-retry-gateway e2e\n");
-  } finally {
+  } finally {
-    gateway.child.kill();
+    gateway.child.kill();
-    upstream.close();
+    upstream.close();
     await once(upstream, "close");
     await rm(tempRoot, { recursive: true, force: true });
   }

ui-src/src/App.tsx

@@ -23,6 +23,7 @@ type GatewayConfig = {
 
 type Metrics = {
   started_at?: string;
+  persistent_since?: string | null;
   total_proxy_request_count?: number;
   inspected_response_count?: number;
   matched_response_count?: number;
@@ -230,7 +231,7 @@ type ProfileProbePayload = {
 const pageCopy: Record<PageKey, { title: string; subtitle: string; index: string }> = {
   overview: {
     title: "概览",
-    subtitle: "当前 gateway 运行态和本次启动以来的累计统计。",
+    subtitle: "当前 gateway 运行态，以及基于持久请求记录回灌的累计统计。",
     index: "01",
   },
   requests: {
@@ -808,7 +809,7 @@ export default function App() {
                   </div>
                 </Card>
 
-                <Card title="Token Totals" hint="按本次 gateway 启动以来累计；后续可以接持久化统计。">
+                <Card title="Token Totals" hint="按持久请求记录累计；重启后会从 requests history 回灌。">
                   <div className="token-strip">
                     <TokenCard label="Input" value={numberFormat(effectiveInputTotal ?? 0)} />
                     <TokenCard label="Output" value={numberFormat(tokens.output_tokens || 0)} />
@@ -839,7 +840,8 @@ export default function App() {
                       rows={[
                         ["516 命中", numberFormat(metrics.reasoning_516_count || 0)],
                         ["516 占比", percent(metrics.reasoning_516_ratio || 0)],
-                        ["启动时间", timestamp(metrics.started_at)],
+                        ["累计起点", timestamp(metrics.persistent_since || metrics.started_at)],
+                        ["本次启动", timestamp(metrics.started_at)],
                         ["Config", status?.paths?.config_path || "-"],
                         ["备份", status?.state?.latest_backup_path || "-"],
                       ]}