persist gateway metrics across restarts
This commit is contained in:
@@ -202,7 +202,7 @@ gateway 运行时只负责 API 与静态文件服务，不再把复杂 UI 硬写
|
||||
页面里可以直接做这几件事：
|
||||
|
||||
- 看当前监听地址、真实上游、当前 provider、当前 Codex base URL
|
||||
- 看本次 gateway 启动以来的实时统计
|
||||
- 看基于持久请求记录累计的实时统计
|
||||
- 代理请求总数
|
||||
- 被检查响应总数
|
||||
- 累计 input / output / total / reasoning tokens
|
||||
@@ -229,11 +229,11 @@ gateway 运行时只负责 API 与静态文件服务，不再把复杂 UI 硬写
|
||||
- 页面点“恢复 Codex 原设置并关闭网关”后，当前页面会失联，这是预期行为
|
||||
- 日常恢复优先用 UI；`restore-codex-config.ps1` 作为脚本级应急回滚入口保留
|
||||
- UI 恢复不会再额外拉起恢复子进程，而是由当前 gateway 直接完成恢复并退出
|
||||
- 统计口径默认按“本次 gateway 启动以来”累计
|
||||
- 统计口径默认按持久请求记录累计，重启后会从 `requests.sqlite` / `requests.jsonl` 回灌
|
||||
- `516` 占比 = `reasoning_tokens = 516` 的响应次数 / 被检查响应总数
|
||||
- 请求历史只记录元数据、请求体字节数和 token usage，不保存请求正文或响应正文；默认展示最近 200 条
|
||||
- gateway 日志持久化到 `~/.codex-retry-gateway/logs/gateway.log`
|
||||
- 请求记录持久化到 `~/.codex-retry-gateway/logs/requests.jsonl`，重启后仍可用于 UI 请求页和 token totals
|
||||
- 请求记录持久化到 `~/.codex-retry-gateway/logs/requests.jsonl` 与 `~/.codex-retry-gateway/logs/requests.sqlite`，重启后会用于 UI 请求页以及 overview 的累计 token / reasoning 统计
|
||||
- `manual_bearer` 的手动 token/password 只写入系统 secret 文件，API/UI 不读回明文；profile env 只保存 secret 文件路径
|
||||
- 其他 profile env 不保存明文 `sk-...`；固定密钥请使用 env/file 引用，或用 `auth_json` 指向 `auth.json` 字段名
|
||||
|
||||
|
||||
@@ -81,7 +81,7 @@ http://127.0.0.1:4610/__codex_retry_gateway/ui
|
||||
页面支持：
|
||||
|
||||
- 查看当前接管状态
|
||||
- 查看本次启动以来的实时日志
|
||||
- 查看跨重启保留的请求累计统计，以及当前进程的实时日志
|
||||
- 查看 `516` 命中次数与 `516` 占比
|
||||
- 热更新 `reasoning_equals` / `endpoints` / `non_stream_status_code` / `log_match`
|
||||
- 一键恢复 Codex 原设置并关闭 gateway
|
||||
|
||||
+61
-4
@@ -360,6 +360,7 @@ function buildGatewayErrorBody(message) {
|
||||
function createMonitor() {
|
||||
return {
|
||||
started_at: new Date().toISOString(),
|
||||
persistent_since: null,
|
||||
next_log_seq: 1,
|
||||
next_request_seq: 1,
|
||||
log_entries: [],
|
||||
@@ -613,15 +614,70 @@ function buildRequestQueryFilters({ query, filter }) {
|
||||
};
|
||||
}
|
||||
|
||||
async function hydrateMonitorFromDisk(monitor, paths, requestHistoryLimit) {
|
||||
const requestEntries = await readJsonlFile(paths.requestsPath);
|
||||
async function hydrateMonitorFromDisk(monitor, paths, requestHistoryLimit, requestsDb = null) {
|
||||
monitor.request_entries = [];
|
||||
|
||||
if (requestsDb) {
|
||||
const totalsRow = requestsDb.prepare(`
|
||||
SELECT
|
||||
COUNT(*) AS total_proxy_request_count,
|
||||
COALESCE(SUM(CASE WHEN inspected = 1 THEN 1 ELSE 0 END), 0) AS inspected_response_count,
|
||||
COALESCE(SUM(CASE WHEN matched = 1 THEN 1 ELSE 0 END), 0) AS matched_response_count,
|
||||
COALESCE(SUM(COALESCE(input_tokens, 0)), 0) AS input_tokens,
|
||||
COALESCE(SUM(COALESCE(output_tokens, 0)), 0) AS output_tokens,
|
||||
COALESCE(SUM(COALESCE(total_tokens, 0)), 0) AS total_tokens,
|
||||
COALESCE(SUM(COALESCE(reasoning_tokens, 0)), 0) AS reasoning_tokens,
|
||||
COALESCE(SUM(COALESCE(cached_tokens, 0)), 0) AS cached_tokens,
|
||||
MIN(NULLIF(started_at, '')) AS persistent_since,
|
||||
MAX(seq) AS max_seq
|
||||
FROM requests
|
||||
`).get();
|
||||
|
||||
monitor.total_proxy_request_count = totalsRow?.total_proxy_request_count || 0;
|
||||
monitor.inspected_response_count = totalsRow?.inspected_response_count || 0;
|
||||
monitor.matched_response_count = totalsRow?.matched_response_count || 0;
|
||||
monitor.token_totals = {
|
||||
input_tokens: totalsRow?.input_tokens || 0,
|
||||
output_tokens: totalsRow?.output_tokens || 0,
|
||||
total_tokens: totalsRow?.total_tokens || 0,
|
||||
reasoning_tokens: totalsRow?.reasoning_tokens || 0,
|
||||
cached_tokens: totalsRow?.cached_tokens || 0,
|
||||
};
|
||||
monitor.persistent_since = totalsRow?.persistent_since || null;
|
||||
monitor.next_request_seq = Number.isInteger(totalsRow?.max_seq) ? totalsRow.max_seq + 1 : 1;
|
||||
monitor.observed_reasoning_counts = {};
|
||||
|
||||
const reasoningRows = requestsDb.prepare(`
|
||||
SELECT reasoning_tokens, COUNT(*) AS count
|
||||
FROM requests
|
||||
WHERE reasoning_tokens IS NOT NULL
|
||||
GROUP BY reasoning_tokens
|
||||
`).all();
|
||||
for (const row of reasoningRows) {
|
||||
if (!Number.isInteger(row?.reasoning_tokens)) {
|
||||
continue;
|
||||
}
|
||||
monitor.observed_reasoning_counts[`${row.reasoning_tokens}`] = row.count || 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const requestEntries = await readJsonlFile(paths.requestsPath);
|
||||
monitor.next_request_seq = requestEntries.reduce((maxSeq, entry) => {
|
||||
return Math.max(maxSeq, Number.isInteger(entry.seq) ? entry.seq + 1 : maxSeq);
|
||||
}, 1);
|
||||
for (const entry of requestEntries) {
|
||||
monitor.total_proxy_request_count += 1;
|
||||
if (entry.inspected) {
|
||||
recordInspectedResponse(monitor, entry.reasoning_tokens ?? entry.usage?.reasoning_tokens ?? null, Boolean(entry.matched));
|
||||
}
|
||||
addTokenTotals(monitor, entry.usage);
|
||||
}
|
||||
const firstStartedAt = requestEntries
|
||||
.map((entry) => `${entry?.started_at || ""}`.trim())
|
||||
.filter(Boolean)
|
||||
.sort()[0];
|
||||
monitor.persistent_since = firstStartedAt || null;
|
||||
}
|
||||
|
||||
function createMonitorRecorder(monitor) {
|
||||
@@ -980,6 +1036,7 @@ function buildMetricsSnapshot(monitor) {
|
||||
const inspectedResponseCount = monitor.inspected_response_count;
|
||||
return {
|
||||
started_at: monitor.started_at,
|
||||
persistent_since: monitor.persistent_since,
|
||||
total_proxy_request_count: monitor.total_proxy_request_count,
|
||||
inspected_response_count: inspectedResponseCount,
|
||||
matched_response_count: monitor.matched_response_count,
|
||||
@@ -2684,9 +2741,9 @@ async function main() {
|
||||
requestsDb,
|
||||
server: null,
|
||||
};
|
||||
await hydrateMonitorFromDisk(monitor, runtime.paths, config.request_history_limit);
|
||||
const importedCount = await importRequestsJsonlToDb(requestsDb, runtime.paths.requestsPath);
|
||||
logger(`[start] hydrated token totals from jsonl path=${runtime.paths.requestsPath}`);
|
||||
await hydrateMonitorFromDisk(monitor, runtime.paths, config.request_history_limit, requestsDb);
|
||||
logger(`[start] hydrated persistent request metrics from db path=${runtime.paths.requestsDbPath}`);
|
||||
logger(`[start] requests db ready path=${runtime.paths.requestsDbPath} imported_jsonl_rows=${importedCount}`);
|
||||
|
||||
const server = http.createServer(async (req, res) => {
|
||||
|
||||
@@ -299,8 +299,8 @@ async function run() {
|
||||
|
||||
await writeFile(configPath, JSON.stringify(config, null, 2), "utf8");
|
||||
|
||||
const upstream = await startFakeUpstream(upstreamPort);
|
||||
const gateway = startGateway(configPath, logPath);
|
||||
const upstream = await startFakeUpstream(upstreamPort);
|
||||
let gateway = startGateway(configPath, logPath);
|
||||
|
||||
try {
|
||||
try {
|
||||
@@ -427,6 +427,26 @@ async function run() {
|
||||
);
|
||||
assert(terminatedStream.status === 502, `/responses 上游半路断流未返回 502: ${terminatedStream.status}`);
|
||||
|
||||
const metricsBeforeRestartResponse = await fetch(`http://127.0.0.1:${gatewayPort}/__codex_retry_gateway/api/status`);
|
||||
const metricsBeforeRestart = await metricsBeforeRestartResponse.json();
|
||||
assert(metricsBeforeRestartResponse.status === 200, `status API 状态异常: ${metricsBeforeRestartResponse.status}`);
|
||||
assert(metricsBeforeRestart?.metrics?.reasoning_516_count >= 1, "重启前 reasoning_516_count 未累计");
|
||||
assert(metricsBeforeRestart?.metrics?.observed_reasoning_counts?.["128"] >= 1, "重启前 reasoning 128 未累计");
|
||||
assert(metricsBeforeRestart?.metrics?.total_proxy_request_count >= 1, "重启前 total_proxy_request_count 未累计");
|
||||
|
||||
gateway.child.kill();
|
||||
await once(gateway.child, "exit");
|
||||
gateway = startGateway(configPath, logPath);
|
||||
await waitForHealth(`http://127.0.0.1:${gatewayPort}${config.health_path}`);
|
||||
|
||||
const metricsAfterRestartResponse = await fetch(`http://127.0.0.1:${gatewayPort}/__codex_retry_gateway/api/status`);
|
||||
const metricsAfterRestart = await metricsAfterRestartResponse.json();
|
||||
assert(metricsAfterRestartResponse.status === 200, `重启后 status API 状态异常: ${metricsAfterRestartResponse.status}`);
|
||||
assert(metricsAfterRestart?.metrics?.reasoning_516_count >= metricsBeforeRestart?.metrics?.reasoning_516_count, "重启后 reasoning_516_count 未保留");
|
||||
assert(metricsAfterRestart?.metrics?.observed_reasoning_counts?.["128"] >= metricsBeforeRestart?.metrics?.observed_reasoning_counts?.["128"], "重启后 reasoning 128 计数未保留");
|
||||
assert(metricsAfterRestart?.metrics?.total_proxy_request_count >= metricsBeforeRestart?.metrics?.total_proxy_request_count, "重启后 total_proxy_request_count 未保留");
|
||||
assert(metricsAfterRestart?.metrics?.persistent_since, "重启后未返回 persistent_since");
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 120));
|
||||
const logText = await readFile(logPath, "utf8");
|
||||
assert(
|
||||
@@ -435,9 +455,9 @@ async function run() {
|
||||
);
|
||||
|
||||
process.stdout.write("PASS codex-retry-gateway e2e\n");
|
||||
} finally {
|
||||
gateway.child.kill();
|
||||
upstream.close();
|
||||
} finally {
|
||||
gateway.child.kill();
|
||||
upstream.close();
|
||||
await once(upstream, "close");
|
||||
await rm(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
+5
-3
@@ -23,6 +23,7 @@ type GatewayConfig = {
|
||||
|
||||
type Metrics = {
|
||||
started_at?: string;
|
||||
persistent_since?: string | null;
|
||||
total_proxy_request_count?: number;
|
||||
inspected_response_count?: number;
|
||||
matched_response_count?: number;
|
||||
@@ -230,7 +231,7 @@ type ProfileProbePayload = {
|
||||
const pageCopy: Record<PageKey, { title: string; subtitle: string; index: string }> = {
|
||||
overview: {
|
||||
title: "概览",
|
||||
subtitle: "当前 gateway 运行态和本次启动以来的累计统计。",
|
||||
subtitle: "当前 gateway 运行态，以及基于持久请求记录回灌的累计统计。",
|
||||
index: "01",
|
||||
},
|
||||
requests: {
|
||||
@@ -808,7 +809,7 @@ export default function App() {
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
<Card title="Token Totals" hint="按本次 gateway 启动以来累计；后续可以接持久化统计。">
|
||||
<Card title="Token Totals" hint="按持久请求记录累计；重启后会从 requests history 回灌。">
|
||||
<div className="token-strip">
|
||||
<TokenCard label="Input" value={numberFormat(effectiveInputTotal ?? 0)} />
|
||||
<TokenCard label="Output" value={numberFormat(tokens.output_tokens || 0)} />
|
||||
@@ -839,7 +840,8 @@ export default function App() {
|
||||
rows={[
|
||||
["516 命中", numberFormat(metrics.reasoning_516_count || 0)],
|
||||
["516 占比", percent(metrics.reasoning_516_ratio || 0)],
|
||||
["启动时间", timestamp(metrics.started_at)],
|
||||
["累计起点", timestamp(metrics.persistent_since || metrics.started_at)],
|
||||
["本次启动", timestamp(metrics.started_at)],
|
||||
["Config", status?.paths?.config_path || "-"],
|
||||
["备份", status?.state?.latest_backup_path || "-"],
|
||||
]}
|
||||
|
||||
Reference in New Issue
Block a user