feat: show per-event traffic errors

This commit is contained in:
2026-08-04 02:25:33 +08:00
parent d9a9fbaaad
commit 1db6b20dcc
5 changed files with 198 additions and 90 deletions
+83 -7
View File
@@ -4,6 +4,7 @@ import contextlib
import copy
import datetime as dt
import io
import json
import os
from pathlib import Path
import importlib.util
@@ -113,8 +114,8 @@ def workspace_payload_fixture() -> dict[str, object]:
"errors": [
{
"instance": "server6",
"latest_at": "2026-08-03T11:58:00Z",
"error_count": 4,
"id": "21",
"created_at": "2026-08-03T11:58:00Z",
"status_code": 502,
"inbound_status_code": 500,
"upstream_status_code": 502,
@@ -762,6 +763,27 @@ class WorkspaceTests(unittest.TestCase):
def test_workspace_adapters_preserve_ids_and_map_compact_traffic(self) -> None:
mod = load_module()
payload = workspace_payload_fixture()
payload["traffic"]["requests"].append(
{"instance": "server4", "id": "71", "created_at": "2026-08-03T11:57:00Z"}
)
payload["traffic"]["keys"].append(
{"instance": "server4", "api_key_id": "71", "api_key_name": "server4-copy"}
)
payload["traffic"]["errors"].append(
{
"instance": "server4",
"id": "22",
"created_at": "2026-08-03T11:57:00Z",
"status_code": 503,
"api_key_id": "8",
"account_id": "10",
"error_type": "upstream",
"error_source": "upstream_http",
"error_detail": "forbidden-detail",
"request_id": "forbidden-request-id",
"user_email": "forbidden@example.test",
}
)
accounts = mod.workspace_accounts_payload(payload)
status = mod.workspace_status_payload(payload)
@@ -773,6 +795,7 @@ class WorkspaceTests(unittest.TestCase):
self.assertEqual(accounts["accounts"][0]["id"], "9007199254740993")
self.assertEqual(status["channel_monitors"]["items"][0]["id"], "9007199254740995")
self.assertEqual(pricing["sources"][0]["name"], "code-plan")
self.assertEqual(len(logs["data"]["items"]), 1)
self.assertEqual(logs["data"]["items"][0]["id"], "9007199254740997")
self.assertEqual(logs["data"]["items"][0]["instance"], "server6")
normalized_accounts = mod.normalize_account_rows(accounts, pricing_payload=pricing)
@@ -781,6 +804,7 @@ class WorkspaceTests(unittest.TestCase):
self.assertEqual(normalized_logs[0]["id"], 9007199254740997)
self.assertEqual(normalized_logs[0]["cost"], 0.25)
self.assertEqual(mod.as_int("9007199254740999"), 9007199254740999)
self.assertEqual(len(keys["items"]), 1)
self.assertEqual(mod.normalize_key_rows(keys)[0]["name"], "wmy")
self.assertEqual(mod.normalize_key_rows(keys)[0]["cost"], 0.25)
self.assertEqual(keys["period_kind"], "calendar_day")
@@ -788,7 +812,13 @@ class WorkspaceTests(unittest.TestCase):
self.assertEqual(keys["timezone"], "Asia/Shanghai")
self.assertIn(f"today {keys['date']} (Asia/Shanghai)", mod.usage_period_label(keys))
self.assertEqual(errors["time_range"], f"today {keys['date']} (Asia/Shanghai)")
self.assertEqual(errors["items"][0]["error_count"], 4)
self.assertEqual({item["_node"] for item in errors["items"]}, {"server6", "server4"})
self.assertNotIn("forbidden-detail", json.dumps(errors))
self.assertNotIn("forbidden-request-id", json.dumps(errors))
self.assertNotIn("forbidden@example.test", json.dumps(errors))
self.assertEqual(errors["items"][0]["id"], "21")
self.assertEqual(errors["items"][0]["created_at"], "2026-08-03T11:58:00Z")
self.assertNotIn("error_count", errors["items"][0])
self.assertEqual(errors["items"][0]["phase"], "upstream_http")
self.assertEqual(errors["sources"]["server6"]["total"], 4)
self.assertEqual(errors["sources"]["server4"]["total"], 0)
@@ -805,8 +835,8 @@ class WorkspaceTests(unittest.TestCase):
self.assertEqual(fallback_row["key"], "#9007199254740999")
self.assertEqual(fallback_row["account"], "#9007199254740998")
normalized_errors = mod.normalize_error_rows(errors)
self.assertEqual(normalized_errors[0]["count"], 4)
self.assertIn("count 4", mod.error_detail_line(normalized_errors[0]))
self.assertNotIn("count", normalized_errors[0])
self.assertNotIn("| count ", mod.error_detail_line(normalized_errors[0]))
def test_workspace_payload_validation_rejects_nonfinite_and_oversized_rows(self) -> None:
mod = load_module()
@@ -846,6 +876,18 @@ class WorkspaceTests(unittest.TestCase):
with self.assertRaisesRegex(RuntimeError, "invalid workspace traffic period"):
mod.fetch_workspace_payload("https://workspace.example.test/data", 3)
noncanonical_request = workspace_payload_fixture()
noncanonical_request["traffic"]["requests"][0]["instance"] = "server4"
with mock.patch.object(mod, "fetch_payload", return_value=noncanonical_request):
with self.assertRaisesRegex(RuntimeError, "noncanonical workspace requests"):
mod.fetch_workspace_payload("https://workspace.example.test/data", 3)
missing_key_provenance = workspace_payload_fixture()
missing_key_provenance["traffic"]["keys"][0].pop("instance")
with mock.patch.object(mod, "fetch_payload", return_value=missing_key_provenance):
with self.assertRaisesRegex(RuntimeError, "noncanonical workspace keys"):
mod.fetch_workspace_payload("https://workspace.example.test/data", 3)
def test_workspace_cache_coalesces_reads_and_returns_copies(self) -> None:
mod = load_module()
calls: list[str] = []
@@ -1200,6 +1242,39 @@ class Sub2APILogsTests(unittest.TestCase):
self.assertEqual([row["id"] for row in mod.normalize_error_rows(payload, "us")], [2])
self.assertEqual(mod.normalize_error_rows(payload, "no-match"), [])
unsafe = {
"items": [
{
"id": 3,
"_node": "server6",
"created_at": "2026-07-24T12:00:00+08:00",
"status_code": 502,
"api_key_name": "safe-key",
"account_name": "safe-account",
"error_type": "upstream",
"error_source": "upstream_http",
"error_detail": "private detail token=redacted-test-value",
"message": "private message",
"request_id": "private-request-id",
"client_request_id": "private-client-request-id",
"user_email": "private@example.test",
}
]
}
safe_row = mod.normalize_error_rows(unsafe)[0]
serialized = json.dumps(safe_row)
for private_value in (
"redacted-test-value",
"private message",
"private-request-id",
"private-client-request-id",
"private@example.test",
):
self.assertNotIn(private_value, serialized)
self.assertEqual(safe_row["message"], "")
self.assertEqual(safe_row["request_id"], "")
self.assertEqual(safe_row["user"], "-")
def test_fetch_merged_errors_labels_nodes_and_tolerates_partial_failure(self) -> None:
mod = load_module()
@@ -1239,7 +1314,7 @@ class Sub2APILogsTests(unittest.TestCase):
self.assertTrue(payload["sources"]["cn"]["ok"])
self.assertEqual(payload["sources"]["cn"]["total"], 3)
self.assertFalse(payload["sources"]["us"]["ok"])
self.assertIn("us down", payload["sources"]["us"]["error"])
self.assertEqual(payload["sources"]["us"]["error"], "unavailable")
self.assertIn("cn 1/3", mod.errors_summary_line(payload, 1))
self.assertIn("us err", mod.errors_summary_line(payload, 1))
@@ -1515,7 +1590,8 @@ class DashboardLayoutTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(type(screen).__name__, "DashboardScreen")
self.assertEqual(app.sub_title, "Dashboard")
self.assertEqual(screen.account_rows[0]["name"], "oai-quota-code-plan")
self.assertEqual(screen.error_rows[0]["count"], 4)
self.assertEqual(screen.error_rows[0]["key"], "wmy")
self.assertNotIn("count", screen.error_rows[0])
self.assertEqual(len(fetch.call_args_list), 1)
await pilot.press("p")