feat: show per-event traffic errors
This commit is contained in:
+83
-7
@@ -4,6 +4,7 @@ import contextlib
|
||||
import copy
|
||||
import datetime as dt
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import importlib.util
|
||||
@@ -113,8 +114,8 @@ def workspace_payload_fixture() -> dict[str, object]:
|
||||
"errors": [
|
||||
{
|
||||
"instance": "server6",
|
||||
"latest_at": "2026-08-03T11:58:00Z",
|
||||
"error_count": 4,
|
||||
"id": "21",
|
||||
"created_at": "2026-08-03T11:58:00Z",
|
||||
"status_code": 502,
|
||||
"inbound_status_code": 500,
|
||||
"upstream_status_code": 502,
|
||||
@@ -762,6 +763,27 @@ class WorkspaceTests(unittest.TestCase):
|
||||
def test_workspace_adapters_preserve_ids_and_map_compact_traffic(self) -> None:
|
||||
mod = load_module()
|
||||
payload = workspace_payload_fixture()
|
||||
payload["traffic"]["requests"].append(
|
||||
{"instance": "server4", "id": "71", "created_at": "2026-08-03T11:57:00Z"}
|
||||
)
|
||||
payload["traffic"]["keys"].append(
|
||||
{"instance": "server4", "api_key_id": "71", "api_key_name": "server4-copy"}
|
||||
)
|
||||
payload["traffic"]["errors"].append(
|
||||
{
|
||||
"instance": "server4",
|
||||
"id": "22",
|
||||
"created_at": "2026-08-03T11:57:00Z",
|
||||
"status_code": 503,
|
||||
"api_key_id": "8",
|
||||
"account_id": "10",
|
||||
"error_type": "upstream",
|
||||
"error_source": "upstream_http",
|
||||
"error_detail": "forbidden-detail",
|
||||
"request_id": "forbidden-request-id",
|
||||
"user_email": "forbidden@example.test",
|
||||
}
|
||||
)
|
||||
|
||||
accounts = mod.workspace_accounts_payload(payload)
|
||||
status = mod.workspace_status_payload(payload)
|
||||
@@ -773,6 +795,7 @@ class WorkspaceTests(unittest.TestCase):
|
||||
self.assertEqual(accounts["accounts"][0]["id"], "9007199254740993")
|
||||
self.assertEqual(status["channel_monitors"]["items"][0]["id"], "9007199254740995")
|
||||
self.assertEqual(pricing["sources"][0]["name"], "code-plan")
|
||||
self.assertEqual(len(logs["data"]["items"]), 1)
|
||||
self.assertEqual(logs["data"]["items"][0]["id"], "9007199254740997")
|
||||
self.assertEqual(logs["data"]["items"][0]["instance"], "server6")
|
||||
normalized_accounts = mod.normalize_account_rows(accounts, pricing_payload=pricing)
|
||||
@@ -781,6 +804,7 @@ class WorkspaceTests(unittest.TestCase):
|
||||
self.assertEqual(normalized_logs[0]["id"], 9007199254740997)
|
||||
self.assertEqual(normalized_logs[0]["cost"], 0.25)
|
||||
self.assertEqual(mod.as_int("9007199254740999"), 9007199254740999)
|
||||
self.assertEqual(len(keys["items"]), 1)
|
||||
self.assertEqual(mod.normalize_key_rows(keys)[0]["name"], "wmy")
|
||||
self.assertEqual(mod.normalize_key_rows(keys)[0]["cost"], 0.25)
|
||||
self.assertEqual(keys["period_kind"], "calendar_day")
|
||||
@@ -788,7 +812,13 @@ class WorkspaceTests(unittest.TestCase):
|
||||
self.assertEqual(keys["timezone"], "Asia/Shanghai")
|
||||
self.assertIn(f"today {keys['date']} (Asia/Shanghai)", mod.usage_period_label(keys))
|
||||
self.assertEqual(errors["time_range"], f"today {keys['date']} (Asia/Shanghai)")
|
||||
self.assertEqual(errors["items"][0]["error_count"], 4)
|
||||
self.assertEqual({item["_node"] for item in errors["items"]}, {"server6", "server4"})
|
||||
self.assertNotIn("forbidden-detail", json.dumps(errors))
|
||||
self.assertNotIn("forbidden-request-id", json.dumps(errors))
|
||||
self.assertNotIn("forbidden@example.test", json.dumps(errors))
|
||||
self.assertEqual(errors["items"][0]["id"], "21")
|
||||
self.assertEqual(errors["items"][0]["created_at"], "2026-08-03T11:58:00Z")
|
||||
self.assertNotIn("error_count", errors["items"][0])
|
||||
self.assertEqual(errors["items"][0]["phase"], "upstream_http")
|
||||
self.assertEqual(errors["sources"]["server6"]["total"], 4)
|
||||
self.assertEqual(errors["sources"]["server4"]["total"], 0)
|
||||
@@ -805,8 +835,8 @@ class WorkspaceTests(unittest.TestCase):
|
||||
self.assertEqual(fallback_row["key"], "#9007199254740999")
|
||||
self.assertEqual(fallback_row["account"], "#9007199254740998")
|
||||
normalized_errors = mod.normalize_error_rows(errors)
|
||||
self.assertEqual(normalized_errors[0]["count"], 4)
|
||||
self.assertIn("count 4", mod.error_detail_line(normalized_errors[0]))
|
||||
self.assertNotIn("count", normalized_errors[0])
|
||||
self.assertNotIn("| count ", mod.error_detail_line(normalized_errors[0]))
|
||||
|
||||
def test_workspace_payload_validation_rejects_nonfinite_and_oversized_rows(self) -> None:
|
||||
mod = load_module()
|
||||
@@ -846,6 +876,18 @@ class WorkspaceTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(RuntimeError, "invalid workspace traffic period"):
|
||||
mod.fetch_workspace_payload("https://workspace.example.test/data", 3)
|
||||
|
||||
noncanonical_request = workspace_payload_fixture()
|
||||
noncanonical_request["traffic"]["requests"][0]["instance"] = "server4"
|
||||
with mock.patch.object(mod, "fetch_payload", return_value=noncanonical_request):
|
||||
with self.assertRaisesRegex(RuntimeError, "noncanonical workspace requests"):
|
||||
mod.fetch_workspace_payload("https://workspace.example.test/data", 3)
|
||||
|
||||
missing_key_provenance = workspace_payload_fixture()
|
||||
missing_key_provenance["traffic"]["keys"][0].pop("instance")
|
||||
with mock.patch.object(mod, "fetch_payload", return_value=missing_key_provenance):
|
||||
with self.assertRaisesRegex(RuntimeError, "noncanonical workspace keys"):
|
||||
mod.fetch_workspace_payload("https://workspace.example.test/data", 3)
|
||||
|
||||
def test_workspace_cache_coalesces_reads_and_returns_copies(self) -> None:
|
||||
mod = load_module()
|
||||
calls: list[str] = []
|
||||
@@ -1200,6 +1242,39 @@ class Sub2APILogsTests(unittest.TestCase):
|
||||
self.assertEqual([row["id"] for row in mod.normalize_error_rows(payload, "us")], [2])
|
||||
self.assertEqual(mod.normalize_error_rows(payload, "no-match"), [])
|
||||
|
||||
unsafe = {
|
||||
"items": [
|
||||
{
|
||||
"id": 3,
|
||||
"_node": "server6",
|
||||
"created_at": "2026-07-24T12:00:00+08:00",
|
||||
"status_code": 502,
|
||||
"api_key_name": "safe-key",
|
||||
"account_name": "safe-account",
|
||||
"error_type": "upstream",
|
||||
"error_source": "upstream_http",
|
||||
"error_detail": "private detail token=redacted-test-value",
|
||||
"message": "private message",
|
||||
"request_id": "private-request-id",
|
||||
"client_request_id": "private-client-request-id",
|
||||
"user_email": "private@example.test",
|
||||
}
|
||||
]
|
||||
}
|
||||
safe_row = mod.normalize_error_rows(unsafe)[0]
|
||||
serialized = json.dumps(safe_row)
|
||||
for private_value in (
|
||||
"redacted-test-value",
|
||||
"private message",
|
||||
"private-request-id",
|
||||
"private-client-request-id",
|
||||
"private@example.test",
|
||||
):
|
||||
self.assertNotIn(private_value, serialized)
|
||||
self.assertEqual(safe_row["message"], "")
|
||||
self.assertEqual(safe_row["request_id"], "")
|
||||
self.assertEqual(safe_row["user"], "-")
|
||||
|
||||
def test_fetch_merged_errors_labels_nodes_and_tolerates_partial_failure(self) -> None:
|
||||
mod = load_module()
|
||||
|
||||
@@ -1239,7 +1314,7 @@ class Sub2APILogsTests(unittest.TestCase):
|
||||
self.assertTrue(payload["sources"]["cn"]["ok"])
|
||||
self.assertEqual(payload["sources"]["cn"]["total"], 3)
|
||||
self.assertFalse(payload["sources"]["us"]["ok"])
|
||||
self.assertIn("us down", payload["sources"]["us"]["error"])
|
||||
self.assertEqual(payload["sources"]["us"]["error"], "unavailable")
|
||||
self.assertIn("cn 1/3", mod.errors_summary_line(payload, 1))
|
||||
self.assertIn("us err", mod.errors_summary_line(payload, 1))
|
||||
|
||||
@@ -1515,7 +1590,8 @@ class DashboardLayoutTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(type(screen).__name__, "DashboardScreen")
|
||||
self.assertEqual(app.sub_title, "Dashboard")
|
||||
self.assertEqual(screen.account_rows[0]["name"], "oai-quota-code-plan")
|
||||
self.assertEqual(screen.error_rows[0]["count"], 4)
|
||||
self.assertEqual(screen.error_rows[0]["key"], "wmy")
|
||||
self.assertNotIn("count", screen.error_rows[0])
|
||||
self.assertEqual(len(fetch.call_args_list), 1)
|
||||
|
||||
await pilot.press("p")
|
||||
|
||||
Reference in New Issue
Block a user