perf: cache the CLI catalog with validated startup fallback
Ports #6 with optional-field validation, conversion before cache writes, an upstream-specific cache directory, and startup regression tests. Co-authored-by: dualface <dualface@gmail.com>
This commit is contained in:
Kashyab Ambarani
co-authored by
dualface
dualface
parent
5d6090ae46
commit
e6b5247cc0
@@ -0,0 +1,65 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import {
|
||||
catalogCachePath,
|
||||
isUsableCatalog,
|
||||
isCatalogCacheFresh,
|
||||
readCatalogCache,
|
||||
writeCatalogCache,
|
||||
} from "../src/catalog-cache.ts";
|
||||
|
||||
const catalog = {
|
||||
families: [{
|
||||
family_label: "SWE-2",
|
||||
family_uid: "swe-2",
|
||||
slug: "swe-2",
|
||||
variants: [{ model_uid: "swe-2-high", label: "SWE-2 High" }],
|
||||
}],
|
||||
};
|
||||
|
||||
test("writes and reads a validated catalog atomically", (t) => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "pi-devin-cache-"));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
const path = join(directory, "nested", "models.json");
|
||||
writeCatalogCache(catalog, path, 1_000);
|
||||
assert.deepEqual(readCatalogCache(path), { catalog, fetchedAt: 1_000 });
|
||||
assert.equal(JSON.parse(readFileSync(path, "utf8")).version, 1);
|
||||
});
|
||||
|
||||
test("rejects malformed cache content", (t) => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "pi-devin-cache-"));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
const path = join(directory, "models.json");
|
||||
writeFileSync(path, JSON.stringify({ version: 1, fetchedAt: 1_000, catalog: { families: [{}] } }));
|
||||
assert.throws(() => readCatalogCache(path), /Invalid Devin model catalog cache/);
|
||||
});
|
||||
|
||||
test("rejects catalogs without usable model families", () => {
|
||||
assert.equal(isUsableCatalog({ families: [] }), false);
|
||||
assert.equal(isUsableCatalog({
|
||||
families: [{ family_label: "Empty", family_uid: "empty", slug: "empty", variants: [] }],
|
||||
}), false);
|
||||
});
|
||||
|
||||
test("checks cache age and future timestamps", () => {
|
||||
const cached = { catalog, fetchedAt: 1_000 };
|
||||
assert.equal(isCatalogCacheFresh(cached, 1_500, 500), true);
|
||||
assert.equal(isCatalogCacheFresh(cached, 1_501, 500), false);
|
||||
assert.equal(isCatalogCacheFresh(cached, 999, 500), false);
|
||||
});
|
||||
|
||||
test("uses XDG cache directory when configured", () => {
|
||||
assert.equal(catalogCachePath({ XDG_CACHE_HOME: "/cache" }, "/home/test"), "/cache/pi-devin/models.json");
|
||||
assert.equal(catalogCachePath({}, "/home/test"), "/home/test/.cache/pi-devin/models.json");
|
||||
});
|
||||
|
||||
for (const [field, value] of Object.entries({ cost_summary: {}, max_context_tokens: "1000", max_output_tokens: -1, model_uid: "", is_beta: "yes" })) {
|
||||
test(`rejects invalid ${field} before model conversion`, () => {
|
||||
const malformed = structuredClone(catalog);
|
||||
malformed.families[0].variants[0][field] = value;
|
||||
assert.equal(isUsableCatalog(malformed), false);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test, { after } from "node:test";
|
||||
import fs from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
import { catalogCachePath, writeCatalogCache } from "../src/catalog-cache.ts";
|
||||
|
||||
const directory = fs.mkdtempSync(join(tmpdir(), "pi-devin-catalog-test-"));
|
||||
after(() => fs.rmSync(directory, { recursive: true, force: true }));
|
||||
const cli = join(directory, "devin");
|
||||
const counter = join(directory, "calls");
|
||||
const catalog = { families: [{ family_label: "Test", family_uid: "test", slug: "test", variants: [{ model_uid: "test-high", label: "Test high", cost_summary: "$1/MTok In, $2/MTok Out" }] }] };
|
||||
fs.writeFileSync(cli, `#!${process.execPath}\nconst fs = require('node:fs');\nfs.appendFileSync(${JSON.stringify(counter)}, 'call\\n');\nsetTimeout(() => { console.log(process.env.TEST_CATALOG); process.exit(Number(process.env.TEST_CLI_EXIT || 0)); }, 30);\n`, { mode: 0o700 });
|
||||
const originalEnv = { ...process.env };
|
||||
process.env.DEVIN_CLI = cli;
|
||||
const { default: extension } = await import("../extensions/index.ts");
|
||||
const { clearDevinBinCache } = await import("../src/cli.ts");
|
||||
after(() => { if (originalEnv.DEVIN_CLI === undefined) delete process.env.DEVIN_CLI; else process.env.DEVIN_CLI = originalEnv.DEVIN_CLI; });
|
||||
|
||||
function setup(t) {
|
||||
clearDevinBinCache();
|
||||
const root = fs.mkdtempSync(join(directory, "cache-"));
|
||||
const previous = { ...process.env };
|
||||
process.env.XDG_CACHE_HOME = root;
|
||||
process.env.TEST_CATALOG = JSON.stringify(catalog);
|
||||
delete process.env.PI_OFFLINE;
|
||||
delete process.env.TEST_CLI_EXIT;
|
||||
fs.writeFileSync(counter, "");
|
||||
const read = fs.readFileSync, exists = fs.existsSync;
|
||||
const isCredentials = (path) => String(path).endsWith("devin/credentials.toml");
|
||||
t.mock.method(fs, "readFileSync", (path, ...args) => isCredentials(path) ? 'api_key = "synthetic-test-key"\n' : read(path, ...args));
|
||||
t.mock.method(fs, "existsSync", (path) => isCredentials(path) || exists(path));
|
||||
t.mock.method(console, "warn", () => {});
|
||||
syncBuiltinESMExports();
|
||||
const registrations = [], hooks = {}, commands = {};
|
||||
const pi = { registerProvider: (_id, config) => registrations.push(config), on: (name, fn) => hooks[name] = fn, registerCommand: (name, command) => commands[name] = command };
|
||||
t.after(async () => {
|
||||
await hooks.session_shutdown?.();
|
||||
t.mock.restoreAll(); syncBuiltinESMExports();
|
||||
for (const key of ["XDG_CACHE_HOME", "PI_OFFLINE", "TEST_CATALOG", "TEST_CLI_EXIT"]) {
|
||||
if (previous[key] === undefined) delete process.env[key]; else process.env[key] = previous[key];
|
||||
}
|
||||
});
|
||||
return { registrations, hooks, commands, pi, path: catalogCachePath(), calls: () => fs.readFileSync(counter, "utf8").split("\n").filter(Boolean).length };
|
||||
}
|
||||
|
||||
test("fresh cache registers once without invoking the CLI", async (t) => {
|
||||
const x = setup(t); writeCatalogCache(catalog);
|
||||
await extension(x.pi);
|
||||
assert.equal(x.calls(), 0); assert.equal(x.registrations.length, 1);
|
||||
assert.equal(x.registrations[0].models[0].id, "test-high");
|
||||
});
|
||||
|
||||
test("malformed cached cost data cannot prevent provider and command registration", async (t) => {
|
||||
const x = setup(t); process.env.PI_OFFLINE = "1";
|
||||
const malformed = structuredClone(catalog); malformed.families[0].variants[0].cost_summary = {};
|
||||
fs.mkdirSync(join(process.env.XDG_CACHE_HOME, "pi-devin"));
|
||||
fs.writeFileSync(x.path, JSON.stringify({ version: 1, fetchedAt: Date.now(), catalog: malformed }));
|
||||
await extension(x.pi);
|
||||
assert.equal(x.registrations.length, 1); assert.ok(x.registrations[0].models.length);
|
||||
assert.ok(x.commands["devin-refresh"]); assert.equal(x.calls(), 0);
|
||||
});
|
||||
|
||||
test("cold startup loads and caches the CLI catalog", async (t) => {
|
||||
const x = setup(t); await extension(x.pi);
|
||||
assert.equal(x.calls(), 1); assert.equal(x.registrations.at(-1).models[0].id, "test-high");
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(x.path, "utf8")).catalog, catalog);
|
||||
});
|
||||
|
||||
test("stale cache shares its background request with manual refresh", async (t) => {
|
||||
const x = setup(t); writeCatalogCache(catalog, x.path, 1);
|
||||
await extension(x.pi);
|
||||
const notifications = [];
|
||||
await x.commands["devin-refresh"].handler("", { ui: { notify: (...args) => notifications.push(args) } });
|
||||
assert.equal(x.calls(), 1); assert.equal(notifications.at(-1)[1], "info");
|
||||
assert.equal(x.registrations.at(-1).models[0].id, "test-high");
|
||||
});
|
||||
|
||||
test("failed refresh retains cached models and does not overwrite the cache", async (t) => {
|
||||
const x = setup(t); writeCatalogCache(catalog, x.path, 1); process.env.TEST_CATALOG = '{"families":[{}]}';
|
||||
await extension(x.pi);
|
||||
const notifications = [];
|
||||
await x.commands["devin-refresh"].handler("", { ui: { notify: (...args) => notifications.push(args) } });
|
||||
assert.equal(notifications.at(-1)[1], "error"); assert.equal(x.registrations.length, 1);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(x.path, "utf8")).catalog, catalog);
|
||||
});
|
||||
|
||||
test("offline cold startup uses fallback without running the CLI", async (t) => {
|
||||
const x = setup(t); process.env.PI_OFFLINE = "true";
|
||||
await extension(x.pi); assert.equal(x.calls(), 0); assert.ok(x.registrations[0].models.length);
|
||||
});
|
||||
|
||||
test("shutdown prevents a late background result from registering a provider", async (t) => {
|
||||
const x = setup(t); writeCatalogCache(catalog, x.path, 1);
|
||||
await extension(x.pi); await x.hooks.session_shutdown();
|
||||
await x.commands["devin-refresh"].handler("", { ui: { notify() {} } });
|
||||
assert.equal(x.registrations.length, 1);
|
||||
});
|
||||
Reference in New Issue
Block a user