From e6b5247cc07e3be13be4c3f3499b42aaec94501c Mon Sep 17 00:00:00 2001 From: Kashyab Ambarani Date: Fri, 25 Sep 2026 03:41:59 -0700 Subject: [PATCH] perf: cache the CLI catalog with validated startup fallback Ports #6 with optional-field validation, conversion before cache writes, an upstream-specific cache directory, and startup regression tests. Co-authored-by: dualface --- README.md | 2 + extensions/index.ts | 81 +++++++++++++++++++------- src/catalog-cache.ts | 104 ++++++++++++++++++++++++++++++++++ test/catalog-cache.test.mjs | 65 +++++++++++++++++++++ test/catalog-startup.test.mjs | 99 ++++++++++++++++++++++++++++++++ 5 files changed, 331 insertions(+), 20 deletions(-) create mode 100644 src/catalog-cache.ts create mode 100644 test/catalog-cache.test.mjs create mode 100644 test/catalog-startup.test.mjs diff --git a/README.md b/README.md index 00c7781..f6616df 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ Commands: - `/devin-status` — CLI path, version, auth - `/devin-refresh` — reload `devin models list --format json` +The model catalog is cached for six hours in `$XDG_CACHE_HOME/pi-devin/models.json` (or `~/.cache/pi-devin/models.json`). A fresh cache avoids the CLI call at startup; an older cache remains available while it refreshes in the background. Set `PI_OFFLINE=1` to skip automatic catalog refreshes. `/devin-refresh` still requests a refresh explicitly. + ## What this is / is not | This package | Not this package | diff --git a/extensions/index.ts b/extensions/index.ts index ac6ae16..ae57f81 100644 --- a/extensions/index.ts +++ b/extensions/index.ts @@ -2,6 +2,13 @@ import type { ExtensionAPI, ProviderModelConfig } from "@earendil-works/pi-codin import type { Api, Model, OAuthCredentials, OAuthLoginCallbacks } from "@earendil-works/pi-ai"; import { authStatus, loginWithCli, readCredentials } from "../src/credentials.js"; import { whichDevin, devinVersion } from "../src/cli.js"; +import { + type CachedDevinCatalog, + isUsableCatalog, + isCatalogCacheFresh, + readCatalogCache, + writeCatalogCache, +} from "../src/catalog-cache.js"; import { FALLBACK_MODELS, loadCliCatalog, modelsFromCatalog } from "../src/models.js"; import { CLIENT_IDE, CLIENT_VERSION } from "../src/metadata.js"; import { streamDevin } from "../src/stream.js"; @@ -10,6 +17,37 @@ const PROVIDER_ID = "devin"; const PLACEHOLDER_BASE_URL = "https://server.codeium.com"; let _pi: ExtensionAPI | null = null; +let catalogRequest: Promise | null = null; + +function isOffline(): boolean { + const value = process.env.PI_OFFLINE?.toLowerCase(); + return value === "1" || value === "true" || value === "yes"; +} + +function refreshCatalog(pi: ExtensionAPI): Promise { + if (!catalogRequest) { + const pending = loadCliCatalog().then((catalog) => { + if (!isUsableCatalog(catalog)) { + throw new Error("Devin CLI returned no usable model families"); + } + const models = modelsFromCatalog(catalog); + try { + writeCatalogCache(catalog); + } catch (error) { + console.warn(`Devin: failed to cache model catalog: ${error instanceof Error ? error.message : String(error)}`); + } + return models; + }); + catalogRequest = pending; + void pending.finally(() => { + if (catalogRequest === pending) catalogRequest = null; + }).catch(() => {}); + } + return catalogRequest.then((models) => { + if (_pi === pi) registerDevinProvider(pi, models); + return models; + }); +} function registerDevinProvider(pi: ExtensionAPI, models: ProviderModelConfig[]): void { pi.registerProvider(PROVIDER_ID, { @@ -23,8 +61,7 @@ function registerDevinProvider(pi: ExtensionAPI, models: ProviderModelConfig[]): const creds = await loginWithCli(); if (_pi) { try { - const catalog = await loadCliCatalog(); - registerDevinProvider(_pi, modelsFromCatalog(catalog)); + await refreshCatalog(_pi); } catch { // keep current models } @@ -57,26 +94,32 @@ function registerDevinProvider(pi: ExtensionAPI, models: ProviderModelConfig[]): export default async function (pi: ExtensionAPI): Promise { _pi = pi; - registerDevinProvider(pi, FALLBACK_MODELS); - + let cached: CachedDevinCatalog | null | undefined; + let initialModels = FALLBACK_MODELS; try { - if (readCredentials()) { - const catalog = await loadCliCatalog(); - registerDevinProvider(pi, modelsFromCatalog(catalog)); - } - } catch { - // fallback models already registered + cached = readCatalogCache(); + if (cached) initialModels = modelsFromCatalog(cached.catalog); + } catch (error) { + cached = null; + console.warn(`Devin: failed to read model catalog cache: ${error instanceof Error ? error.message : String(error)}`); } + registerDevinProvider(pi, initialModels); - pi.on("session_start", async () => { + if (!isOffline()) { try { - if (!_pi || !readCredentials()) return; - const catalog = await loadCliCatalog(); - registerDevinProvider(_pi, modelsFromCatalog(catalog)); + if (readCredentials()) { + if (!cached) { + await refreshCatalog(pi); + } else if (!isCatalogCacheFresh(cached)) { + void refreshCatalog(pi).catch((error) => { + console.warn(`Devin: background model catalog refresh failed: ${error instanceof Error ? error.message : String(error)}`); + }); + } + } } catch { - // keep current models + // Cached or fallback models stay registered when credential discovery or a cold refresh fails. } - }); + } pi.registerCommand("devin-status", { description: "Show Devin CLI auth + binary status", @@ -100,9 +143,7 @@ export default async function (pi: ExtensionAPI): Promise { description: "Refresh Devin Local model catalog from `devin models list`", handler: async (_args, ctx) => { try { - const catalog = await loadCliCatalog(); - const models = modelsFromCatalog(catalog); - registerDevinProvider(pi, models); + const models = await refreshCatalog(pi); ctx.ui.notify(`Devin: loaded ${models.length} families from the local CLI.`, "info"); } catch (error) { ctx.ui.notify( @@ -114,6 +155,6 @@ export default async function (pi: ExtensionAPI): Promise { }); pi.on("session_shutdown", async () => { - _pi = null; + if (_pi === pi) _pi = null; }); } diff --git a/src/catalog-cache.ts b/src/catalog-cache.ts new file mode 100644 index 0000000..2d3fd72 --- /dev/null +++ b/src/catalog-cache.ts @@ -0,0 +1,104 @@ +import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; +import type { DevinCatalog } from "./models.js"; + +const CACHE_VERSION = 1; + +export const CATALOG_CACHE_MAX_AGE_MS = 6 * 60 * 60 * 1000; + +export interface CachedDevinCatalog { + catalog: DevinCatalog; + fetchedAt: number; +} + +interface CatalogCacheFile extends CachedDevinCatalog { + version: typeof CACHE_VERSION; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function optionalString(value: unknown): boolean { + return value === undefined || typeof value === "string"; +} + +function optionalTokenLimit(value: unknown): boolean { + return value === undefined || (typeof value === "number" && Number.isSafeInteger(value) && value > 0); +} + +export function isUsableCatalog(value: unknown): value is DevinCatalog { + if (!isRecord(value) || !Array.isArray(value.families) || value.families.length === 0) return false; + return value.families.some((family) => isRecord(family) && Array.isArray(family.variants) && family.variants.length > 0) + && value.families.every((family) => + isRecord(family) + && typeof family.family_label === "string" + && typeof family.family_uid === "string" + && typeof family.slug === "string" + && (family.aliases === undefined || (Array.isArray(family.aliases) && family.aliases.every((alias) => typeof alias === "string"))) + && Array.isArray(family.variants) + && family.variants.every((variant) => + isRecord(variant) + && typeof variant.model_uid === "string" + && variant.model_uid.trim().length > 0 + && typeof variant.label === "string" + && optionalString(variant.cost_summary) + && optionalString(variant.cost_tier) + && optionalTokenLimit(variant.max_context_tokens) + && optionalTokenLimit(variant.max_output_tokens) + && (variant.is_new === undefined || typeof variant.is_new === "boolean") + && (variant.is_beta === undefined || typeof variant.is_beta === "boolean") + ) + ); +} + +export function catalogCachePath( + env: NodeJS.ProcessEnv = process.env, + home: string = homedir(), +): string { + const cacheRoot = env.XDG_CACHE_HOME || join(home, ".cache"); + return join(cacheRoot, "pi-devin", "models.json"); +} + +export function readCatalogCache(path: string = catalogCachePath()): CachedDevinCatalog | null { + if (!existsSync(path)) return null; + const parsed: unknown = JSON.parse(readFileSync(path, "utf8")); + if ( + !isRecord(parsed) + || parsed.version !== CACHE_VERSION + || typeof parsed.fetchedAt !== "number" + || !Number.isFinite(parsed.fetchedAt) + || !isUsableCatalog(parsed.catalog) + ) { + throw new Error(`Invalid Devin model catalog cache: ${path}`); + } + return { catalog: parsed.catalog, fetchedAt: parsed.fetchedAt }; +} + +export function writeCatalogCache( + catalog: DevinCatalog, + path: string = catalogCachePath(), + fetchedAt: number = Date.now(), +): void { + if (!isUsableCatalog(catalog)) throw new Error("Refusing to cache an invalid Devin model catalog"); + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const temporaryPath = `${path}.${randomUUID()}.tmp`; + const body: CatalogCacheFile = { version: CACHE_VERSION, fetchedAt, catalog }; + try { + writeFileSync(temporaryPath, `${JSON.stringify(body)}\n`, { mode: 0o600 }); + renameSync(temporaryPath, path); + } finally { + if (existsSync(temporaryPath)) unlinkSync(temporaryPath); + } +} + +export function isCatalogCacheFresh( + cached: CachedDevinCatalog, + now: number = Date.now(), + maxAgeMs: number = CATALOG_CACHE_MAX_AGE_MS, +): boolean { + const age = now - cached.fetchedAt; + return age >= 0 && age <= maxAgeMs; +} diff --git a/test/catalog-cache.test.mjs b/test/catalog-cache.test.mjs new file mode 100644 index 0000000..4669244 --- /dev/null +++ b/test/catalog-cache.test.mjs @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { + catalogCachePath, + isUsableCatalog, + isCatalogCacheFresh, + readCatalogCache, + writeCatalogCache, +} from "../src/catalog-cache.ts"; + +const catalog = { + families: [{ + family_label: "SWE-2", + family_uid: "swe-2", + slug: "swe-2", + variants: [{ model_uid: "swe-2-high", label: "SWE-2 High" }], + }], +}; + +test("writes and reads a validated catalog atomically", (t) => { + const directory = mkdtempSync(join(tmpdir(), "pi-devin-cache-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const path = join(directory, "nested", "models.json"); + writeCatalogCache(catalog, path, 1_000); + assert.deepEqual(readCatalogCache(path), { catalog, fetchedAt: 1_000 }); + assert.equal(JSON.parse(readFileSync(path, "utf8")).version, 1); +}); + +test("rejects malformed cache content", (t) => { + const directory = mkdtempSync(join(tmpdir(), "pi-devin-cache-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const path = join(directory, "models.json"); + writeFileSync(path, JSON.stringify({ version: 1, fetchedAt: 1_000, catalog: { families: [{}] } })); + assert.throws(() => readCatalogCache(path), /Invalid Devin model catalog cache/); +}); + +test("rejects catalogs without usable model families", () => { + assert.equal(isUsableCatalog({ families: [] }), false); + assert.equal(isUsableCatalog({ + families: [{ family_label: "Empty", family_uid: "empty", slug: "empty", variants: [] }], + }), false); +}); + +test("checks cache age and future timestamps", () => { + const cached = { catalog, fetchedAt: 1_000 }; + assert.equal(isCatalogCacheFresh(cached, 1_500, 500), true); + assert.equal(isCatalogCacheFresh(cached, 1_501, 500), false); + assert.equal(isCatalogCacheFresh(cached, 999, 500), false); +}); + +test("uses XDG cache directory when configured", () => { + assert.equal(catalogCachePath({ XDG_CACHE_HOME: "/cache" }, "/home/test"), "/cache/pi-devin/models.json"); + assert.equal(catalogCachePath({}, "/home/test"), "/home/test/.cache/pi-devin/models.json"); +}); + +for (const [field, value] of Object.entries({ cost_summary: {}, max_context_tokens: "1000", max_output_tokens: -1, model_uid: "", is_beta: "yes" })) { + test(`rejects invalid ${field} before model conversion`, () => { + const malformed = structuredClone(catalog); + malformed.families[0].variants[0][field] = value; + assert.equal(isUsableCatalog(malformed), false); + }); +} diff --git a/test/catalog-startup.test.mjs b/test/catalog-startup.test.mjs new file mode 100644 index 0000000..2a4f8f4 --- /dev/null +++ b/test/catalog-startup.test.mjs @@ -0,0 +1,99 @@ +import assert from "node:assert/strict"; +import test, { after } from "node:test"; +import fs from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { syncBuiltinESMExports } from "node:module"; +import { catalogCachePath, writeCatalogCache } from "../src/catalog-cache.ts"; + +const directory = fs.mkdtempSync(join(tmpdir(), "pi-devin-catalog-test-")); +after(() => fs.rmSync(directory, { recursive: true, force: true })); +const cli = join(directory, "devin"); +const counter = join(directory, "calls"); +const catalog = { families: [{ family_label: "Test", family_uid: "test", slug: "test", variants: [{ model_uid: "test-high", label: "Test high", cost_summary: "$1/MTok In, $2/MTok Out" }] }] }; +fs.writeFileSync(cli, `#!${process.execPath}\nconst fs = require('node:fs');\nfs.appendFileSync(${JSON.stringify(counter)}, 'call\\n');\nsetTimeout(() => { console.log(process.env.TEST_CATALOG); process.exit(Number(process.env.TEST_CLI_EXIT || 0)); }, 30);\n`, { mode: 0o700 }); +const originalEnv = { ...process.env }; +process.env.DEVIN_CLI = cli; +const { default: extension } = await import("../extensions/index.ts"); +const { clearDevinBinCache } = await import("../src/cli.ts"); +after(() => { if (originalEnv.DEVIN_CLI === undefined) delete process.env.DEVIN_CLI; else process.env.DEVIN_CLI = originalEnv.DEVIN_CLI; }); + +function setup(t) { + clearDevinBinCache(); + const root = fs.mkdtempSync(join(directory, "cache-")); + const previous = { ...process.env }; + process.env.XDG_CACHE_HOME = root; + process.env.TEST_CATALOG = JSON.stringify(catalog); + delete process.env.PI_OFFLINE; + delete process.env.TEST_CLI_EXIT; + fs.writeFileSync(counter, ""); + const read = fs.readFileSync, exists = fs.existsSync; + const isCredentials = (path) => String(path).endsWith("devin/credentials.toml"); + t.mock.method(fs, "readFileSync", (path, ...args) => isCredentials(path) ? 'api_key = "synthetic-test-key"\n' : read(path, ...args)); + t.mock.method(fs, "existsSync", (path) => isCredentials(path) || exists(path)); + t.mock.method(console, "warn", () => {}); + syncBuiltinESMExports(); + const registrations = [], hooks = {}, commands = {}; + const pi = { registerProvider: (_id, config) => registrations.push(config), on: (name, fn) => hooks[name] = fn, registerCommand: (name, command) => commands[name] = command }; + t.after(async () => { + await hooks.session_shutdown?.(); + t.mock.restoreAll(); syncBuiltinESMExports(); + for (const key of ["XDG_CACHE_HOME", "PI_OFFLINE", "TEST_CATALOG", "TEST_CLI_EXIT"]) { + if (previous[key] === undefined) delete process.env[key]; else process.env[key] = previous[key]; + } + }); + return { registrations, hooks, commands, pi, path: catalogCachePath(), calls: () => fs.readFileSync(counter, "utf8").split("\n").filter(Boolean).length }; +} + +test("fresh cache registers once without invoking the CLI", async (t) => { + const x = setup(t); writeCatalogCache(catalog); + await extension(x.pi); + assert.equal(x.calls(), 0); assert.equal(x.registrations.length, 1); + assert.equal(x.registrations[0].models[0].id, "test-high"); +}); + +test("malformed cached cost data cannot prevent provider and command registration", async (t) => { + const x = setup(t); process.env.PI_OFFLINE = "1"; + const malformed = structuredClone(catalog); malformed.families[0].variants[0].cost_summary = {}; + fs.mkdirSync(join(process.env.XDG_CACHE_HOME, "pi-devin")); + fs.writeFileSync(x.path, JSON.stringify({ version: 1, fetchedAt: Date.now(), catalog: malformed })); + await extension(x.pi); + assert.equal(x.registrations.length, 1); assert.ok(x.registrations[0].models.length); + assert.ok(x.commands["devin-refresh"]); assert.equal(x.calls(), 0); +}); + +test("cold startup loads and caches the CLI catalog", async (t) => { + const x = setup(t); await extension(x.pi); + assert.equal(x.calls(), 1); assert.equal(x.registrations.at(-1).models[0].id, "test-high"); + assert.deepEqual(JSON.parse(fs.readFileSync(x.path, "utf8")).catalog, catalog); +}); + +test("stale cache shares its background request with manual refresh", async (t) => { + const x = setup(t); writeCatalogCache(catalog, x.path, 1); + await extension(x.pi); + const notifications = []; + await x.commands["devin-refresh"].handler("", { ui: { notify: (...args) => notifications.push(args) } }); + assert.equal(x.calls(), 1); assert.equal(notifications.at(-1)[1], "info"); + assert.equal(x.registrations.at(-1).models[0].id, "test-high"); +}); + +test("failed refresh retains cached models and does not overwrite the cache", async (t) => { + const x = setup(t); writeCatalogCache(catalog, x.path, 1); process.env.TEST_CATALOG = '{"families":[{}]}'; + await extension(x.pi); + const notifications = []; + await x.commands["devin-refresh"].handler("", { ui: { notify: (...args) => notifications.push(args) } }); + assert.equal(notifications.at(-1)[1], "error"); assert.equal(x.registrations.length, 1); + assert.deepEqual(JSON.parse(fs.readFileSync(x.path, "utf8")).catalog, catalog); +}); + +test("offline cold startup uses fallback without running the CLI", async (t) => { + const x = setup(t); process.env.PI_OFFLINE = "true"; + await extension(x.pi); assert.equal(x.calls(), 0); assert.ok(x.registrations[0].models.length); +}); + +test("shutdown prevents a late background result from registering a provider", async (t) => { + const x = setup(t); writeCatalogCache(catalog, x.path, 1); + await extension(x.pi); await x.hooks.session_shutdown(); + await x.commands["devin-refresh"].handler("", { ui: { notify() {} } }); + assert.equal(x.registrations.length, 1); +});