diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index ae6cd74..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Tests -on: - push: - pull_request: -permissions: - contents: read -jobs: - test: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - node: [22, 24] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node }} - - run: npm install --ignore-scripts --no-audit --no-fund - - run: npm run typecheck - - run: npm test diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 881646a..0000000 --- a/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -node_modules/ -dist/ -package-lock.json -.DS_Store -*.log -.env -.reference/ diff --git a/README.md b/README.md index d2aa043..9d57e62 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,9 @@ Those models are available through the local Devin CLI. This package uses that C ## Requirements -- Pi Coding Agent 0.86+ +- Pi Coding Agent 0.80+ - A signed-in [Devin CLI](https://docs.devin.ai/cli) (`devin auth status`) -- Node 22.19+ (required by Pi 0.86) +- Node 18+ The CLI binary is resolved in this order: @@ -27,8 +27,6 @@ The CLI binary is resolved in this order: 3. Devin.app's bundled `devin` binary 4. `which devin` -On Windows, discovery checks the CLI installer and Devin Desktop locations, then uses `where.exe` to search PATH. Native executables and `.cmd`/`.bat` wrappers are supported, including paths with spaces. `DEVIN_CLI` takes precedence on every platform. - ## Install From git: @@ -37,16 +35,12 @@ From git: pi install git:github.com/kashyab12/pi-devin ``` -From npm: +After npm publish: ```bash pi install npm:pi-devin ``` -Version 0.2.0 requires Pi 0.86+ and Node 22.19+. It restores system instructions and tools on the current Pi transcript format and sends system instructions through Devin's dedicated prompt field. Version 0.1.2 predates Pi 0.86 support. - -`pi-devin-local` is a separate npm package maintained in the [mizorewww/pi-devin fork](https://github.com/mizorewww/pi-devin). Changes merged here do not update that package. Install only one: both packages register the `devin` provider, so their registrations can overwrite each other. - Local checkout: ```bash @@ -71,8 +65,6 @@ Commands: - `/devin-status` — CLI path, version, auth - `/devin-refresh` — reload `devin models list --format json` -The model catalog is cached for six hours in `$XDG_CACHE_HOME/pi-devin/models.json` (or `~/.cache/pi-devin/models.json`). A fresh cache avoids the CLI call at startup; an older cache remains available while it refreshes in the background. Set `PI_OFFLINE=1` to skip automatic catalog refreshes. `/devin-refresh` still requests a refresh explicitly. - ## What this is / is not | This package | Not this package | diff --git a/extensions/index.ts b/extensions/index.ts index ae57f81..801eb9f 100644 --- a/extensions/index.ts +++ b/extensions/index.ts @@ -1,14 +1,10 @@ import type { ExtensionAPI, ProviderModelConfig } from "@earendil-works/pi-coding-agent"; import type { Api, Model, OAuthCredentials, OAuthLoginCallbacks } from "@earendil-works/pi-ai"; +import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; import { authStatus, loginWithCli, readCredentials } from "../src/credentials.js"; import { whichDevin, devinVersion } from "../src/cli.js"; -import { - type CachedDevinCatalog, - isUsableCatalog, - isCatalogCacheFresh, - readCatalogCache, - writeCatalogCache, -} from "../src/catalog-cache.js"; import { FALLBACK_MODELS, loadCliCatalog, modelsFromCatalog } from "../src/models.js"; import { CLIENT_IDE, CLIENT_VERSION } from "../src/metadata.js"; import { streamDevin } from "../src/stream.js"; @@ -16,39 +12,37 @@ import { streamDevin } from "../src/stream.js"; const PROVIDER_ID = "devin"; const PLACEHOLDER_BASE_URL = "https://server.codeium.com"; -let _pi: ExtensionAPI | null = null; -let catalogRequest: Promise | null = null; +const CACHE_PATH = join(homedir(), ".pi", "agent", "devin-catalog-cache.json"); +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours -function isOffline(): boolean { - const value = process.env.PI_OFFLINE?.toLowerCase(); - return value === "1" || value === "true" || value === "yes"; +interface CachedCatalog { + catalog: unknown; + timestamp: number; } -function refreshCatalog(pi: ExtensionAPI): Promise { - if (!catalogRequest) { - const pending = loadCliCatalog().then((catalog) => { - if (!isUsableCatalog(catalog)) { - throw new Error("Devin CLI returned no usable model families"); - } - const models = modelsFromCatalog(catalog); - try { - writeCatalogCache(catalog); - } catch (error) { - console.warn(`Devin: failed to cache model catalog: ${error instanceof Error ? error.message : String(error)}`); - } - return models; - }); - catalogRequest = pending; - void pending.finally(() => { - if (catalogRequest === pending) catalogRequest = null; - }).catch(() => {}); +function loadCachedCatalog(): unknown | null { + try { + if (!existsSync(CACHE_PATH)) return null; + const raw = readFileSync(CACHE_PATH, "utf-8"); + const cached = JSON.parse(raw) as CachedCatalog; + if (Date.now() - cached.timestamp > CACHE_TTL_MS) return null; + return cached.catalog; + } catch { + return null; } - return catalogRequest.then((models) => { - if (_pi === pi) registerDevinProvider(pi, models); - return models; - }); } +function saveCachedCatalog(catalog: unknown): void { + try { + mkdirSync(join(homedir(), ".pi", "agent"), { recursive: true }); + writeFileSync(CACHE_PATH, JSON.stringify({ catalog, timestamp: Date.now() })); + } catch { + // ignore cache write failures + } +} + +let _pi: ExtensionAPI | null = null; + function registerDevinProvider(pi: ExtensionAPI, models: ProviderModelConfig[]): void { pi.registerProvider(PROVIDER_ID, { name: "Devin Local", @@ -61,7 +55,9 @@ function registerDevinProvider(pi: ExtensionAPI, models: ProviderModelConfig[]): const creds = await loginWithCli(); if (_pi) { try { - await refreshCatalog(_pi); + const catalog = await loadCliCatalog(); + saveCachedCatalog(catalog); + registerDevinProvider(_pi, modelsFromCatalog(catalog)); } catch { // keep current models } @@ -94,33 +90,48 @@ function registerDevinProvider(pi: ExtensionAPI, models: ProviderModelConfig[]): export default async function (pi: ExtensionAPI): Promise { _pi = pi; - let cached: CachedDevinCatalog | null | undefined; - let initialModels = FALLBACK_MODELS; - try { - cached = readCatalogCache(); - if (cached) initialModels = modelsFromCatalog(cached.catalog); - } catch (error) { - cached = null; - console.warn(`Devin: failed to read model catalog cache: ${error instanceof Error ? error.message : String(error)}`); - } + + // Synchronously load cached catalog so subagent processes have the full + // model list immediately (they may not live long enough for the async + // refresh to complete). + const cached = loadCachedCatalog(); + const initialModels = cached + ? modelsFromCatalog(cached as Parameters[0]) + : FALLBACK_MODELS; registerDevinProvider(pi, initialModels); - if (!isOffline()) { - try { - if (readCredentials()) { - if (!cached) { - await refreshCatalog(pi); - } else if (!isCatalogCacheFresh(cached)) { - void refreshCatalog(pi).catch((error) => { - console.warn(`Devin: background model catalog refresh failed: ${error instanceof Error ? error.message : String(error)}`); - }); + try { + if (readCredentials()) { + // Refresh in background without blocking startup + loadCliCatalog().then((catalog) => { + if (catalog) { + saveCachedCatalog(catalog); + if (_pi) registerDevinProvider(_pi, modelsFromCatalog(catalog)); } - } - } catch { - // Cached or fallback models stay registered when credential discovery or a cold refresh fails. + }).catch(() => {}); } + } catch { + // fallback models already registered } + pi.on("session_start", async () => { + try { + if (!_pi || !readCredentials()) return; + const cached = loadCachedCatalog(); + if (cached) { + registerDevinProvider(_pi, modelsFromCatalog(cached as Parameters[0])); + } + loadCliCatalog().then((catalog) => { + if (catalog) { + saveCachedCatalog(catalog); + if (_pi) registerDevinProvider(_pi, modelsFromCatalog(catalog)); + } + }).catch(() => {}); + } catch { + // keep current models + } + }); + pi.registerCommand("devin-status", { description: "Show Devin CLI auth + binary status", handler: async (_args, ctx) => { @@ -143,7 +154,10 @@ export default async function (pi: ExtensionAPI): Promise { description: "Refresh Devin Local model catalog from `devin models list`", handler: async (_args, ctx) => { try { - const models = await refreshCatalog(pi); + const catalog = await loadCliCatalog(); + saveCachedCatalog(catalog); + const models = modelsFromCatalog(catalog); + registerDevinProvider(pi, models); ctx.ui.notify(`Devin: loaded ${models.length} families from the local CLI.`, "info"); } catch (error) { ctx.ui.notify( @@ -155,6 +169,6 @@ export default async function (pi: ExtensionAPI): Promise { }); pi.on("session_shutdown", async () => { - if (_pi === pi) _pi = null; + _pi = null; }); } diff --git a/package.json b/package.json index 9647efb..f108db7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-devin", - "version": "0.2.1", + "version": "0.1.2", "description": "Pi provider for Devin Local models. Auth and catalog come from the local Devin CLI; Pi stays the harness.", "type": "module", "keywords": [ @@ -29,19 +29,17 @@ "LICENSE" ], "scripts": { - "test": "node test-runner.mjs", "typecheck": "tsc --noEmit" }, "peerDependencies": { - "@earendil-works/pi-ai": ">=0.86.0", - "@earendil-works/pi-coding-agent": ">=0.86.0" + "@earendil-works/pi-ai": "*", + "@earendil-works/pi-coding-agent": "*" }, "devDependencies": { "@types/node": "^24.13.3", - "tsx": "^4.23.15", "typescript": "^5.9.2" }, "engines": { - "node": ">=22.19.0" + "node": ">=18.0.0" } } diff --git a/src/catalog-cache.ts b/src/catalog-cache.ts deleted file mode 100644 index 2d3fd72..0000000 --- a/src/catalog-cache.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; -import { randomUUID } from "node:crypto"; -import { homedir } from "node:os"; -import { dirname, join } from "node:path"; -import type { DevinCatalog } from "./models.js"; - -const CACHE_VERSION = 1; - -export const CATALOG_CACHE_MAX_AGE_MS = 6 * 60 * 60 * 1000; - -export interface CachedDevinCatalog { - catalog: DevinCatalog; - fetchedAt: number; -} - -interface CatalogCacheFile extends CachedDevinCatalog { - version: typeof CACHE_VERSION; -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function optionalString(value: unknown): boolean { - return value === undefined || typeof value === "string"; -} - -function optionalTokenLimit(value: unknown): boolean { - return value === undefined || (typeof value === "number" && Number.isSafeInteger(value) && value > 0); -} - -export function isUsableCatalog(value: unknown): value is DevinCatalog { - if (!isRecord(value) || !Array.isArray(value.families) || value.families.length === 0) return false; - return value.families.some((family) => isRecord(family) && Array.isArray(family.variants) && family.variants.length > 0) - && value.families.every((family) => - isRecord(family) - && typeof family.family_label === "string" - && typeof family.family_uid === "string" - && typeof family.slug === "string" - && (family.aliases === undefined || (Array.isArray(family.aliases) && family.aliases.every((alias) => typeof alias === "string"))) - && Array.isArray(family.variants) - && family.variants.every((variant) => - isRecord(variant) - && typeof variant.model_uid === "string" - && variant.model_uid.trim().length > 0 - && typeof variant.label === "string" - && optionalString(variant.cost_summary) - && optionalString(variant.cost_tier) - && optionalTokenLimit(variant.max_context_tokens) - && optionalTokenLimit(variant.max_output_tokens) - && (variant.is_new === undefined || typeof variant.is_new === "boolean") - && (variant.is_beta === undefined || typeof variant.is_beta === "boolean") - ) - ); -} - -export function catalogCachePath( - env: NodeJS.ProcessEnv = process.env, - home: string = homedir(), -): string { - const cacheRoot = env.XDG_CACHE_HOME || join(home, ".cache"); - return join(cacheRoot, "pi-devin", "models.json"); -} - -export function readCatalogCache(path: string = catalogCachePath()): CachedDevinCatalog | null { - if (!existsSync(path)) return null; - const parsed: unknown = JSON.parse(readFileSync(path, "utf8")); - if ( - !isRecord(parsed) - || parsed.version !== CACHE_VERSION - || typeof parsed.fetchedAt !== "number" - || !Number.isFinite(parsed.fetchedAt) - || !isUsableCatalog(parsed.catalog) - ) { - throw new Error(`Invalid Devin model catalog cache: ${path}`); - } - return { catalog: parsed.catalog, fetchedAt: parsed.fetchedAt }; -} - -export function writeCatalogCache( - catalog: DevinCatalog, - path: string = catalogCachePath(), - fetchedAt: number = Date.now(), -): void { - if (!isUsableCatalog(catalog)) throw new Error("Refusing to cache an invalid Devin model catalog"); - mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - const temporaryPath = `${path}.${randomUUID()}.tmp`; - const body: CatalogCacheFile = { version: CACHE_VERSION, fetchedAt, catalog }; - try { - writeFileSync(temporaryPath, `${JSON.stringify(body)}\n`, { mode: 0o600 }); - renameSync(temporaryPath, path); - } finally { - if (existsSync(temporaryPath)) unlinkSync(temporaryPath); - } -} - -export function isCatalogCacheFresh( - cached: CachedDevinCatalog, - now: number = Date.now(), - maxAgeMs: number = CATALOG_CACHE_MAX_AGE_MS, -): boolean { - const age = now - cached.fetchedAt; - return age >= 0 && age <= maxAgeMs; -} diff --git a/src/cli.ts b/src/cli.ts index df448ff..f8627ba 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,4 +1,4 @@ -import { statSync } from "node:fs"; +import { existsSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { execFile, spawn } from "node:child_process"; @@ -6,50 +6,21 @@ import { promisify } from "node:util"; const execFileAsync = promisify(execFile); -function candidateBins(): string[] { - const localAppData = process.env.LOCALAPPDATA ?? join(homedir(), "AppData/Local"); - return [ - // Explicit override always wins. Read lazily so a value set after module - // load (e.g. by the host process) is still honoured. - process.env.DEVIN_CLI, - ...(process.platform === "win32" ? [ - // Official installer location (%LOCALAPPDATA%\devin\cli\bin\devin.exe) - join(localAppData, "devin/cli/bin/devin.exe"), - // Devin Desktop (Windsurf-based) bundled CLI, per-user install - join(localAppData, "Programs/Devin/resources/app/extensions/windsurf/devin/bin/devin.exe"), - // Devin Desktop bundled CLI, machine-wide install - process.env.ProgramFiles && - join(process.env.ProgramFiles, "Devin/resources/app/extensions/windsurf/devin/bin/devin.exe"), - // Set by the official Windows installer when it completes normally. - process.env.LOCALAPPDATA && join(process.env.LOCALAPPDATA, "devin/bin/devin.exe"), - // Manual fallback for installs that could not finish while Devin was running. - join(homedir(), ".devin-cli/devin.exe"), - ] : [ - join(homedir(), ".local/bin/devin"), - join(homedir(), ".devin/bin/devin"), - "/opt/homebrew/bin/devin", - "/usr/local/bin/devin", - "/Applications/Devin.app/Contents/Resources/app/extensions/windsurf/devin/bin/devin", - ]), - ].filter((p): p is string => Boolean(p)); -} +const KNOWN_BINS = [ + process.env.DEVIN_CLI, + join(homedir(), ".local/bin/devin"), + join(homedir(), ".devin/bin/devin"), + "/opt/homebrew/bin/devin", + "/usr/local/bin/devin", + "/Applications/Devin.app/Contents/Resources/app/extensions/windsurf/devin/bin/devin", +].filter((p): p is string => Boolean(p)); let cachedBin: string | null | undefined; -function isFile(path: string): boolean { - try { - return statSync(path).isFile(); - } catch { - return false; - } -} - export function findDevinBin(): string | null { - // An explicit override must take effect even after a successful discovery. - if (process.env.DEVIN_CLI && isFile(process.env.DEVIN_CLI)) return process.env.DEVIN_CLI; if (cachedBin !== undefined) return cachedBin; - for (const bin of candidateBins()) { - if (isFile(bin)) { + for (const bin of KNOWN_BINS) { + if (existsSync(bin)) { cachedBin = bin; return bin; } @@ -65,24 +36,15 @@ export function clearDevinBinCache(): void { export async function whichDevin(): Promise { const known = findDevinBin(); if (known) return known; - // Windows has no `which`; use `where` and probe the returned paths directly. - const isWindows = process.platform === "win32"; - const locator = isWindows ? "where.exe" : "/usr/bin/which"; - const candidates = isWindows ? ["devin.exe", "devin.cmd", "devin"] : ["devin"]; - for (const candidate of candidates) { - try { - const { stdout } = await execFileAsync(locator, [candidate], { timeout: 5_000 }); - // `where` may return several lines; take the first existing one. - for (const line of stdout.split(/\r?\n/)) { - const path = line.trim(); - if (path && isFile(path)) { - cachedBin = path; - return path; - } - } - } catch { - // not on PATH + try { + const { stdout } = await execFileAsync("/usr/bin/which", ["devin"], { timeout: 5_000 }); + const path = stdout.trim(); + if (path && existsSync(path)) { + cachedBin = path; + return path; } + } catch { + // not on PATH } return null; } @@ -98,18 +60,9 @@ export async function runDevin( ); } - // Batch wrappers need cmd.exe. Quote paths with spaces, and reject shell - // expansion characters rather than interpreting them as part of a command. - const wrapper = process.platform === "win32" && /\.(cmd|bat)$/i.test(bin); - if (wrapper && [bin, ...args].some((part) => /["%!\r\n]/.test(part))) { - throw new Error("Unsupported shell characters in Devin wrapper path or arguments. Set DEVIN_CLI to devin.exe."); - } - const executable = wrapper ? process.env.ComSpec || "cmd.exe" : bin; - const commandArgs = wrapper ? ["/d", "/s", "/c", `"${[bin, ...args].map((part) => `"${part}"`).join(" ")}"`] : args; - if (opts.inheritStdio) { return await new Promise((resolve, reject) => { - const child = spawn(executable, commandArgs, { stdio: "inherit", windowsVerbatimArguments: wrapper }); + const child = spawn(bin, args, { stdio: "inherit" }); child.on("error", reject); child.on("close", (code) => { resolve({ stdout: "", stderr: "", code: code ?? 1 }); @@ -118,10 +71,9 @@ export async function runDevin( } try { - const { stdout, stderr } = await execFileAsync(executable, commandArgs, { + const { stdout, stderr } = await execFileAsync(bin, args, { timeout: opts.timeoutMs ?? 30_000, maxBuffer: 8 * 1024 * 1024, - windowsVerbatimArguments: wrapper, }); return { stdout, stderr, code: 0 }; } catch (error) { diff --git a/src/context-estimate.ts b/src/context-estimate.ts new file mode 100644 index 0000000..0532384 --- /dev/null +++ b/src/context-estimate.ts @@ -0,0 +1,137 @@ +import type { Api, Context, Message, Model, Usage } from "@earendil-works/pi-ai"; + +/** + * Local port of pi-ai's `utils/estimate.js` + `clampMaxTokensToContext`. + * Extensions can only resolve the pi-ai compat entry, so the upstream helpers + * are reproduced here to keep the maxTokens context clamp identical. + */ +const CHARS_PER_TOKEN = 4; +const ESTIMATED_IMAGE_CHARS = 4800; +const CONTEXT_SAFETY_TOKENS = 4096; +const MIN_MAX_TOKENS = 1; + +function calculateContextTokens(usage: Usage): number { + return usage.totalTokens || usage.input + usage.output + usage.cacheRead + usage.cacheWrite; +} + +function safeJsonStringify(value: unknown): string { + try { + return JSON.stringify(value) ?? "undefined"; + } catch { + return "[unserializable]"; + } +} + +function estimateTextAndImageContentChars(content: Message["content"]): number { + if (typeof content === "string") return content.length; + let chars = 0; + for (const block of content) chars += block.type === "text" ? block.text.length : ESTIMATED_IMAGE_CHARS; + return chars; +} + +function estimateTextTokens(text: string): number { + return Math.ceil(text.length / CHARS_PER_TOKEN); +} + +function estimateTextAndImageContentTokens(content: Message["content"]): number { + return Math.ceil(estimateTextAndImageContentChars(content) / CHARS_PER_TOKEN); +} + +function estimateMessageTokens(message: Message): number { + if (message.role === "user" || message.role === "toolResult") { + return estimateTextAndImageContentTokens(message.content); + } + let chars = 0; + for (const block of message.content) { + if (block.type === "text") chars += block.text.length; + else if (block.type === "thinking") chars += block.thinking.length; + else chars += block.name.length + safeJsonStringify(block.arguments).length; + } + return Math.ceil(chars / CHARS_PER_TOKEN); +} + +interface UsageInfo { + usage: Usage; + index: number; +} + +function getLastAssistantUsageInfo(messages: Message[]): UsageInfo | undefined { + let latestPrefixTimestamp = Number.NEGATIVE_INFINITY; + let usageInfo: UsageInfo | undefined; + for (let i = 0; i < messages.length; i++) { + const message = messages[i]; + if (message.role === "assistant") { + const usageAppliesToPrefix = message.timestamp >= latestPrefixTimestamp; + if ( + usageAppliesToPrefix && + message.stopReason !== "aborted" && + message.stopReason !== "error" && + calculateContextTokens(message.usage) > 0 + ) { + usageInfo = { usage: message.usage, index: i }; + } + } + latestPrefixTimestamp = Math.max(latestPrefixTimestamp, message.timestamp); + } + return usageInfo; +} + +interface ContextEstimate { + tokens: number; + usageTokens: number; + trailingTokens: number; + lastUsageIndex: number | null; +} + +function estimateMessages(messages: Message[]): ContextEstimate { + const usageInfo = getLastAssistantUsageInfo(messages); + if (usageInfo) { + const usageTokens = calculateContextTokens(usageInfo.usage); + let trailingTokens = 0; + for (let i = usageInfo.index + 1; i < messages.length; i++) { + trailingTokens += estimateMessageTokens(messages[i]); + } + return { tokens: usageTokens + trailingTokens, usageTokens, trailingTokens, lastUsageIndex: usageInfo.index }; + } + let tokens = 0; + for (const message of messages) tokens += estimateMessageTokens(message); + return { tokens, usageTokens: 0, trailingTokens: tokens, lastUsageIndex: null }; +} + +function estimateToolsTokens(tools: Context["tools"]): number { + if (!tools || tools.length === 0) return 0; + return estimateTextTokens(safeJsonStringify(tools)); +} + +export function estimateContextTokens(context: Context): ContextEstimate { + const estimate = estimateMessages(context.messages); + if (estimate.lastUsageIndex !== null) { + const addedNames = new Set( + context.messages + .slice(estimate.lastUsageIndex + 1) + .filter((message) => message.role === "toolResult") + .flatMap((message) => (message.role === "toolResult" ? (message.addedToolNames ?? []) : [])), + ); + const addedToolTokens = estimateToolsTokens(context.tools?.filter((tool) => addedNames.has(tool.name))); + return { + tokens: estimate.tokens + addedToolTokens, + usageTokens: estimate.usageTokens, + trailingTokens: estimate.trailingTokens + addedToolTokens, + lastUsageIndex: estimate.lastUsageIndex, + }; + } + const prefixTokens = + (context.systemPrompt ? estimateTextTokens(context.systemPrompt) : 0) + estimateToolsTokens(context.tools); + return { + tokens: estimate.tokens + prefixTokens, + usageTokens: estimate.usageTokens, + trailingTokens: estimate.trailingTokens + prefixTokens, + lastUsageIndex: estimate.lastUsageIndex, + }; +} + +export function clampMaxTokensToContext(model: Model, context: Context, maxTokens: number): number { + if (model.contextWindow <= 0) return Math.max(MIN_MAX_TOKENS, maxTokens); + const available = model.contextWindow - estimateContextTokens(context).tokens - CONTEXT_SAFETY_TOKENS; + return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, available)); +} diff --git a/src/context-map.ts b/src/context-map.ts index d54fc3d..137c480 100644 --- a/src/context-map.ts +++ b/src/context-map.ts @@ -1,12 +1,4 @@ -import { - collapseSystemMessages, - getCurrentSystemPrompt, - getCurrentTools, - withoutInitialSystemMessage, - type Message, - type Tool, - type TranscriptContext, -} from "@earendil-works/pi-ai"; +import type { Api, Context, Message, Model, Tool } from "@earendil-works/pi-ai"; export interface ContentPart { type: "text" | "image"; @@ -17,9 +9,14 @@ export interface ContentPart { export interface ChatHistoryItem { role: "user" | "assistant" | "tool"; - content: string | ContentPart[]; + content: ContentPart[]; tool_call_id?: string; tool_calls?: Array<{ id: string; name: string; arguments: string }>; + tool_result_is_error?: boolean; + thinking?: string; + signature?: string; + thinking_redacted?: boolean; + message_id?: string; } export interface ToolDef { @@ -29,15 +26,14 @@ export interface ToolDef { } export interface MappedChat { - /** Sent in GetChatMessageRequest.prompt, separately from conversation history. */ systemPrompt?: string; messages: ChatHistoryItem[]; tools: ToolDef[]; } -function userContent(content: Message["content"]): string | ContentPart[] { - if (typeof content === "string") return content; - if (!Array.isArray(content)) return ""; +function contentParts(content: Message["content"] | string): ContentPart[] { + if (typeof content === "string") return content ? [{ type: "text", text: content }] : []; + if (!Array.isArray(content)) return []; const parts: ContentPart[] = []; for (const part of content) { if (!part || typeof part !== "object") continue; @@ -49,22 +45,38 @@ function userContent(content: Message["content"]): string | ContentPart[] { return parts; } -export function mapContextToChat(context: TranscriptContext): MappedChat { - // Fold later system updates into the single prompt sent separately to Devin. - const transcript = collapseSystemMessages(context); - const systemPrompt = getCurrentSystemPrompt(transcript.messages); +export function mapContextToChat(context: Context, model: Model): MappedChat { const messages: ChatHistoryItem[] = []; - for (const message of withoutInitialSystemMessage(transcript.messages)) { + + for (const message of context.messages) { if (message.role === "user") { - messages.push({ role: "user", content: userContent(message.content) }); + const content = contentParts(message.content); + if (content.length === 0) continue; + messages.push({ role: "user", content }); continue; } if (message.role === "assistant") { - const texts: string[] = []; + const content: ContentPart[] = []; const toolCalls: Array<{ id: string; name: string; arguments: string }> = []; + let thinking = ""; + let signature: string | undefined; + let thinkingRedacted = false; + // Only messages produced by this provider/model carry wire-replayable + // thinking signatures and response ids. + const native = + message.api === model.api && + message.provider === model.provider && + message.model === model.id; for (const part of message.content) { - if (part.type === "text") texts.push(part.text); - if (part.type === "toolCall") { + if (part.type === "text") { + content.push({ type: "text", text: part.text }); + } else if (part.type === "thinking") { + thinking += part.thinking; + if (part.redacted) thinkingRedacted = true; + if (native && !signature && part.thinkingSignature) { + signature = part.thinkingSignature; + } + } else if (part.type === "toolCall") { toolCalls.push({ id: part.id, name: part.name, @@ -72,34 +84,37 @@ export function mapContextToChat(context: TranscriptContext): MappedChat { }); } } + if (content.length === 0 && !thinking && !signature && toolCalls.length === 0) continue; messages.push({ role: "assistant", - content: texts.join("\n"), + content, tool_calls: toolCalls.length > 0 ? toolCalls : undefined, + thinking: thinking || undefined, + signature, + thinking_redacted: thinkingRedacted || undefined, + message_id: native ? message.responseId : undefined, }); continue; } if (message.role === "toolResult") { - const text = - typeof message.content === "string" - ? message.content - : message.content - .filter((part) => part.type === "text") - .map((part) => part.text) - .join("\n"); messages.push({ role: "tool", - content: text, + content: contentParts(message.content), tool_call_id: message.toolCallId, + tool_result_is_error: message.isError || undefined, }); } } - const tools: ToolDef[] = getCurrentTools(transcript.messages).map((tool: Tool) => ({ + const tools: ToolDef[] = (context.tools ?? []).map((tool: Tool) => ({ name: tool.name, description: tool.description, parameters: tool.parameters, })); - return { systemPrompt: systemPrompt || undefined, messages, tools }; + const systemPrompt = Array.isArray(context.systemPrompt) + ? context.systemPrompt.join("\n\n") + : context.systemPrompt; + + return { systemPrompt, messages, tools }; } diff --git a/src/jwt.ts b/src/jwt.ts index 4d8f621..cec4b30 100644 --- a/src/jwt.ts +++ b/src/jwt.ts @@ -7,7 +7,7 @@ export interface MintedUserJwt { expiresAt: number; } -function anySignal(signals: AbortSignal[]): AbortSignal { +export function anySignal(signals: AbortSignal[]): AbortSignal { const builtin = (AbortSignal as unknown as { any?: (s: AbortSignal[]) => AbortSignal }).any; if (typeof builtin === "function") return builtin(signals); const controller = new AbortController(); diff --git a/src/metadata.ts b/src/metadata.ts index e85af18..86d007f 100644 --- a/src/metadata.ts +++ b/src/metadata.ts @@ -11,7 +11,9 @@ import { * Luna) by client ide name. With ide="windsurf" GetChatMessage rejects them * with "This model is only in Devin Local."; with ide="devin-desktop" the * server serves them and the response header echoes the exact model - * (verified: "GPT-5.6 Sol High Thinking" for gpt-5-6-sol-high, 2026-08-29). + * (verified: gpt-5-6-sol-high, 2026-09-15). The version string is still + * gated against a known Windsurf/Devin Desktop release: prefer the installed + * Devin.app product.json, then a current desktop release. */ const FALLBACK_WINDSURF_VERSION = "3.6.27"; const PRODUCT_JSON = diff --git a/src/models.ts b/src/models.ts index e141b97..ac3137f 100644 --- a/src/models.ts +++ b/src/models.ts @@ -65,8 +65,13 @@ function variantKey(uid: string): string | null { "max", "minimal", ]; + // Catalog uids are not uniform: some families use `MODEL_GPT_5_2_HIGH`-style + // uppercase/underscore ids and long-context twins append `-1m` after the + // thinking suffix. Normalize case, separators, and the context suffix before + // matching so every variant lands on its real thinking level. + const normalized = uid.toLowerCase().replace(/_/g, "-").replace(/-1m$/, ""); for (const suffix of suffixes) { - if (uid === suffix || uid.endsWith(`-${suffix}`)) return suffix; + if (normalized === suffix || normalized.endsWith(`-${suffix}`)) return suffix; } return null; } @@ -92,39 +97,82 @@ function preferredDefault(map: ThinkingLevelMap): string | undefined { return undefined; } +function contextLabel(tokens: number | undefined): string { + if (!tokens || tokens <= 0) return ""; + if (tokens % 1_000_000 === 0) return `${tokens / 1_000_000}M`; + return `${Math.round(tokens / 1_000)}K`; +} + +function variantsToModel( + family: DevinFamily, + variants: DevinVariant[], + name?: string, +): ProviderModelConfig | null { + // Seed every level with null so Pi treats levels the catalog lacks as + // explicitly unsupported instead of selectable-but-silently-remapped. + const thinkingLevelMap: ThinkingLevelMap = { + off: null, + minimal: null, + low: null, + medium: null, + high: null, + xhigh: null, + max: null, + }; + for (const variant of variants) { + const level = thinkingFromSuffix(variantKey(variant.model_uid)); + if (level && thinkingLevelMap[level] == null) { + thinkingLevelMap[level] = variant.model_uid; + } + } + + const defaultUid = preferredDefault(thinkingLevelMap) ?? variants[0]?.model_uid ?? family.family_uid; + const sample = variants.find((variant) => variant.model_uid === defaultUid) ?? variants[0]; + if (!sample) return null; + + const mappedLevels = THINKING_ORDER.filter((level) => typeof thinkingLevelMap[level] === "string"); + const reasoning = mappedLevels.length > 1; + + return { + id: defaultUid, + name: name ?? (family.family_label || family.slug || defaultUid), + reasoning, + thinkingLevelMap: reasoning ? thinkingLevelMap : undefined, + input: ["text", "image"], + cost: parseCost(sample.cost_summary), + contextWindow: sample.max_context_tokens ?? 256_000, + maxTokens: sample.max_output_tokens ?? 128_000, + }; +} + function familyToModels(family: DevinFamily): ProviderModelConfig[] { const usable = family.variants.filter((variant) => { const key = variantKey(variant.model_uid); return !key || (!key.includes("priority") && !key.includes("fast") && key !== "thinking-1m"); }); const source = usable.length > 0 ? usable : family.variants; - const thinkingLevelMap: ThinkingLevelMap = {}; + // Group by advertised context window so long-context twins (`*-1m`) remain + // selectable instead of collapsing into the primary variant group. The + // smallest-context group keeps the plain family label for stable model ids. + const groups = new Map(); for (const variant of source) { - const level = thinkingFromSuffix(variantKey(variant.model_uid)); - if (level && thinkingLevelMap[level] === undefined) { - thinkingLevelMap[level] = variant.model_uid; - } + const context = variant.max_context_tokens ?? 0; + const group = groups.get(context) ?? []; + group.push(variant); + groups.set(context, group); } - - const defaultUid = preferredDefault(thinkingLevelMap) ?? source[0]?.model_uid ?? family.family_uid; - const sample = source.find((variant) => variant.model_uid === defaultUid) ?? source[0]; - if (!sample) return []; - - const mappedLevels = THINKING_ORDER.filter((level) => typeof thinkingLevelMap[level] === "string"); - const reasoning = mappedLevels.length > 1; - - return [ - { - id: defaultUid, - name: family.family_label || family.slug || defaultUid, - reasoning, - thinkingLevelMap: reasoning ? thinkingLevelMap : undefined, - input: ["text", "image"], - cost: parseCost(sample.cost_summary), - contextWindow: sample.max_context_tokens ?? 256_000, - maxTokens: sample.max_output_tokens ?? 128_000, - }, - ]; + const ordered = [...groups.values()].sort( + (a, b) => (a[0]?.max_context_tokens ?? 0) - (b[0]?.max_context_tokens ?? 0), + ); + const models: ProviderModelConfig[] = []; + ordered.forEach((variants, index) => { + const context = variants[0]?.max_context_tokens; + const name = + index === 0 ? undefined : `${family.family_label || family.slug} (${contextLabel(context)})`; + const model = variantsToModel(family, variants, name); + if (model) models.push(model); + }); + return models; } export const FALLBACK_MODELS: ProviderModelConfig[] = [ @@ -133,6 +181,8 @@ export const FALLBACK_MODELS: ProviderModelConfig[] = [ name: "Claude Opus 5", reasoning: true, thinkingLevelMap: { + off: null, + minimal: null, low: "claude-opus-5-low", medium: "claude-opus-5-medium", high: "claude-opus-5-high", @@ -149,6 +199,8 @@ export const FALLBACK_MODELS: ProviderModelConfig[] = [ name: "Claude Fable 5", reasoning: true, thinkingLevelMap: { + off: null, + minimal: null, low: "claude-5-fable-low", medium: "claude-5-fable-medium", high: "claude-5-fable-high", @@ -166,6 +218,7 @@ export const FALLBACK_MODELS: ProviderModelConfig[] = [ reasoning: true, thinkingLevelMap: { off: "gpt-5-6-sol-none", + minimal: null, low: "gpt-5-6-sol-low", medium: "gpt-5-6-sol-medium", high: "gpt-5-6-sol-high", @@ -181,6 +234,15 @@ export const FALLBACK_MODELS: ProviderModelConfig[] = [ id: "swe-1-7", name: "SWE-1.7", reasoning: true, + thinkingLevelMap: { + off: null, + minimal: null, + low: null, + medium: "swe-1-7", + high: "swe-1-7", + xhigh: null, + max: null, + }, input: ["text", "image"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 262_000, @@ -211,11 +273,12 @@ export function resolveModelUid( thinkingLevelMap: ThinkingLevelMap | undefined, reasoning?: string, ): string { - if (reasoning && thinkingLevelMap) { - const mapped = thinkingLevelMap[reasoning as keyof ThinkingLevelMap]; - if (typeof mapped === "string") return mapped; - } if (thinkingLevelMap) { + // Pi encodes thinkingLevel "off" as an absent reasoning option; resolve it + // against the explicit "off" mapping (the catalog's *-none variant) before + // falling back to the family's preferred default. + const mapped = thinkingLevelMap[(reasoning ?? "off") as keyof ThinkingLevelMap]; + if (typeof mapped === "string") return mapped; const fallback = preferredDefault(thinkingLevelMap); if (fallback) return fallback; } diff --git a/src/stream.ts b/src/stream.ts index 88e89c5..6fd71dd 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -4,15 +4,15 @@ import { type Api, type AssistantMessage, type AssistantMessageEventStream, + type Context, type Model, type SimpleStreamOptions, - type TranscriptContext, calculateCost, createAssistantMessageEventStream, - parseStreamingJson, } from "@earendil-works/pi-ai"; -import { mapContextToChat, type ChatHistoryItem, type ContentPart, type ToolDef } from "./context-map.js"; -import { getCachedUserJwt } from "./jwt.js"; +import { clampMaxTokensToContext } from "./context-estimate.js"; +import { mapContextToChat, type ChatHistoryItem, type ToolDef } from "./context-map.js"; +import { anySignal, getCachedUserJwt } from "./jwt.js"; import { buildMetadata } from "./metadata.js"; import { resolveModelUid } from "./models.js"; import { @@ -24,32 +24,40 @@ import { iterFields, } from "./wire.js"; -const SOURCE_BY_ROLE: Record = { +const SOURCE_BY_ROLE: Record = { user: 1, assistant: 2, tool: 4, }; +/** Connect frame payloads larger than this indicate a corrupt length prefix. */ +const MAX_CONNECT_FRAME_PAYLOAD = 16 * 1024 * 1024; +/** + * Recovery heuristic for opaque Devin `invalid_argument`/`internal error` + * trailers: compactable message history this large is likely the cause, so the + * failure is reported in a shape Pi recognizes as context overflow. + */ +const LARGE_HISTORY_RECOVERY_BYTES = 512 * 1024; + export type CloudChatEvent = | { kind: "text"; text: string } | { kind: "reasoning"; text: string } + | { kind: "signature"; text: string } + | { kind: "thinking_redacted" } + | { kind: "message_id"; id: string } + | { kind: "actual_model"; uid: string } | { kind: "tool_call_start"; id: string; name: string } | { kind: "tool_call_args"; argsDelta: string; id?: string } - | { kind: "finish"; reason: "stop" | "tool_calls" | "length" | "content_filter" } + | { kind: "finish"; reason: "stop" | "tool_calls" | "length" | "content_filter" | "error" } | { kind: "usage"; - promptTokens?: number; - completionTokens?: number; - totalTokens?: number; - cachedInputTokens?: number; - cacheCreationInputTokens?: number; + input?: number; + output?: number; + cacheRead?: number; + cacheWrite?: number; + modelUid?: string; }; -function normalizeContent(content: string | ContentPart[]): ContentPart[] { - if (typeof content === "string") return [{ type: "text", text: content }]; - return content; -} - function encodeImageData(img: { mimeType?: string; base64Data?: string }): Buffer { return Buffer.concat([ encodeString(1, img.base64Data ?? ""), @@ -61,39 +69,41 @@ function encodeChatToolCall(tc: { id: string; name: string; arguments: string }) return Buffer.concat([encodeString(1, tc.id), encodeString(2, tc.name), encodeString(3, tc.arguments)]); } -function encodeChatMessagePrompt( - content: ContentPart[], - source: number, - opts?: { toolCallId?: string; toolCalls?: Array<{ id: string; name: string; arguments: string }> }, -): Buffer { - const text = content +export function encodeChatMessagePrompt(item: ChatHistoryItem, messageId: string): Buffer { + const text = item.content .filter((part) => part.type === "text") .map((part) => part.text ?? "") .join("\n"); const parts: Buffer[] = [ - encodeVarintField(2, source), + encodeString(1, messageId), + encodeVarintField(2, SOURCE_BY_ROLE[item.role] ?? 1), encodeString(3, text), - encodeVarintField(4, Math.max(1, Math.floor(text.length / 4))), - encodeVarintField(5, 1), ]; - if (opts?.toolCallId) parts.push(encodeString(7, opts.toolCallId)); - for (const tc of opts?.toolCalls ?? []) parts.push(encodeMessage(6, encodeChatToolCall(tc))); - for (const img of content.filter((part) => part.type === "image")) { + for (const tc of item.tool_calls ?? []) parts.push(encodeMessage(6, encodeChatToolCall(tc))); + if (item.tool_call_id) parts.push(encodeString(7, item.tool_call_id)); + if (item.tool_result_is_error) parts.push(encodeVarintField(9, 1)); + for (const img of item.content.filter((part) => part.type === "image")) { parts.push(encodeMessage(10, encodeImageData(img))); } + if (item.thinking) parts.push(encodeString(11, item.thinking)); + if (item.signature) parts.push(encodeString(12, item.signature)); + if (item.thinking_redacted) parts.push(encodeVarintField(13, 1)); return Buffer.concat(parts); } -function encodeCompletionConfiguration(maxOutputTokens?: number): Buffer { +function encodeCompletionConfiguration(args: { + maxTokens: number; + temperature: number; + topP: number; + topK: number; +}): Buffer { return Buffer.concat([ encodeVarintField(1, 1), - encodeVarintField(2, 64_000), - encodeVarintField(3, maxOutputTokens ?? 128_000), - encodeFixed64Field(5, 0.7), - encodeFixed64Field(6, 0.95), - encodeVarintField(7, 50), - encodeFixed64Field(8, 1.0), - encodeFixed64Field(11, 1.0), + encodeVarintField(2, args.maxTokens), + encodeVarintField(3, 400), + encodeFixed64Field(5, args.temperature), + encodeVarintField(7, args.topK), + encodeFixed64Field(8, args.topP), ]); } @@ -106,7 +116,7 @@ function encodeToolDef(tool: ToolDef): Buffer { ]); } -function buildGetChatMessageRequest(args: { +export interface WireRequestArgs { apiKey: string; userJwt: string; modelUid: string; @@ -118,8 +128,13 @@ function buildGetChatMessageRequest(args: { sessionId: string; requestId: bigint; triggerId: string; - maxOutputTokens?: number; -}): Buffer { + maxTokens: number; + temperature: number; + topP: number; + topK: number; +} + +export function buildGetChatMessageRequest(args: WireRequestArgs): { body: Buffer; historyBytes: number } { const metadata = buildMetadata({ apiKey: args.apiKey, userJwt: args.userJwt, @@ -127,36 +142,46 @@ function buildGetChatMessageRequest(args: { requestId: args.requestId, triggerId: args.triggerId, }); - const prompts = args.messages.map((message) => - encodeMessage( - 3, - encodeChatMessagePrompt(normalizeContent(message.content), SOURCE_BY_ROLE[message.role], { - toolCallId: message.role === "tool" ? message.tool_call_id : undefined, - toolCalls: message.role === "assistant" ? message.tool_calls : undefined, - }), - ), - ); - return Buffer.concat([ + const promptFields: Buffer[] = []; + args.messages.forEach((message, index) => { + const messageId = message.message_id ?? `${args.cascadeId}-${index}`; + promptFields.push(encodeMessage(3, encodeChatMessagePrompt(message, messageId))); + }); + const history = Buffer.concat(promptFields); + const body = Buffer.concat([ encodeMessage(1, metadata), - ...(args.systemPrompt ? [encodeString(2, args.systemPrompt)] : []), - ...prompts, + args.systemPrompt ? encodeString(2, args.systemPrompt) : Buffer.alloc(0), + history, encodeVarintField(7, 5), - encodeMessage(8, encodeCompletionConfiguration(args.maxOutputTokens)), + encodeMessage(8, encodeCompletionConfiguration(args)), ...(args.tools ?? []).map((tool) => encodeMessage(10, encodeToolDef(tool))), encodeString(16, args.cascadeId), + encodeString(17, args.promptId), + encodeVarintField(20, 1), encodeString(21, args.modelUid), - encodeString(22, args.promptId), ]); + return { body, historyBytes: history.length }; } -function* decodeChatFrame(proto: Buffer): Generator { +export function* decodeChatFrame(proto: Buffer): Generator { for (const field of iterFields(proto)) { - if (field.num === 3 && field.wire === 2 && Buffer.isBuffer(field.value)) { + if (field.num === 1 && field.wire === 2 && Buffer.isBuffer(field.value)) { + const id = field.value.toString("utf8"); + if (id) yield { kind: "message_id", id }; + } else if (field.num === 3 && field.wire === 2 && Buffer.isBuffer(field.value)) { const text = field.value.toString("utf8"); if (text) yield { kind: "text", text }; } else if (field.num === 9 && field.wire === 2 && Buffer.isBuffer(field.value)) { const text = field.value.toString("utf8"); if (text) yield { kind: "reasoning", text }; + } else if (field.num === 10 && field.wire === 2 && Buffer.isBuffer(field.value)) { + const signature = field.value.toString("utf8"); + if (signature) yield { kind: "signature", text: signature }; + } else if (field.num === 11 && field.wire === 0 && field.value) { + yield { kind: "thinking_redacted" }; + } else if (field.num === 23 && field.wire === 2 && Buffer.isBuffer(field.value)) { + const uid = field.value.toString("utf8"); + if (uid) yield { kind: "actual_model", uid }; } else if (field.num === 6 && field.wire === 2 && Buffer.isBuffer(field.value)) { let id: string | undefined; let name: string | undefined; @@ -173,59 +198,77 @@ function* decodeChatFrame(proto: Buffer): Generator { if (argsDelta !== undefined) yield { kind: "tool_call_args", argsDelta, ...(id ? { id } : {}) }; } else if (field.num === 5 && field.wire === 0) { const value = Number(field.value); - let reason: Extract['reason'] = "stop"; + let reason: Extract["reason"] = "stop"; if (value === 10) reason = "tool_calls"; + else if (value === 1 || value === 3 || value === 5) reason = "length"; else if (value === 11) reason = "content_filter"; - else if (value === 1 || value === 3) reason = "length"; + else if (value === 12 || value === 13) reason = "error"; yield { kind: "finish", reason }; - } else if (field.num === 28 && field.wire === 2 && Buffer.isBuffer(field.value)) { - const usage = decodeUsage(field.value); + } else if (field.num === 7 && field.wire === 2 && Buffer.isBuffer(field.value)) { + const usage = decodeUsageStats(field.value); if (usage) yield usage; } } } -function decodeUsage(buf: Buffer): CloudChatEvent | null { - let promptTokens: number | undefined; - let completionTokens: number | undefined; - let cachedInputTokens: number | undefined; - let cacheCreationInputTokens: number | undefined; +function decodeUsageStats(buf: Buffer): CloudChatEvent | null { + let input: number | undefined; + let output: number | undefined; + let cacheWrite: number | undefined; + let cacheRead: number | undefined; + let modelUid: string | undefined; for (const field of iterFields(buf)) { - if (field.num !== 2 || field.wire !== 2 || !Buffer.isBuffer(field.value)) continue; - let metric: string | undefined; - let value: number | undefined; - for (const inner of iterFields(field.value)) { - if (inner.num === 5 && inner.wire === 2 && Buffer.isBuffer(inner.value)) { - metric = inner.value.toString("utf8"); - } else if (inner.num === 4 && inner.wire === 2 && Buffer.isBuffer(inner.value)) { - for (const dim of iterFields(inner.value)) { - if (dim.num === 2 && dim.wire === 5 && Buffer.isBuffer(dim.value)) { - value = dim.value.readFloatLE(0); - } - } - } + if (field.wire === 0) { + const n = Number(field.value); + if (field.num === 2) input = n; + else if (field.num === 3) output = n; + else if (field.num === 4) cacheWrite = n; + else if (field.num === 5) cacheRead = n; + } else if (field.num === 9 && field.wire === 2 && Buffer.isBuffer(field.value)) { + modelUid = field.value.toString("utf8"); } - if (!metric || value === undefined || !Number.isFinite(value)) continue; - const n = Math.round(value); - if (metric === "input_tokens") promptTokens = n; - else if (metric === "output_tokens") completionTokens = n; - else if (metric.includes("cached") || metric.includes("cache_read")) cachedInputTokens = n; - else if (metric.includes("cache_creation")) cacheCreationInputTokens = n; } - if ([promptTokens, completionTokens, cachedInputTokens, cacheCreationInputTokens].every((n) => n === undefined)) return null; - return { - kind: "usage", - promptTokens, - completionTokens, - totalTokens: (promptTokens ?? 0) + (completionTokens ?? 0) + (cachedInputTokens ?? 0) + (cacheCreationInputTokens ?? 0), - cachedInputTokens, - cacheCreationInputTokens, - }; + if (input === undefined && output === undefined && cacheRead === undefined && cacheWrite === undefined) { + return null; + } + return { kind: "usage", input, output, cacheRead, cacheWrite, modelUid }; +} + +export interface TrailerError { + code: string; + message: string; +} + +export function readTrailerError(text: string): TrailerError | null { + if (!text.includes('"error"')) return null; + try { + const parsed = JSON.parse(text) as { error?: { code?: unknown; message?: unknown } }; + if (!parsed.error || typeof parsed.error !== "object") return null; + return { + code: typeof parsed.error.code === "string" ? parsed.error.code : "", + message: typeof parsed.error.message === "string" ? parsed.error.message : "", + }; + } catch { + return { code: "", message: text.slice(0, 512) }; + } } const sessionCache = new Map(); -function sessionIds(apiKey: string, host: string) { +export function sessionIds( + apiKey: string, + host: string, + options?: { sessionId?: string; cacheRetention?: string }, +): { sessionId: string; cascadeId: string } { + // Compaction and other no-cache callers must not share or reuse a cascade. + if (options?.cacheRetention === "none") { + return { sessionId: randomUUID(), cascadeId: randomUUID() }; + } + // Pi threads its session id through so each session tree keeps its own + // Devin cascade instead of sharing one process-wide cascade. + if (options?.sessionId) { + return { sessionId: options.sessionId, cascadeId: options.sessionId }; + } const key = `${host}\x1f${apiKey}`; let ids = sessionCache.get(key); if (!ids) { @@ -235,45 +278,41 @@ function sessionIds(apiKey: string, host: string) { return ids; } +function mergeHeaders(extra: Record | undefined): Record { + const headers = new Map([ + ["content-type", "application/connect+proto"], + ["connect-protocol-version", "1"], + ["connect-accept-encoding", "gzip"], + ["user-agent", "connect-go/1.18.1 (go1.26.3)"], + ]); + for (const [name, value] of Object.entries(extra ?? {})) { + if (value == null) headers.delete(name.toLowerCase()); + else headers.set(name.toLowerCase(), value); + } + return Object.fromEntries(headers); +} + async function* streamChatEvents(args: { - apiKey: string; host: string; - modelUid: string; - systemPrompt?: string; - messages: ChatHistoryItem[]; - tools?: ToolDef[]; - maxOutputTokens?: number; + request: { body: Buffer; historyBytes: number }; + headers?: Record; signal?: AbortSignal; + fetchFn?: typeof fetch; + onResponse?: (response: { status: number; headers: Record }) => void | Promise; }): AsyncGenerator { const host = args.host.replace(/\/$/, ""); - const userJwt = await getCachedUserJwt(args.apiKey, host, args.signal); - const ids = sessionIds(args.apiKey, host); - const proto = buildGetChatMessageRequest({ - apiKey: args.apiKey, - userJwt, - modelUid: args.modelUid, - systemPrompt: args.systemPrompt, - messages: args.messages, - tools: args.tools, - cascadeId: ids.cascadeId, - promptId: randomUUID(), - sessionId: ids.sessionId, - requestId: BigInt(Date.now()), - triggerId: randomUUID(), - maxOutputTokens: args.maxOutputTokens, - }); - - const resp = await fetch(`${host}/exa.api_server_pb.ApiServerService/GetChatMessage`, { - method: "POST", - headers: { - "Content-Type": "application/connect+proto", - "Connect-Protocol-Version": "1", - "Connect-Content-Encoding": "gzip", - "Connect-Accept-Encoding": "gzip", + const resp = await (args.fetchFn ?? fetch)( + `${host}/exa.api_server_pb.ApiServerService/GetChatMessage`, + { + method: "POST", + headers: mergeHeaders(args.headers), + body: new Uint8Array(frameConnectStream(args.request.body, false)), + signal: args.signal, }, - body: new Uint8Array(frameConnectStream(proto, true)), - signal: args.signal, - }); + ); + if (args.onResponse) { + await args.onResponse({ status: resp.status, headers: Object.fromEntries(resp.headers.entries()) }); + } if (!resp.ok) { throw new Error(`GetChatMessage HTTP ${resp.status}: ${(await resp.text()).slice(0, 300)}`); } @@ -283,7 +322,8 @@ async function* streamChatEvents(args: { const queue: Buffer[] = []; let queued = 0; let sawEos = false; - let trailerError: string | null = null; + let producedOutput = false; + let trailerError: TrailerError | null = null; const peek = (n: number): Buffer | null => { if (queued < n) return null; @@ -331,6 +371,9 @@ async function* streamChatEvents(args: { if (!header) break; const flags = header[0]; const len = header.readUInt32BE(1); + if (len > MAX_CONNECT_FRAME_PAYLOAD) { + throw new Error(`Devin stream frame too large: ${len} bytes`); + } if (queued < 5 + len) break; drop(5); const raw = peek(len) ?? Buffer.alloc(0); @@ -339,18 +382,15 @@ async function* streamChatEvents(args: { if (flags & 0x01) payload = zlib.gunzipSync(raw); if (flags & 0x02) { sawEos = true; - const text = payload.toString("utf8"); - if (text.includes('"error"')) { - try { - const parsed = JSON.parse(text) as { error?: { message?: string } }; - trailerError = parsed.error?.message ?? text; - } catch { - trailerError = text; - } - } + trailerError = readTrailerError(payload.toString("utf8").trim()); continue; } - yield* decodeChatFrame(payload); + for (const event of decodeChatFrame(payload)) { + if (event.kind === "text" || event.kind === "reasoning" || event.kind === "tool_call_start") { + producedOutput = true; + } + yield event; + } } } } finally { @@ -360,19 +400,35 @@ async function* streamChatEvents(args: { // ignore } try { - await resp.body?.cancel(); + void resp.body?.cancel(); } catch { // ignore } } - if (trailerError) throw new Error(trailerError); + if (trailerError) { + if ( + !producedOutput && + trailerError.code.toLowerCase() === "invalid_argument" && + /\binternal error\b/i.test(trailerError.message) && + args.request.historyBytes >= LARGE_HISTORY_RECOVERY_BYTES + ) { + throw new Error( + `Devin stream rejected: ${trailerError.message} (message history of ${args.request.historyBytes} bytes likely exceeds the context window)`, + ); + } + throw new Error( + trailerError.code + ? `Devin stream rejected (${trailerError.code}): ${trailerError.message}` + : `Devin stream rejected: ${trailerError.message}`, + ); + } if (!sawEos) throw new Error("Devin stream ended without an EOS trailer"); } export function streamDevin( model: Model, - context: TranscriptContext, + context: Context, options?: SimpleStreamOptions, ): AssistantMessageEventStream { const stream = createAssistantMessageEventStream(); @@ -398,10 +454,9 @@ export function streamDevin( let textOpen = false; let thinkingOpen = false; - let toolIndex = -1; - let partialJson = ""; - let toolId = ""; - let toolName = ""; + const openTools = new Map(); + const toolJson = new Map(); + let lastToolId: string | undefined; const closeText = () => { if (!textOpen) return; @@ -421,38 +476,92 @@ export function streamDevin( } thinkingOpen = false; }; - const closeTool = () => { - if (toolIndex < 0) return; - const block = output.content[toolIndex]; + const closeTool = (id: string) => { + const idx = openTools.get(id); + if (idx === undefined) return; + openTools.delete(id); + const block = output.content[idx]; if (block.type === "toolCall") { - block.arguments = parseStreamingJson(partialJson); + const json = toolJson.get(id) ?? ""; + try { + block.arguments = JSON.parse(json); + } catch { + // keep last parsed object + } stream.push({ type: "toolcall_end", - contentIndex: toolIndex, - toolCall: { type: "toolCall", id: toolId, name: toolName, arguments: block.arguments }, + contentIndex: idx, + toolCall: { type: "toolCall", id, name: block.name, arguments: block.arguments }, partial: output, }); } - toolIndex = -1; + toolJson.delete(id); + if (lastToolId === id) lastToolId = undefined; + }; + const closeAllTools = () => { + for (const id of [...openTools.keys()]) closeTool(id); + }; + const currentThinkingBlock = () => { + const block = output.content[output.content.length - 1]; + return block?.type === "thinking" ? block : undefined; }; try { const apiKey = options?.apiKey; if (!apiKey) throw new Error("No Devin credentials. Run /login devin (uses the local Devin CLI)."); + if (options?.deferred) throw new Error("The devin provider does not support deferred responses"); const host = (options?.env?.DEVIN_API_SERVER_URL || "https://server.codeium.com").replace(/\/$/, ""); const modelUid = resolveModelUid(model.id, model.thinkingLevelMap, options?.reasoning); - const mapped = mapContextToChat(context); - stream.push({ type: "start", partial: output }); + output.providerThinkingLevel = modelUid; + output.responseModel = modelUid; + const mapped = mapContextToChat(context, model); + const timeoutMs = options?.timeoutMs; + const signal = + timeoutMs && timeoutMs > 0 + ? anySignal( + [options?.signal, AbortSignal.timeout(timeoutMs)].filter((s): s is AbortSignal => Boolean(s)), + ) + : options?.signal; + const ids = sessionIds(apiKey, host, options); + const userJwt = await getCachedUserJwt(apiKey, host, signal); - for await (const event of streamChatEvents({ + const sampling = + model.samplingParams || options?.samplingParams + ? { ...model.samplingParams, ...options?.samplingParams } + : {}; + const wireArgs = { apiKey, - host, + userJwt, modelUid, systemPrompt: mapped.systemPrompt, messages: mapped.messages, tools: mapped.tools.length > 0 ? mapped.tools : undefined, - maxOutputTokens: options?.maxTokens, - signal: options?.signal, + cascadeId: ids.cascadeId, + promptId: randomUUID(), + sessionId: ids.sessionId, + requestId: BigInt(Date.now()), + triggerId: randomUUID(), + maxTokens: clampMaxTokensToContext(model, context, options?.maxTokens ?? model.maxTokens), + temperature: options?.temperature ?? 1.0, + topP: typeof sampling.topP === "number" ? sampling.topP : 0.95, + topK: typeof sampling.topK === "number" ? sampling.topK : 40, + }; + const overridden = await options?.onPayload?.(wireArgs, model); + const finalArgs = + overridden && typeof overridden === "object" ? { ...wireArgs, ...overridden } : wireArgs; + const request = buildGetChatMessageRequest(finalArgs); + + stream.push({ type: "start", partial: output }); + + for await (const event of streamChatEvents({ + host, + request, + headers: options?.headers, + signal, + fetchFn: options?.fetch, + onResponse: options?.onResponse + ? (response) => options?.onResponse?.(response, model) + : undefined, })) { if (event.kind === "text") { closeThinking(); @@ -480,46 +589,66 @@ export function streamDevin( block.thinking += event.text; stream.push({ type: "thinking_delta", contentIndex: idx, delta: event.text, partial: output }); } + } else if (event.kind === "signature") { + const block = currentThinkingBlock(); + if (block) block.thinkingSignature = event.text; + } else if (event.kind === "thinking_redacted") { + const block = currentThinkingBlock(); + if (block) block.redacted = true; + } else if (event.kind === "message_id") { + if (!output.responseId) output.responseId = event.id; + } else if (event.kind === "actual_model") { + output.responseModel = event.uid; } else if (event.kind === "tool_call_start") { closeText(); closeThinking(); - closeTool(); - toolId = event.id; - toolName = event.name; - partialJson = ""; output.content.push({ type: "toolCall", id: event.id, name: event.name, arguments: {} }); - toolIndex = output.content.length - 1; - stream.push({ type: "toolcall_start", contentIndex: toolIndex, partial: output }); + openTools.set(event.id, output.content.length - 1); + toolJson.set(event.id, ""); + lastToolId = event.id; + stream.push({ type: "toolcall_start", contentIndex: output.content.length - 1, partial: output }); } else if (event.kind === "tool_call_args") { - if (toolIndex < 0) continue; - partialJson += event.argsDelta; - const block = output.content[toolIndex]; + const id = event.id ?? lastToolId; + const idx = id === undefined ? undefined : openTools.get(id); + if (id === undefined || idx === undefined) continue; + const json = (toolJson.get(id) ?? "") + event.argsDelta; + toolJson.set(id, json); + const block = output.content[idx]; if (block.type === "toolCall") { - block.arguments = parseStreamingJson(partialJson); + try { + block.arguments = JSON.parse(json); + } catch { + // incomplete json + } } - stream.push({ type: "toolcall_delta", contentIndex: toolIndex, delta: event.argsDelta, partial: output }); + stream.push({ type: "toolcall_delta", contentIndex: idx, delta: event.argsDelta, partial: output }); } else if (event.kind === "finish") { closeText(); closeThinking(); - closeTool(); + closeAllTools(); + if (event.reason === "content_filter") { + throw new Error("Devin stop_reason: content_filter"); + } + if (event.reason === "error") { + throw new Error("Devin stop_reason: error"); + } output.stopReason = event.reason === "tool_calls" ? "toolUse" : event.reason === "length" ? "length" : "stop"; } else if (event.kind === "usage") { - // Empty trailers must not erase usage already received. Metrics are - // snapshots, and a frame may contain only some of the components. - if (!event.totalTokens) continue; - output.usage.input = event.promptTokens ?? output.usage.input; - output.usage.output = event.completionTokens ?? output.usage.output; - output.usage.cacheRead = event.cachedInputTokens ?? output.usage.cacheRead; - output.usage.cacheWrite = event.cacheCreationInputTokens ?? output.usage.cacheWrite; - output.usage.totalTokens = output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; + output.usage.input = event.input ?? 0; + output.usage.output = event.output ?? 0; + output.usage.cacheRead = event.cacheRead ?? 0; + output.usage.cacheWrite = event.cacheWrite ?? 0; + output.usage.totalTokens = + output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; + if (!output.responseModel && event.modelUid) output.responseModel = event.modelUid; calculateCost(model, output.usage); } } closeText(); closeThinking(); - closeTool(); + closeAllTools(); stream.push({ type: "done", reason: output.stopReason as "stop" | "length" | "toolUse", message: output }); stream.end(); } catch (error) { diff --git a/test-runner.mjs b/test-runner.mjs deleted file mode 100644 index 6b25dfe..0000000 --- a/test-runner.mjs +++ /dev/null @@ -1,8 +0,0 @@ -import { readdirSync } from "node:fs"; -import { spawnSync } from "node:child_process"; -import { join } from "node:path"; - -const files = readdirSync("test").filter((file) => file.endsWith(".test.mjs")).sort().map((file) => join("test", file)); -const result = spawnSync(process.execPath, ["--import", "tsx", "--test", ...files], { stdio: "inherit" }); -if (result.error) throw result.error; -process.exit(result.status ?? 1); diff --git a/test/catalog-cache.test.mjs b/test/catalog-cache.test.mjs deleted file mode 100644 index f2db4ed..0000000 --- a/test/catalog-cache.test.mjs +++ /dev/null @@ -1,65 +0,0 @@ -import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, readFileSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import test from "node:test"; -import { - catalogCachePath, - isUsableCatalog, - isCatalogCacheFresh, - readCatalogCache, - writeCatalogCache, -} from "../src/catalog-cache.ts"; - -const catalog = { - families: [{ - family_label: "SWE-2", - family_uid: "swe-2", - slug: "swe-2", - variants: [{ model_uid: "swe-2-high", label: "SWE-2 High" }], - }], -}; - -test("writes and reads a validated catalog atomically", (t) => { - const directory = mkdtempSync(join(tmpdir(), "pi-devin-cache-")); - t.after(() => rmSync(directory, { recursive: true, force: true })); - const path = join(directory, "nested", "models.json"); - writeCatalogCache(catalog, path, 1_000); - assert.deepEqual(readCatalogCache(path), { catalog, fetchedAt: 1_000 }); - assert.equal(JSON.parse(readFileSync(path, "utf8")).version, 1); -}); - -test("rejects malformed cache content", (t) => { - const directory = mkdtempSync(join(tmpdir(), "pi-devin-cache-")); - t.after(() => rmSync(directory, { recursive: true, force: true })); - const path = join(directory, "models.json"); - writeFileSync(path, JSON.stringify({ version: 1, fetchedAt: 1_000, catalog: { families: [{}] } })); - assert.throws(() => readCatalogCache(path), /Invalid Devin model catalog cache/); -}); - -test("rejects catalogs without usable model families", () => { - assert.equal(isUsableCatalog({ families: [] }), false); - assert.equal(isUsableCatalog({ - families: [{ family_label: "Empty", family_uid: "empty", slug: "empty", variants: [] }], - }), false); -}); - -test("checks cache age and future timestamps", () => { - const cached = { catalog, fetchedAt: 1_000 }; - assert.equal(isCatalogCacheFresh(cached, 1_500, 500), true); - assert.equal(isCatalogCacheFresh(cached, 1_501, 500), false); - assert.equal(isCatalogCacheFresh(cached, 999, 500), false); -}); - -test("uses XDG cache directory when configured", () => { - assert.equal(catalogCachePath({ XDG_CACHE_HOME: "/cache" }, "/home/test"), join("/cache", "pi-devin", "models.json")); - assert.equal(catalogCachePath({}, "/home/test"), join("/home/test", ".cache", "pi-devin", "models.json")); -}); - -for (const [field, value] of Object.entries({ cost_summary: {}, max_context_tokens: "1000", max_output_tokens: -1, model_uid: "", is_beta: "yes" })) { - test(`rejects invalid ${field} before model conversion`, () => { - const malformed = structuredClone(catalog); - malformed.families[0].variants[0][field] = value; - assert.equal(isUsableCatalog(malformed), false); - }); -} diff --git a/test/catalog-startup.test.mjs b/test/catalog-startup.test.mjs deleted file mode 100644 index 6243f24..0000000 --- a/test/catalog-startup.test.mjs +++ /dev/null @@ -1,101 +0,0 @@ -import assert from "node:assert/strict"; -import test, { after } from "node:test"; -import fs from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { syncBuiltinESMExports } from "node:module"; -import { catalogCachePath, writeCatalogCache } from "../src/catalog-cache.ts"; - -const directory = fs.mkdtempSync(join(tmpdir(), "pi-devin-catalog-test-")); -after(() => fs.rmSync(directory, { recursive: true, force: true })); -const cli = join(directory, process.platform === "win32" ? "devin.cmd" : "devin"); -const counter = join(directory, "calls"); -const catalog = { families: [{ family_label: "Test", family_uid: "test", slug: "test", variants: [{ model_uid: "test-high", label: "Test high", cost_summary: "$1/MTok In, $2/MTok Out" }] }] }; -const script = join(directory, "catalog.cjs"); -fs.writeFileSync(script, `const fs = require('node:fs');\nfs.appendFileSync(${JSON.stringify(counter)}, 'call\\n');\nsetTimeout(() => { console.log(process.env.TEST_CATALOG); process.exit(Number(process.env.TEST_CLI_EXIT || 0)); }, 30);\n`); -fs.writeFileSync(cli, process.platform === "win32" ? `@echo off\r\n"${process.execPath}" "${script}"\r\n` : `#!/bin/sh\nexec "${process.execPath}" "${script}"\n`, { mode: 0o700 }); -const originalEnv = { ...process.env }; -process.env.DEVIN_CLI = cli; -const { default: extension } = await import("../extensions/index.ts"); -const { clearDevinBinCache } = await import("../src/cli.ts"); -after(() => { if (originalEnv.DEVIN_CLI === undefined) delete process.env.DEVIN_CLI; else process.env.DEVIN_CLI = originalEnv.DEVIN_CLI; }); - -function setup(t) { - clearDevinBinCache(); - const root = fs.mkdtempSync(join(directory, "cache-")); - const previous = { ...process.env }; - process.env.XDG_CACHE_HOME = root; - process.env.TEST_CATALOG = JSON.stringify(catalog); - delete process.env.PI_OFFLINE; - delete process.env.TEST_CLI_EXIT; - fs.writeFileSync(counter, ""); - const read = fs.readFileSync, exists = fs.existsSync; - const isCredentials = (path) => String(path).replaceAll("\\", "/").endsWith("devin/credentials.toml"); - t.mock.method(fs, "readFileSync", (path, ...args) => isCredentials(path) ? 'api_key = "synthetic-test-key"\n' : read(path, ...args)); - t.mock.method(fs, "existsSync", (path) => isCredentials(path) || exists(path)); - t.mock.method(console, "warn", () => {}); - syncBuiltinESMExports(); - const registrations = [], hooks = {}, commands = {}; - const pi = { registerProvider: (_id, config) => registrations.push(config), on: (name, fn) => hooks[name] = fn, registerCommand: (name, command) => commands[name] = command }; - t.after(async () => { - await hooks.session_shutdown?.(); - t.mock.restoreAll(); syncBuiltinESMExports(); - for (const key of ["XDG_CACHE_HOME", "PI_OFFLINE", "TEST_CATALOG", "TEST_CLI_EXIT"]) { - if (previous[key] === undefined) delete process.env[key]; else process.env[key] = previous[key]; - } - }); - return { registrations, hooks, commands, pi, path: catalogCachePath(), calls: () => fs.readFileSync(counter, "utf8").split("\n").filter(Boolean).length }; -} - -test("fresh cache registers once without invoking the CLI", async (t) => { - const x = setup(t); writeCatalogCache(catalog); - await extension(x.pi); - assert.equal(x.calls(), 0); assert.equal(x.registrations.length, 1); - assert.equal(x.registrations[0].models[0].id, "test-high"); -}); - -test("malformed cached cost data cannot prevent provider and command registration", async (t) => { - const x = setup(t); process.env.PI_OFFLINE = "1"; - const malformed = structuredClone(catalog); malformed.families[0].variants[0].cost_summary = {}; - fs.mkdirSync(join(process.env.XDG_CACHE_HOME, "pi-devin")); - fs.writeFileSync(x.path, JSON.stringify({ version: 1, fetchedAt: Date.now(), catalog: malformed })); - await extension(x.pi); - assert.equal(x.registrations.length, 1); assert.ok(x.registrations[0].models.length); - assert.ok(x.commands["devin-refresh"]); assert.equal(x.calls(), 0); -}); - -test("cold startup loads and caches the CLI catalog", async (t) => { - const x = setup(t); await extension(x.pi); - assert.equal(x.calls(), 1); assert.equal(x.registrations.at(-1).models[0].id, "test-high"); - assert.deepEqual(JSON.parse(fs.readFileSync(x.path, "utf8")).catalog, catalog); -}); - -test("stale cache shares its background request with manual refresh", async (t) => { - const x = setup(t); writeCatalogCache(catalog, x.path, 1); - await extension(x.pi); - const notifications = []; - await x.commands["devin-refresh"].handler("", { ui: { notify: (...args) => notifications.push(args) } }); - assert.equal(x.calls(), 1); assert.equal(notifications.at(-1)[1], "info"); - assert.equal(x.registrations.at(-1).models[0].id, "test-high"); -}); - -test("failed refresh retains cached models and does not overwrite the cache", async (t) => { - const x = setup(t); writeCatalogCache(catalog, x.path, 1); process.env.TEST_CATALOG = '{"families":[{}]}'; - await extension(x.pi); - const notifications = []; - await x.commands["devin-refresh"].handler("", { ui: { notify: (...args) => notifications.push(args) } }); - assert.equal(notifications.at(-1)[1], "error"); assert.equal(x.registrations.length, 1); - assert.deepEqual(JSON.parse(fs.readFileSync(x.path, "utf8")).catalog, catalog); -}); - -test("offline cold startup uses fallback without running the CLI", async (t) => { - const x = setup(t); process.env.PI_OFFLINE = "true"; - await extension(x.pi); assert.equal(x.calls(), 0); assert.ok(x.registrations[0].models.length); -}); - -test("shutdown prevents a late background result from registering a provider", async (t) => { - const x = setup(t); writeCatalogCache(catalog, x.path, 1); - await extension(x.pi); await x.hooks.session_shutdown(); - await x.commands["devin-refresh"].handler("", { ui: { notify() {} } }); - assert.equal(x.registrations.length, 1); -}); diff --git a/test/cli.test.mjs b/test/cli.test.mjs deleted file mode 100644 index ee90fd0..0000000 --- a/test/cli.test.mjs +++ /dev/null @@ -1,73 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { mkdtempSync, mkdirSync, writeFileSync, rmSync, realpathSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; -import { spawnSync } from "node:child_process"; -import { clearDevinBinCache, findDevinBin, whichDevin, runDevin } from "../src/cli.ts"; - -function fixture(t) { - const directory = mkdtempSync(join(tmpdir(), "pi devin cli ")); - // Windows environment lookups ignore case; a spread object does not. - const keys = ["DEVIN_CLI", "LOCALAPPDATA", "ProgramFiles", "PATH"]; - const original = Object.fromEntries(keys.map((key) => [key, process.env[key]])); - clearDevinBinCache(); - t.after(() => { - for (const key of keys) { - if (original[key] === undefined) delete process.env[key]; else process.env[key] = original[key]; - } - clearDevinBinCache(); rmSync(directory, { recursive: true, force: true }); - }); - return directory; -} - -test("honors an override set after the discovery cache has already been populated", async (t) => { - const directory = fixture(t); - const first = join(directory, "first"), second = join(directory, "second"); - writeFileSync(first, ""); writeFileSync(second, ""); - process.env.DEVIN_CLI = first; assert.equal(findDevinBin(), first); - process.env.DEVIN_CLI = second; assert.equal(await whichDevin(), second); -}); - -test("runs a native executable directly with argument boundaries intact", async (t) => { - fixture(t); process.env.DEVIN_CLI = process.execPath; - const value = 'spaces & punctuation "stay literal"'; - const result = await runDevin(["-e", "process.stdout.write(process.argv[1])", value]); - assert.equal(result.code, 0); assert.equal(result.stdout, value); -}); - -test("finds the Windows installer binary and ignores a directory override", { skip: process.platform !== "win32" }, (t) => { - const directory = fixture(t); process.env.LOCALAPPDATA = directory; - process.env.DEVIN_CLI = directory; - const binary = join(directory, "devin", "cli", "bin", "devin.exe"); - mkdirSync(dirname(binary), { recursive: true }); writeFileSync(binary, ""); - assert.equal(findDevinBin(), binary); -}); - -test("Windows PATH lookup uses where.exe and reads multiple results", { skip: process.platform !== "win32" }, async (t) => { - const directory = fixture(t); delete process.env.DEVIN_CLI; - process.env.LOCALAPPDATA = directory; process.env.ProgramFiles = directory; - const bin = join(directory, "path bin"); mkdirSync(bin); - const binary = join(bin, "devin.exe"); writeFileSync(binary, ""); - const otherBin = join(directory, "other bin"); mkdirSync(otherBin); - writeFileSync(join(otherBin, "devin.exe"), ""); - process.env.PATH = `${bin};${otherBin};${process.env.PATH}`; - const located = await whichDevin(); - const probe = spawnSync("where.exe", ["devin.exe"], { encoding: "utf8" }); - assert.ok(located, JSON.stringify({ status: probe.status, stdout: probe.stdout, stderr: probe.stderr, error: probe.error?.message })); - // where.exe expands Windows short paths such as RUNNER~1. - assert.equal(realpathSync.native(located), realpathSync.native(binary)); - assert.ok(probe.stdout.trim().split(/\r?\n/).length >= 2); -}); - -test("Windows batch wrappers work in paths with spaces, including inherited stdio", { skip: process.platform !== "win32" }, async (t) => { - const directory = fixture(t), script = join(directory, "echo.cjs"), wrapper = join(directory, "devin.cmd"); - writeFileSync(script, 'process.stdout.write(JSON.stringify(process.argv.slice(2)));'); - writeFileSync(wrapper, `@echo off\r\n"${process.execPath}" "${script}" %*\r\n`); - process.env.DEVIN_CLI = wrapper; - const result = await runDevin(["models", "list", "--format", "json", "a b"]); - assert.equal(result.code, 0, result.stderr); - assert.deepEqual(JSON.parse(result.stdout), ["models", "list", "--format", "json", "a b"]); - assert.equal((await runDevin(["version"], { inheritStdio: true })).code, 0); - await assert.rejects(runDevin(["%PATH%"]), /Unsupported shell characters/); -}); diff --git a/test/context-map.test.mjs b/test/context-map.test.mjs deleted file mode 100644 index 9db1e41..0000000 --- a/test/context-map.test.mjs +++ /dev/null @@ -1,53 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { normalizeContext } from "@earendil-works/pi-ai"; -import { mapContextToChat } from "../src/context-map.ts"; - -const tool = (name) => ({ - name, - description: name, - parameters: { type: "object", properties: {} }, -}); - -// Regression for #7: Pi 0.86 moves prompt and tools into system messages. -test("preserves the prompt and tools from a normalized transcript", () => { - const read = tool("read"); - const context = normalizeContext({ - systemPrompt: "Inspect the repository.", - tools: [read], - messages: [{ role: "user", content: "Read README", timestamp: 1 }], - }); - assert.deepEqual(mapContextToChat(context), { - systemPrompt: "Inspect the repository.", - messages: [ - { role: "user", content: "Read README" }, - ], - tools: [read], - }); -}); - -test("applies later instruction, section, and tool changes", () => { - const bash = tool("bash"); - const context = { - messages: [ - { role: "system", content: "Inspect the repository.", sections: { policy: "Old policy", obsolete: "Remove me" }, toolsAdded: [tool("read")], timestamp: 0 }, - { role: "user", content: "Continue", timestamp: 1 }, - { role: "system", content: "Use bash.", sections: { policy: "Do not edit.", obsolete: null }, toolsRemoved: [{ name: "read" }], toolsAdded: [bash], timestamp: 2 }, - ], - }; - assert.deepEqual(mapContextToChat(context), { - systemPrompt: "Inspect the repository.\n\nUse bash.\n\nDo not edit.", - messages: [ - { role: "user", content: "Continue" }, - ], - tools: [bash], - }); -}); - -test("accepts an empty transcript", () => { - assert.deepEqual(mapContextToChat(normalizeContext({ messages: [] })), { - systemPrompt: undefined, - messages: [], - tools: [], - }); -}); diff --git a/test/stream-abort.test.mjs b/test/stream-abort.test.mjs deleted file mode 100644 index e5d24b7..0000000 --- a/test/stream-abort.test.mjs +++ /dev/null @@ -1,104 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { normalizeContext } from "@earendil-works/pi-ai"; -import { clearCachedUserJwt } from "../src/jwt.ts"; -import { streamDevin } from "../src/stream.ts"; -import { encodeMessage } from "../src/wire.ts"; - -const model = { - id: "test-model", - name: "Test model", - api: "devin-local", - provider: "devin", - baseUrl: "https://devin.invalid", - reasoning: false, - input: ["text", "image"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 262_000, - maxTokens: 1_000, -}; -const user = (content) => ({ role: "user", content, timestamp: 1 }); - -test("aborting an in-flight stream reports aborted without leaking a rejection", async (t) => { - clearCachedUserJwt(); - t.after(clearCachedUserJwt); - - const rejections = []; - const onRejection = (reason) => rejections.push(reason); - process.on("unhandledRejection", onRejection); - t.after(() => process.off("unhandledRejection", onRejection)); - - const ac = new AbortController(); - t.mock.method(globalThis, "fetch", async (url, options) => { - if (url === "https://devin.invalid/exa.auth_pb.AuthService/GetUserJwt") { - return new Response(encodeMessage(1, Buffer.from("eyJtest.jwt"))); - } - assert.equal(url, "https://devin.invalid/exa.api_server_pb.ApiServerService/GetChatMessage"); - // Mirror undici: an aborted fetch errors the response body with signal.reason. - const body = new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode("partial")); - const onAbort = () => controller.error(options.signal.reason); - if (options.signal?.aborted) onAbort(); - else options.signal?.addEventListener("abort", onAbort, { once: true }); - }, - }); - return new Response(body); - }); - - const stream = streamDevin(model, normalizeContext({ messages: [user("hello")] }), { - apiKey: "synthetic-test-key", - env: { DEVIN_API_SERVER_URL: "https://devin.invalid" }, - signal: ac.signal, - }); - - const events = []; - const consume = (async () => { - for await (const event of stream) events.push(event); - })(); - ac.abort(); - await consume; - // Give any floating rejection a chance to surface. - await new Promise((resolve) => setTimeout(resolve, 50)); - - const last = events.at(-1); - assert.equal(last?.type, "error"); - assert.equal(last?.reason, "aborted"); - assert.equal((await stream.result()).stopReason, "aborted"); - assert.deepEqual(rejections, []); -}); - -test("real fetch cancellation after a response chunk leaves the process usable", { timeout: 10_000 }, async (t) => { - const { createServer } = await import("node:http"); - const { once } = await import("node:events"); - const { encodeString, encodeVarintField, frameConnectStream } = await import("../src/wire.ts"); - clearCachedUserJwt(); t.after(clearCachedUserJwt); - let chats = 0; - const server = createServer((req, res) => { - if (req.url.endsWith("/GetUserJwt")) { - res.end(encodeString(1, "eyJtest.jwt")); return; - } - chats++; - res.writeHead(200, { "Content-Type": "application/connect+proto" }); - res.write(frameConnectStream(encodeString(3, "started"), false)); - if (chats > 1) { - res.write(frameConnectStream(encodeVarintField(5, 0), false)); - res.end(Buffer.from([2, 0, 0, 0, 2, 123, 125])); - } - }); - t.after(() => { server.closeAllConnections(); server.close(); }); - server.listen(0, "127.0.0.1"); await once(server, "listening"); - const ac = new AbortController(); - const options = { apiKey: "synthetic-test-key", env: { DEVIN_API_SERVER_URL: `http://127.0.0.1:${server.address().port}` } }; - const stream = streamDevin(model, normalizeContext({ messages: [user("hello")] }), { ...options, signal: ac.signal }); - let sawText = false; - for await (const event of stream) { - if (event.type === "text_delta") { sawText = true; ac.abort(); } - } - assert.equal(sawText, true); - assert.equal((await stream.result()).stopReason, "aborted"); - const next = streamDevin(model, normalizeContext({ messages: [user("again")] }), options); - for await (const event of next) assert.notEqual(event.type, "error"); - assert.equal((await next.result()).stopReason, "stop"); - assert.equal(chats, 2); -}); diff --git a/test/stream-deltas.test.mjs b/test/stream-deltas.test.mjs deleted file mode 100644 index a5a74e5..0000000 --- a/test/stream-deltas.test.mjs +++ /dev/null @@ -1,86 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { setImmediate } from "node:timers/promises"; -import { streamDevin } from "../src/stream.ts"; -import { clearCachedUserJwt } from "../src/jwt.ts"; -import { encodeMessage, encodeString, encodeTag, encodeVarintField, frameConnectStream } from "../src/wire.ts"; - -const model = { - id: "test-model", name: "Test", api: "devin-local", provider: "devin", - baseUrl: "https://devin.invalid", reasoning: false, input: ["text"], - cost: { input: 1, output: 2, cacheRead: 0.1, cacheWrite: 1.25 }, - contextWindow: 1_000_000, maxTokens: 1000, -}; - -async function run(t, frames) { - clearCachedUserJwt(); - t.after(clearCachedUserJwt); - t.mock.method(globalThis, "fetch", async (url) => { - if (url === "https://devin.invalid/exa.auth_pb.AuthService/GetUserJwt") { - return new Response(encodeString(1, "eyJtest.jwt")); - } - assert.equal(url, "https://devin.invalid/exa.api_server_pb.ApiServerService/GetChatMessage"); - const chunks = [...frames.map((frame) => frameConnectStream(frame, false)), Buffer.from([2, 0, 0, 0, 2, 123, 125])]; - return new Response(new ReadableStream({ - async pull(controller) { - await setImmediate(); - if (chunks.length) controller.enqueue(chunks.shift()); - else controller.close(); - }, - })); - }); - const stream = streamDevin(model, { messages: [] }, { - apiKey: "synthetic-test-key", env: { DEVIN_API_SERVER_URL: "https://devin.invalid" }, - }); - const events = []; - for await (const event of stream) events.push(structuredClone(event)); - const result = await stream.result(); - assert.notEqual(result.stopReason, "error", result.errorMessage); - return { events, result }; -} - -function usage(metrics) { - return encodeMessage(28, Buffer.concat(Object.entries(metrics).map(([name, value]) => { - const float = Buffer.alloc(4); - float.writeFloatLE(value); - return encodeMessage(2, Buffer.concat([ - encodeString(5, name), encodeMessage(4, Buffer.concat([encodeTag(2, 5), float])), - ])); - }))); -} - -const metrics = { input_tokens: 461, output_tokens: 262, cached_input_tokens: 228_162, cache_creation_input_tokens: 17 }; - -test("counts all four usage components and preserves them after an empty frame (#3, #4)", async (t) => { - const { result } = await run(t, [usage(metrics), usage(Object.fromEntries(Object.keys(metrics).map((key) => [key, 0])))]); - assert.deepEqual({ ...result.usage, cost: undefined }, { - input: 461, output: 262, cacheRead: 228_162, cacheWrite: 17, totalTokens: 228_902, cost: undefined, - }); - assert.equal(result.usage.cost.input, 461 / 1_000_000); - assert.equal(result.usage.cost.cacheRead, 228_162 * 0.1 / 1_000_000); -}); - -test("accepts cache-only usage and combines separate component snapshots without double counting", async (t) => { - const { result } = await run(t, [ - usage({ cached_input_tokens: 58_905 }), usage({ input_tokens: 280, output_tokens: 118 }), - usage({ input_tokens: 280, output_tokens: 118 }), usage({ cache_creation_input_tokens: 17 }), - ]); - assert.equal(result.usage.totalTokens, 59_320); - assert.equal(result.usage.cacheRead, 58_905); - assert.equal(result.usage.cacheWrite, 17); -}); - -test("keeps growing tool arguments across split keys, strings, and Unicode escapes (#2)", async (t) => { - const fragments = ['{"path":"file.txt","cont', 'ent":"hello ', '\\u26', '3a"}']; - const frames = fragments.map((part, index) => encodeMessage(6, Buffer.concat([ - ...(index === 0 ? [encodeString(1, "call-1"), encodeString(2, "write")] : []), encodeString(3, part), - ]))); - frames.push(encodeVarintField(5, 10)); - const { events, result } = await run(t, frames); - const deltas = events.filter((event) => event.type === "toolcall_delta"); - assert.equal(deltas.length, 4); - for (const event of deltas) assert.equal(event.partial.content[0].arguments.path, "file.txt"); - assert.equal(deltas[1].partial.content[0].arguments.content, "hello"); - assert.deepEqual(result.content[0].arguments, { path: "file.txt", content: "hello ☺" }); - assert.equal(result.stopReason, "toolUse"); -}); diff --git a/test/stream-request.test.mjs b/test/stream-request.test.mjs deleted file mode 100644 index 9537bfb..0000000 --- a/test/stream-request.test.mjs +++ /dev/null @@ -1,164 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { gunzipSync } from "node:zlib"; -import { normalizeContext } from "@earendil-works/pi-ai"; -import { clearCachedUserJwt } from "../src/jwt.ts"; -import { streamDevin } from "../src/stream.ts"; -import { encodeMessage, encodeString, encodeVarintField, frameConnectStream, iterFields } from "../src/wire.ts"; - -const model = { - id: "test-model", - name: "Test model", - api: "devin-local", - provider: "devin", - baseUrl: "https://devin.invalid", - reasoning: false, - input: ["text", "image"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 262_000, - maxTokens: 1_000, -}; -const readTool = { - name: "read", - description: "Read a file", - parameters: { type: "object", properties: { path: { type: "string" } }, required: ["path"] }, -}; -const user = (content) => ({ role: "user", content, timestamp: 1 }); -const fields = (buffer) => [...iterFields(buffer)]; -const messages = (request) => request.filter((field) => field.num === 3).map((field) => fields(field.value)); -const stringField = (fields, number) => fields.find((field) => field.num === number)?.value.toString("utf8"); - -function response(body) { - // Connect's end-of-stream frame contains the JSON trailer {}. - return new Response(Buffer.concat([frameConnectStream(body, false), Buffer.from([2, 0, 0, 0, 2, 123, 125])])); -} - -function mockDevin(t, replies = [Buffer.concat([encodeString(3, "OK"), encodeVarintField(5, 0)])]) { - const requests = []; - clearCachedUserJwt(); - t.after(clearCachedUserJwt); - t.mock.method(globalThis, "fetch", async (url, options) => { - if (url === "https://devin.invalid/exa.auth_pb.AuthService/GetUserJwt") { - return new Response(encodeMessage(1, Buffer.from("eyJtest.jwt"))); - } - assert.equal(url, "https://devin.invalid/exa.api_server_pb.ApiServerService/GetChatMessage"); - const frame = Buffer.from(options.body); - assert.equal(frame[0], 1, "request uses Connect gzip compression"); - assert.equal(frame.readUInt32BE(1), frame.length - 5); - requests.push(fields(gunzipSync(frame.subarray(5)))); - const reply = replies[requests.length - 1]; - assert.ok(reply, "unexpected extra chat request"); - return response(reply); - }); - return requests; -} - -async function complete(context) { - const stream = streamDevin(model, context, { - apiKey: "synthetic-test-key", - env: { DEVIN_API_SERVER_URL: "https://devin.invalid" }, - }); - for await (const event of stream) { - assert.notEqual(event.type, "error", event.error?.errorMessage); - } - return stream.result(); -} - -// #7: verify the normalized transcript survives the complete request-encoding path. -test("encodes the current system prompt once in field 2, with unchanged user text and images", async (t) => { - const requests = mockDevin(t); - const text = " Inspect this: literal user text\nこんにちは "; - const image = { type: "image", mimeType: "image/png", data: "aW1hZ2UtZml4dHVyZQ==" }; - const context = normalizeContext({ - systemPrompt: "Initial instructions.", - tools: [readTool], - messages: [user([{ type: "text", text }, image])], - }); - const tools = Array.from({ length: 90 }, (_, i) => ({ ...readTool, name: `read_${i}` })); - context.messages.push({ - role: "system", - content: "Updated instructions.", - sections: { policy: "Do not edit." }, - toolsRemoved: [{ name: "read" }], - toolsAdded: tools, - timestamp: 2, - }); - await complete(context); - assert.equal(requests.length, 1); - const request = requests[0]; - const promptFields = request.filter((field) => field.num === 2); - assert.equal(promptFields.length, 1); - assert.equal(promptFields[0].wire, 2); - assert.equal(promptFields[0].value.toString(), "Initial instructions.\n\nUpdated instructions.\n\nDo not edit."); - const history = messages(request); - assert.equal(history.length, 1); - assert.equal(history[0].find((field) => field.num === 2).value, 1n); - assert.equal(stringField(history[0], 3), text); - const images = history[0].filter((field) => field.num === 10); - assert.equal(images.length, 1); - assert.equal(stringField(fields(images[0].value), 1), image.data); - assert.equal(stringField(fields(images[0].value), 2), image.mimeType); - assert.deepEqual(request.filter((field) => field.num === 10).map((field) => { - const definition = fields(field.value); - return { name: stringField(definition, 1), description: stringField(definition, 2), parameters: JSON.parse(stringField(definition, 3)) }; - }), tools); -}); - -test("omits an empty system prompt while preserving tools and user content", async (t) => { - const requests = mockDevin(t); - await complete(normalizeContext({ systemPrompt: "", tools: [readTool], messages: [user("Read probe.txt")] })); - assert.equal(requests[0].filter((field) => field.num === 2).length, 0); - assert.equal(requests[0].filter((field) => field.num === 10).length, 1); - assert.deepEqual(messages(requests[0]).map((message) => stringField(message, 3)), ["Read probe.txt"]); -}); - -test("sends a system-only transcript without inventing a user message", async (t) => { - const requests = mockDevin(t); - await complete(normalizeContext({ systemPrompt: "System-only instructions.", messages: [] })); - assert.equal(stringField(requests[0], 2), "System-only instructions."); - assert.deepEqual(messages(requests[0]), []); -}); - -test("encodes an empty transcript without a prompt or history", async (t) => { - const requests = mockDevin(t); - await complete(normalizeContext({ messages: [] })); - assert.equal(stringField(requests[0], 2), undefined); - assert.deepEqual(messages(requests[0]), []); -}); - -test("retains the system prompt and replays assistant tool calls and results on the next request", async (t) => { - const callId = "call-read"; - const args = { path: "probe.txt" }; - const requests = mockDevin(t, [ - Buffer.concat([ - encodeMessage(6, Buffer.concat([encodeString(1, callId), encodeString(2, "read"), encodeString(3, JSON.stringify(args))])), - encodeVarintField(5, 10), - ]), - Buffer.concat([encodeString(3, "file-value"), encodeVarintField(5, 0)]), - ]); - const context = normalizeContext({ systemPrompt: "Use tools; never guess.", tools: [readTool], messages: [user("Read probe.txt")] }); - const first = await complete(context); - assert.equal(first.stopReason, "toolUse"); - assert.deepEqual(first.content, [{ type: "toolCall", id: callId, name: "read", arguments: args }]); - context.messages.push(first, { - role: "toolResult", toolCallId: callId, toolName: "read", - content: [{ type: "text", text: "file-value" }], isError: false, timestamp: 3, - }); - const second = await complete(context); - assert.equal(second.stopReason, "stop"); - assert.equal(second.content[0].text, "file-value"); - assert.equal(requests.length, 2); - for (const request of requests) { - assert.equal(request.filter((field) => field.num === 2).length, 1); - assert.equal(stringField(request, 2), "Use tools; never guess."); - } - const history = messages(requests[1]); - assert.deepEqual(history.map((message) => message.find((field) => field.num === 2).value), [1n, 2n, 4n]); - assert.equal(stringField(history[0], 3), "Read probe.txt"); - const call = fields(history[1].find((field) => field.num === 6).value); - assert.equal(stringField(call, 1), callId); - assert.equal(stringField(call, 2), "read"); - assert.deepEqual(JSON.parse(stringField(call, 3)), args); - assert.equal(stringField(history[2], 7), callId); - assert.equal(stringField(history[2], 3), "file-value"); -}); diff --git a/tsconfig.json b/tsconfig.json deleted file mode 100644 index a70c37a..0000000 --- a/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "strict": true, - "skipLibCheck": true, - "noEmit": true, - "types": ["node"] - }, - "include": ["extensions/**/*.ts", "src/**/*.ts"] -}