From 3b40b4020d3e5e4ec4237b51a5629554f13ddf62 Mon Sep 17 00:00:00 2001 From: Kashyab Ambarani Date: Tue, 22 Sep 2026 15:38:37 -0700 Subject: [PATCH] fix: send system instructions in Devin's dedicated prompt field --- package.json | 3 +- src/context-map.ts | 16 ++-- src/stream.ts | 46 ++-------- test/context-map.test.mjs | 10 ++- test/stream-request.test.mjs | 164 +++++++++++++++++++++++++++++++++++ 5 files changed, 188 insertions(+), 51 deletions(-) create mode 100644 test/stream-request.test.mjs diff --git a/package.json b/package.json index 4af8261..9beb9f6 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "LICENSE" ], "scripts": { - "test": "node --test test/*.test.mjs", + "test": "tsx --test test/*.test.mjs", "typecheck": "tsc --noEmit" }, "peerDependencies": { @@ -38,6 +38,7 @@ }, "devDependencies": { "@types/node": "^24.13.3", + "tsx": "^4.23.15", "typescript": "^5.9.2" }, "engines": { diff --git a/src/context-map.ts b/src/context-map.ts index 3dc107b..d54fc3d 100644 --- a/src/context-map.ts +++ b/src/context-map.ts @@ -2,6 +2,7 @@ import { collapseSystemMessages, getCurrentSystemPrompt, getCurrentTools, + withoutInitialSystemMessage, type Message, type Tool, type TranscriptContext, @@ -15,7 +16,7 @@ export interface ContentPart { } export interface ChatHistoryItem { - role: "user" | "assistant" | "system" | "tool"; + role: "user" | "assistant" | "tool"; content: string | ContentPart[]; tool_call_id?: string; tool_calls?: Array<{ id: string; name: string; arguments: string }>; @@ -28,6 +29,8 @@ export interface ToolDef { } export interface MappedChat { + /** Sent in GetChatMessageRequest.prompt, separately from conversation history. */ + systemPrompt?: string; messages: ChatHistoryItem[]; tools: ToolDef[]; } @@ -47,16 +50,11 @@ function userContent(content: Message["content"]): string | ContentPart[] { } export function mapContextToChat(context: TranscriptContext): MappedChat { - // Devin takes one leading system message and knows no mid-conversation system - // message, so replay the transcript deltas into the leading one first. + // Fold later system updates into the single prompt sent separately to Devin. const transcript = collapseSystemMessages(context); const systemPrompt = getCurrentSystemPrompt(transcript.messages); const messages: ChatHistoryItem[] = []; - if (systemPrompt) { - messages.push({ role: "system", content: systemPrompt }); - } - - for (const message of transcript.messages) { + for (const message of withoutInitialSystemMessage(transcript.messages)) { if (message.role === "user") { messages.push({ role: "user", content: userContent(message.content) }); continue; @@ -103,5 +101,5 @@ export function mapContextToChat(context: TranscriptContext): MappedChat { parameters: tool.parameters, })); - return { messages, tools }; + return { systemPrompt: systemPrompt || undefined, messages, tools }; } diff --git a/src/stream.ts b/src/stream.ts index 8404605..fc2a050 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -23,10 +23,9 @@ import { iterFields, } from "./wire.js"; -const SOURCE_BY_ROLE: Record = { +const SOURCE_BY_ROLE: Record = { user: 1, assistant: 2, - system: 1, tool: 4, }; @@ -50,40 +49,6 @@ function normalizeContent(content: string | ContentPart[]): ContentPart[] { return content; } -function collapseSystemIntoUser(messages: ChatHistoryItem[]): ChatHistoryItem[] { - const out: ChatHistoryItem[] = []; - let pending: string[] = []; - const textOf = (content: string | ContentPart[]) => - normalizeContent(content) - .filter((part) => part.type === "text" && part.text) - .map((part) => part.text as string) - .join("\n"); - - for (const message of messages) { - if (message.role === "system") { - const text = textOf(message.content); - if (text) pending.push(text); - continue; - } - if (message.role === "user" && pending.length > 0) { - const userParts = normalizeContent(message.content); - const userText = textOf(userParts); - const images = userParts.filter((part) => part.type === "image"); - out.push({ - role: "user", - content: [{ type: "text", text: `\n${pending.join("\n\n")}\n\n${userText}` }, ...images], - }); - pending = []; - continue; - } - out.push(message); - } - if (pending.length > 0) { - out.push({ role: "user", content: `\n${pending.join("\n\n")}\n` }); - } - return out; -} - function encodeImageData(img: { mimeType?: string; base64Data?: string }): Buffer { return Buffer.concat([ encodeString(1, img.base64Data ?? ""), @@ -144,6 +109,7 @@ function buildGetChatMessageRequest(args: { apiKey: string; userJwt: string; modelUid: string; + systemPrompt?: string; messages: ChatHistoryItem[]; tools?: ToolDef[]; cascadeId: string; @@ -160,10 +126,10 @@ function buildGetChatMessageRequest(args: { requestId: args.requestId, triggerId: args.triggerId, }); - const prompts = collapseSystemIntoUser(args.messages).map((message) => + const prompts = args.messages.map((message) => encodeMessage( 3, - encodeChatMessagePrompt(normalizeContent(message.content), SOURCE_BY_ROLE[message.role] ?? 1, { + encodeChatMessagePrompt(normalizeContent(message.content), SOURCE_BY_ROLE[message.role], { toolCallId: message.role === "tool" ? message.tool_call_id : undefined, toolCalls: message.role === "assistant" ? message.tool_calls : undefined, }), @@ -171,6 +137,7 @@ function buildGetChatMessageRequest(args: { ); return Buffer.concat([ encodeMessage(1, metadata), + ...(args.systemPrompt ? [encodeString(2, args.systemPrompt)] : []), ...prompts, encodeVarintField(7, 5), encodeMessage(8, encodeCompletionConfiguration(args.maxOutputTokens)), @@ -271,6 +238,7 @@ async function* streamChatEvents(args: { apiKey: string; host: string; modelUid: string; + systemPrompt?: string; messages: ChatHistoryItem[]; tools?: ToolDef[]; maxOutputTokens?: number; @@ -283,6 +251,7 @@ async function* streamChatEvents(args: { apiKey: args.apiKey, userJwt, modelUid: args.modelUid, + systemPrompt: args.systemPrompt, messages: args.messages, tools: args.tools, cascadeId: ids.cascadeId, @@ -482,6 +451,7 @@ export function streamDevin( apiKey, host, modelUid, + systemPrompt: mapped.systemPrompt, messages: mapped.messages, tools: mapped.tools.length > 0 ? mapped.tools : undefined, maxOutputTokens: options?.maxTokens, diff --git a/test/context-map.test.mjs b/test/context-map.test.mjs index e9c99bb..9db1e41 100644 --- a/test/context-map.test.mjs +++ b/test/context-map.test.mjs @@ -18,8 +18,8 @@ test("preserves the prompt and tools from a normalized transcript", () => { messages: [{ role: "user", content: "Read README", timestamp: 1 }], }); assert.deepEqual(mapContextToChat(context), { + systemPrompt: "Inspect the repository.", messages: [ - { role: "system", content: "Inspect the repository." }, { role: "user", content: "Read README" }, ], tools: [read], @@ -36,8 +36,8 @@ test("applies later instruction, section, and tool changes", () => { ], }; assert.deepEqual(mapContextToChat(context), { + systemPrompt: "Inspect the repository.\n\nUse bash.\n\nDo not edit.", messages: [ - { role: "system", content: "Inspect the repository.\n\nUse bash.\n\nDo not edit." }, { role: "user", content: "Continue" }, ], tools: [bash], @@ -45,5 +45,9 @@ test("applies later instruction, section, and tool changes", () => { }); test("accepts an empty transcript", () => { - assert.deepEqual(mapContextToChat(normalizeContext({ messages: [] })), { messages: [], tools: [] }); + assert.deepEqual(mapContextToChat(normalizeContext({ messages: [] })), { + systemPrompt: undefined, + messages: [], + tools: [], + }); }); diff --git a/test/stream-request.test.mjs b/test/stream-request.test.mjs new file mode 100644 index 0000000..9537bfb --- /dev/null +++ b/test/stream-request.test.mjs @@ -0,0 +1,164 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { gunzipSync } from "node:zlib"; +import { normalizeContext } from "@earendil-works/pi-ai"; +import { clearCachedUserJwt } from "../src/jwt.ts"; +import { streamDevin } from "../src/stream.ts"; +import { encodeMessage, encodeString, encodeVarintField, frameConnectStream, iterFields } from "../src/wire.ts"; + +const model = { + id: "test-model", + name: "Test model", + api: "devin-local", + provider: "devin", + baseUrl: "https://devin.invalid", + reasoning: false, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 262_000, + maxTokens: 1_000, +}; +const readTool = { + name: "read", + description: "Read a file", + parameters: { type: "object", properties: { path: { type: "string" } }, required: ["path"] }, +}; +const user = (content) => ({ role: "user", content, timestamp: 1 }); +const fields = (buffer) => [...iterFields(buffer)]; +const messages = (request) => request.filter((field) => field.num === 3).map((field) => fields(field.value)); +const stringField = (fields, number) => fields.find((field) => field.num === number)?.value.toString("utf8"); + +function response(body) { + // Connect's end-of-stream frame contains the JSON trailer {}. + return new Response(Buffer.concat([frameConnectStream(body, false), Buffer.from([2, 0, 0, 0, 2, 123, 125])])); +} + +function mockDevin(t, replies = [Buffer.concat([encodeString(3, "OK"), encodeVarintField(5, 0)])]) { + const requests = []; + clearCachedUserJwt(); + t.after(clearCachedUserJwt); + t.mock.method(globalThis, "fetch", async (url, options) => { + if (url === "https://devin.invalid/exa.auth_pb.AuthService/GetUserJwt") { + return new Response(encodeMessage(1, Buffer.from("eyJtest.jwt"))); + } + assert.equal(url, "https://devin.invalid/exa.api_server_pb.ApiServerService/GetChatMessage"); + const frame = Buffer.from(options.body); + assert.equal(frame[0], 1, "request uses Connect gzip compression"); + assert.equal(frame.readUInt32BE(1), frame.length - 5); + requests.push(fields(gunzipSync(frame.subarray(5)))); + const reply = replies[requests.length - 1]; + assert.ok(reply, "unexpected extra chat request"); + return response(reply); + }); + return requests; +} + +async function complete(context) { + const stream = streamDevin(model, context, { + apiKey: "synthetic-test-key", + env: { DEVIN_API_SERVER_URL: "https://devin.invalid" }, + }); + for await (const event of stream) { + assert.notEqual(event.type, "error", event.error?.errorMessage); + } + return stream.result(); +} + +// #7: verify the normalized transcript survives the complete request-encoding path. +test("encodes the current system prompt once in field 2, with unchanged user text and images", async (t) => { + const requests = mockDevin(t); + const text = " Inspect this: literal user text\nこんにちは "; + const image = { type: "image", mimeType: "image/png", data: "aW1hZ2UtZml4dHVyZQ==" }; + const context = normalizeContext({ + systemPrompt: "Initial instructions.", + tools: [readTool], + messages: [user([{ type: "text", text }, image])], + }); + const tools = Array.from({ length: 90 }, (_, i) => ({ ...readTool, name: `read_${i}` })); + context.messages.push({ + role: "system", + content: "Updated instructions.", + sections: { policy: "Do not edit." }, + toolsRemoved: [{ name: "read" }], + toolsAdded: tools, + timestamp: 2, + }); + await complete(context); + assert.equal(requests.length, 1); + const request = requests[0]; + const promptFields = request.filter((field) => field.num === 2); + assert.equal(promptFields.length, 1); + assert.equal(promptFields[0].wire, 2); + assert.equal(promptFields[0].value.toString(), "Initial instructions.\n\nUpdated instructions.\n\nDo not edit."); + const history = messages(request); + assert.equal(history.length, 1); + assert.equal(history[0].find((field) => field.num === 2).value, 1n); + assert.equal(stringField(history[0], 3), text); + const images = history[0].filter((field) => field.num === 10); + assert.equal(images.length, 1); + assert.equal(stringField(fields(images[0].value), 1), image.data); + assert.equal(stringField(fields(images[0].value), 2), image.mimeType); + assert.deepEqual(request.filter((field) => field.num === 10).map((field) => { + const definition = fields(field.value); + return { name: stringField(definition, 1), description: stringField(definition, 2), parameters: JSON.parse(stringField(definition, 3)) }; + }), tools); +}); + +test("omits an empty system prompt while preserving tools and user content", async (t) => { + const requests = mockDevin(t); + await complete(normalizeContext({ systemPrompt: "", tools: [readTool], messages: [user("Read probe.txt")] })); + assert.equal(requests[0].filter((field) => field.num === 2).length, 0); + assert.equal(requests[0].filter((field) => field.num === 10).length, 1); + assert.deepEqual(messages(requests[0]).map((message) => stringField(message, 3)), ["Read probe.txt"]); +}); + +test("sends a system-only transcript without inventing a user message", async (t) => { + const requests = mockDevin(t); + await complete(normalizeContext({ systemPrompt: "System-only instructions.", messages: [] })); + assert.equal(stringField(requests[0], 2), "System-only instructions."); + assert.deepEqual(messages(requests[0]), []); +}); + +test("encodes an empty transcript without a prompt or history", async (t) => { + const requests = mockDevin(t); + await complete(normalizeContext({ messages: [] })); + assert.equal(stringField(requests[0], 2), undefined); + assert.deepEqual(messages(requests[0]), []); +}); + +test("retains the system prompt and replays assistant tool calls and results on the next request", async (t) => { + const callId = "call-read"; + const args = { path: "probe.txt" }; + const requests = mockDevin(t, [ + Buffer.concat([ + encodeMessage(6, Buffer.concat([encodeString(1, callId), encodeString(2, "read"), encodeString(3, JSON.stringify(args))])), + encodeVarintField(5, 10), + ]), + Buffer.concat([encodeString(3, "file-value"), encodeVarintField(5, 0)]), + ]); + const context = normalizeContext({ systemPrompt: "Use tools; never guess.", tools: [readTool], messages: [user("Read probe.txt")] }); + const first = await complete(context); + assert.equal(first.stopReason, "toolUse"); + assert.deepEqual(first.content, [{ type: "toolCall", id: callId, name: "read", arguments: args }]); + context.messages.push(first, { + role: "toolResult", toolCallId: callId, toolName: "read", + content: [{ type: "text", text: "file-value" }], isError: false, timestamp: 3, + }); + const second = await complete(context); + assert.equal(second.stopReason, "stop"); + assert.equal(second.content[0].text, "file-value"); + assert.equal(requests.length, 2); + for (const request of requests) { + assert.equal(request.filter((field) => field.num === 2).length, 1); + assert.equal(stringField(request, 2), "Use tools; never guess."); + } + const history = messages(requests[1]); + assert.deepEqual(history.map((message) => message.find((field) => field.num === 2).value), [1n, 2n, 4n]); + assert.equal(stringField(history[0], 3), "Read probe.txt"); + const call = fields(history[1].find((field) => field.num === 6).value); + assert.equal(stringField(call, 1), callId); + assert.equal(stringField(call, 2), "read"); + assert.deepEqual(JSON.parse(stringField(call, 3)), args); + assert.equal(stringField(history[2], 7), callId); + assert.equal(stringField(history[2], 3), "file-value"); +});