fix: send system instructions in Devin's dedicated prompt field

2026-09-22 15:38:37 -07:00
parent f42b776b8e
commit 3b40b4020d
5 changed files with 188 additions and 51 deletions
@@ -0,0 +1,164 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { gunzipSync } from "node:zlib";
+import { normalizeContext } from "@earendil-works/pi-ai";
+import { clearCachedUserJwt } from "../src/jwt.ts";
+import { streamDevin } from "../src/stream.ts";
+import { encodeMessage, encodeString, encodeVarintField, frameConnectStream, iterFields } from "../src/wire.ts";
+
+const model = {
+  id: "test-model",
+  name: "Test model",
+  api: "devin-local",
+  provider: "devin",
+  baseUrl: "https://devin.invalid",
+  reasoning: false,
+  input: ["text", "image"],
+  cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+  contextWindow: 262_000,
+  maxTokens: 1_000,
+};
+const readTool = {
+  name: "read",
+  description: "Read a file",
+  parameters: { type: "object", properties: { path: { type: "string" } }, required: ["path"] },
+};
+const user = (content) => ({ role: "user", content, timestamp: 1 });
+const fields = (buffer) => [...iterFields(buffer)];
+const messages = (request) => request.filter((field) => field.num === 3).map((field) => fields(field.value));
+const stringField = (fields, number) => fields.find((field) => field.num === number)?.value.toString("utf8");
+
+function response(body) {
+  // Connect's end-of-stream frame contains the JSON trailer {}.
+  return new Response(Buffer.concat([frameConnectStream(body, false), Buffer.from([2, 0, 0, 0, 2, 123, 125])]));
+}
+
+function mockDevin(t, replies = [Buffer.concat([encodeString(3, "OK"), encodeVarintField(5, 0)])]) {
+  const requests = [];
+  clearCachedUserJwt();
+  t.after(clearCachedUserJwt);
+  t.mock.method(globalThis, "fetch", async (url, options) => {
+    if (url === "https://devin.invalid/exa.auth_pb.AuthService/GetUserJwt") {
+      return new Response(encodeMessage(1, Buffer.from("eyJtest.jwt")));
+    }
+    assert.equal(url, "https://devin.invalid/exa.api_server_pb.ApiServerService/GetChatMessage");
+    const frame = Buffer.from(options.body);
+    assert.equal(frame[0], 1, "request uses Connect gzip compression");
+    assert.equal(frame.readUInt32BE(1), frame.length - 5);
+    requests.push(fields(gunzipSync(frame.subarray(5))));
+    const reply = replies[requests.length - 1];
+    assert.ok(reply, "unexpected extra chat request");
+    return response(reply);
+  });
+  return requests;
+}
+
+async function complete(context) {
+  const stream = streamDevin(model, context, {
+    apiKey: "synthetic-test-key",
+    env: { DEVIN_API_SERVER_URL: "https://devin.invalid" },
+  });
+  for await (const event of stream) {
+    assert.notEqual(event.type, "error", event.error?.errorMessage);
+  }
+  return stream.result();
+}
+
+// #7: verify the normalized transcript survives the complete request-encoding path.
+test("encodes the current system prompt once in field 2, with unchanged user text and images", async (t) => {
+  const requests = mockDevin(t);
+  const text = "  Inspect this: <system>literal user text</system>\nこんにちは  ";
+  const image = { type: "image", mimeType: "image/png", data: "aW1hZ2UtZml4dHVyZQ==" };
+  const context = normalizeContext({
+    systemPrompt: "Initial instructions.",
+    tools: [readTool],
+    messages: [user([{ type: "text", text }, image])],
+  });
+  const tools = Array.from({ length: 90 }, (_, i) => ({ ...readTool, name: `read_${i}` }));
+  context.messages.push({
+    role: "system",
+    content: "Updated instructions.",
+    sections: { policy: "Do not edit." },
+    toolsRemoved: [{ name: "read" }],
+    toolsAdded: tools,
+    timestamp: 2,
+  });
+  await complete(context);
+  assert.equal(requests.length, 1);
+  const request = requests[0];
+  const promptFields = request.filter((field) => field.num === 2);
+  assert.equal(promptFields.length, 1);
+  assert.equal(promptFields[0].wire, 2);
+  assert.equal(promptFields[0].value.toString(), "Initial instructions.\n\nUpdated instructions.\n\nDo not edit.");
+  const history = messages(request);
+  assert.equal(history.length, 1);
+  assert.equal(history[0].find((field) => field.num === 2).value, 1n);
+  assert.equal(stringField(history[0], 3), text);
+  const images = history[0].filter((field) => field.num === 10);
+  assert.equal(images.length, 1);
+  assert.equal(stringField(fields(images[0].value), 1), image.data);
+  assert.equal(stringField(fields(images[0].value), 2), image.mimeType);
+  assert.deepEqual(request.filter((field) => field.num === 10).map((field) => {
+    const definition = fields(field.value);
+    return { name: stringField(definition, 1), description: stringField(definition, 2), parameters: JSON.parse(stringField(definition, 3)) };
+  }), tools);
+});
+
+test("omits an empty system prompt while preserving tools and user content", async (t) => {
+  const requests = mockDevin(t);
+  await complete(normalizeContext({ systemPrompt: "", tools: [readTool], messages: [user("Read probe.txt")] }));
+  assert.equal(requests[0].filter((field) => field.num === 2).length, 0);
+  assert.equal(requests[0].filter((field) => field.num === 10).length, 1);
+  assert.deepEqual(messages(requests[0]).map((message) => stringField(message, 3)), ["Read probe.txt"]);
+});
+
+test("sends a system-only transcript without inventing a user message", async (t) => {
+  const requests = mockDevin(t);
+  await complete(normalizeContext({ systemPrompt: "System-only instructions.", messages: [] }));
+  assert.equal(stringField(requests[0], 2), "System-only instructions.");
+  assert.deepEqual(messages(requests[0]), []);
+});
+
+test("encodes an empty transcript without a prompt or history", async (t) => {
+  const requests = mockDevin(t);
+  await complete(normalizeContext({ messages: [] }));
+  assert.equal(stringField(requests[0], 2), undefined);
+  assert.deepEqual(messages(requests[0]), []);
+});
+
+test("retains the system prompt and replays assistant tool calls and results on the next request", async (t) => {
+  const callId = "call-read";
+  const args = { path: "probe.txt" };
+  const requests = mockDevin(t, [
+    Buffer.concat([
+      encodeMessage(6, Buffer.concat([encodeString(1, callId), encodeString(2, "read"), encodeString(3, JSON.stringify(args))])),
+      encodeVarintField(5, 10),
+    ]),
+    Buffer.concat([encodeString(3, "file-value"), encodeVarintField(5, 0)]),
+  ]);
+  const context = normalizeContext({ systemPrompt: "Use tools; never guess.", tools: [readTool], messages: [user("Read probe.txt")] });
+  const first = await complete(context);
+  assert.equal(first.stopReason, "toolUse");
+  assert.deepEqual(first.content, [{ type: "toolCall", id: callId, name: "read", arguments: args }]);
+  context.messages.push(first, {
+    role: "toolResult", toolCallId: callId, toolName: "read",
+    content: [{ type: "text", text: "file-value" }], isError: false, timestamp: 3,
+  });
+  const second = await complete(context);
+  assert.equal(second.stopReason, "stop");
+  assert.equal(second.content[0].text, "file-value");
+  assert.equal(requests.length, 2);
+  for (const request of requests) {
+    assert.equal(request.filter((field) => field.num === 2).length, 1);
+    assert.equal(stringField(request, 2), "Use tools; never guess.");
+  }
+  const history = messages(requests[1]);
+  assert.deepEqual(history.map((message) => message.find((field) => field.num === 2).value), [1n, 2n, 4n]);
+  assert.equal(stringField(history[0], 3), "Read probe.txt");
+  const call = fields(history[1].find((field) => field.num === 6).value);
+  assert.equal(stringField(call, 1), callId);
+  assert.equal(stringField(call, 2), "read");
+  assert.deepEqual(JSON.parse(stringField(call, 3)), args);
+  assert.equal(stringField(history[2], 7), callId);
+  assert.equal(stringField(history[2], 3), "file-value");
+});