165 lines
7.4 KiB
JavaScript
165 lines
7.4 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
import { gunzipSync } from "node:zlib";
|
|
import { normalizeContext } from "@earendil-works/pi-ai";
|
|
import { clearCachedUserJwt } from "../src/jwt.ts";
|
|
import { streamDevin } from "../src/stream.ts";
|
|
import { encodeMessage, encodeString, encodeVarintField, frameConnectStream, iterFields } from "../src/wire.ts";
|
|
|
|
const model = {
|
|
id: "test-model",
|
|
name: "Test model",
|
|
api: "devin-local",
|
|
provider: "devin",
|
|
baseUrl: "https://devin.invalid",
|
|
reasoning: false,
|
|
input: ["text", "image"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 262_000,
|
|
maxTokens: 1_000,
|
|
};
|
|
const readTool = {
|
|
name: "read",
|
|
description: "Read a file",
|
|
parameters: { type: "object", properties: { path: { type: "string" } }, required: ["path"] },
|
|
};
|
|
const user = (content) => ({ role: "user", content, timestamp: 1 });
|
|
const fields = (buffer) => [...iterFields(buffer)];
|
|
const messages = (request) => request.filter((field) => field.num === 3).map((field) => fields(field.value));
|
|
const stringField = (fields, number) => fields.find((field) => field.num === number)?.value.toString("utf8");
|
|
|
|
function response(body) {
|
|
// Connect's end-of-stream frame contains the JSON trailer {}.
|
|
return new Response(Buffer.concat([frameConnectStream(body, false), Buffer.from([2, 0, 0, 0, 2, 123, 125])]));
|
|
}
|
|
|
|
function mockDevin(t, replies = [Buffer.concat([encodeString(3, "OK"), encodeVarintField(5, 0)])]) {
|
|
const requests = [];
|
|
clearCachedUserJwt();
|
|
t.after(clearCachedUserJwt);
|
|
t.mock.method(globalThis, "fetch", async (url, options) => {
|
|
if (url === "https://devin.invalid/exa.auth_pb.AuthService/GetUserJwt") {
|
|
return new Response(encodeMessage(1, Buffer.from("eyJtest.jwt")));
|
|
}
|
|
assert.equal(url, "https://devin.invalid/exa.api_server_pb.ApiServerService/GetChatMessage");
|
|
const frame = Buffer.from(options.body);
|
|
assert.equal(frame[0], 1, "request uses Connect gzip compression");
|
|
assert.equal(frame.readUInt32BE(1), frame.length - 5);
|
|
requests.push(fields(gunzipSync(frame.subarray(5))));
|
|
const reply = replies[requests.length - 1];
|
|
assert.ok(reply, "unexpected extra chat request");
|
|
return response(reply);
|
|
});
|
|
return requests;
|
|
}
|
|
|
|
async function complete(context) {
|
|
const stream = streamDevin(model, context, {
|
|
apiKey: "synthetic-test-key",
|
|
env: { DEVIN_API_SERVER_URL: "https://devin.invalid" },
|
|
});
|
|
for await (const event of stream) {
|
|
assert.notEqual(event.type, "error", event.error?.errorMessage);
|
|
}
|
|
return stream.result();
|
|
}
|
|
|
|
// #7: verify the normalized transcript survives the complete request-encoding path.
|
|
test("encodes the current system prompt once in field 2, with unchanged user text and images", async (t) => {
|
|
const requests = mockDevin(t);
|
|
const text = " Inspect this: <system>literal user text</system>\nこんにちは ";
|
|
const image = { type: "image", mimeType: "image/png", data: "aW1hZ2UtZml4dHVyZQ==" };
|
|
const context = normalizeContext({
|
|
systemPrompt: "Initial instructions.",
|
|
tools: [readTool],
|
|
messages: [user([{ type: "text", text }, image])],
|
|
});
|
|
const tools = Array.from({ length: 90 }, (_, i) => ({ ...readTool, name: `read_${i}` }));
|
|
context.messages.push({
|
|
role: "system",
|
|
content: "Updated instructions.",
|
|
sections: { policy: "Do not edit." },
|
|
toolsRemoved: [{ name: "read" }],
|
|
toolsAdded: tools,
|
|
timestamp: 2,
|
|
});
|
|
await complete(context);
|
|
assert.equal(requests.length, 1);
|
|
const request = requests[0];
|
|
const promptFields = request.filter((field) => field.num === 2);
|
|
assert.equal(promptFields.length, 1);
|
|
assert.equal(promptFields[0].wire, 2);
|
|
assert.equal(promptFields[0].value.toString(), "Initial instructions.\n\nUpdated instructions.\n\nDo not edit.");
|
|
const history = messages(request);
|
|
assert.equal(history.length, 1);
|
|
assert.equal(history[0].find((field) => field.num === 2).value, 1n);
|
|
assert.equal(stringField(history[0], 3), text);
|
|
const images = history[0].filter((field) => field.num === 10);
|
|
assert.equal(images.length, 1);
|
|
assert.equal(stringField(fields(images[0].value), 1), image.data);
|
|
assert.equal(stringField(fields(images[0].value), 2), image.mimeType);
|
|
assert.deepEqual(request.filter((field) => field.num === 10).map((field) => {
|
|
const definition = fields(field.value);
|
|
return { name: stringField(definition, 1), description: stringField(definition, 2), parameters: JSON.parse(stringField(definition, 3)) };
|
|
}), tools);
|
|
});
|
|
|
|
test("omits an empty system prompt while preserving tools and user content", async (t) => {
|
|
const requests = mockDevin(t);
|
|
await complete(normalizeContext({ systemPrompt: "", tools: [readTool], messages: [user("Read probe.txt")] }));
|
|
assert.equal(requests[0].filter((field) => field.num === 2).length, 0);
|
|
assert.equal(requests[0].filter((field) => field.num === 10).length, 1);
|
|
assert.deepEqual(messages(requests[0]).map((message) => stringField(message, 3)), ["Read probe.txt"]);
|
|
});
|
|
|
|
test("sends a system-only transcript without inventing a user message", async (t) => {
|
|
const requests = mockDevin(t);
|
|
await complete(normalizeContext({ systemPrompt: "System-only instructions.", messages: [] }));
|
|
assert.equal(stringField(requests[0], 2), "System-only instructions.");
|
|
assert.deepEqual(messages(requests[0]), []);
|
|
});
|
|
|
|
test("encodes an empty transcript without a prompt or history", async (t) => {
|
|
const requests = mockDevin(t);
|
|
await complete(normalizeContext({ messages: [] }));
|
|
assert.equal(stringField(requests[0], 2), undefined);
|
|
assert.deepEqual(messages(requests[0]), []);
|
|
});
|
|
|
|
test("retains the system prompt and replays assistant tool calls and results on the next request", async (t) => {
|
|
const callId = "call-read";
|
|
const args = { path: "probe.txt" };
|
|
const requests = mockDevin(t, [
|
|
Buffer.concat([
|
|
encodeMessage(6, Buffer.concat([encodeString(1, callId), encodeString(2, "read"), encodeString(3, JSON.stringify(args))])),
|
|
encodeVarintField(5, 10),
|
|
]),
|
|
Buffer.concat([encodeString(3, "file-value"), encodeVarintField(5, 0)]),
|
|
]);
|
|
const context = normalizeContext({ systemPrompt: "Use tools; never guess.", tools: [readTool], messages: [user("Read probe.txt")] });
|
|
const first = await complete(context);
|
|
assert.equal(first.stopReason, "toolUse");
|
|
assert.deepEqual(first.content, [{ type: "toolCall", id: callId, name: "read", arguments: args }]);
|
|
context.messages.push(first, {
|
|
role: "toolResult", toolCallId: callId, toolName: "read",
|
|
content: [{ type: "text", text: "file-value" }], isError: false, timestamp: 3,
|
|
});
|
|
const second = await complete(context);
|
|
assert.equal(second.stopReason, "stop");
|
|
assert.equal(second.content[0].text, "file-value");
|
|
assert.equal(requests.length, 2);
|
|
for (const request of requests) {
|
|
assert.equal(request.filter((field) => field.num === 2).length, 1);
|
|
assert.equal(stringField(request, 2), "Use tools; never guess.");
|
|
}
|
|
const history = messages(requests[1]);
|
|
assert.deepEqual(history.map((message) => message.find((field) => field.num === 2).value), [1n, 2n, 4n]);
|
|
assert.equal(stringField(history[0], 3), "Read probe.txt");
|
|
const call = fields(history[1].find((field) => field.num === 6).value);
|
|
assert.equal(stringField(call, 1), callId);
|
|
assert.equal(stringField(call, 2), "read");
|
|
assert.deepEqual(JSON.parse(stringField(call, 3)), args);
|
|
assert.equal(stringField(history[2], 7), callId);
|
|
assert.equal(stringField(history[2], 3), "file-value");
|
|
});
|