pi-devin/test/stream-deltas.test.mjs

import assert from "node:assert/strict";
import test from "node:test";
import { setImmediate } from "node:timers/promises";
import { streamDevin } from "../src/stream.ts";
import { clearCachedUserJwt } from "../src/jwt.ts";
import { encodeMessage, encodeString, encodeTag, encodeVarintField, frameConnectStream } from "../src/wire.ts";

const model = {
  id: "test-model", name: "Test", api: "devin-local", provider: "devin",
  baseUrl: "https://devin.invalid", reasoning: false, input: ["text"],
  cost: { input: 1, output: 2, cacheRead: 0.1, cacheWrite: 1.25 },
  contextWindow: 1_000_000, maxTokens: 1000,
};

async function run(t, frames) {
  clearCachedUserJwt();
  t.after(clearCachedUserJwt);
  t.mock.method(globalThis, "fetch", async (url) => {
    if (url === "https://devin.invalid/exa.auth_pb.AuthService/GetUserJwt") {
      return new Response(encodeString(1, "eyJtest.jwt"));
    }
    assert.equal(url, "https://devin.invalid/exa.api_server_pb.ApiServerService/GetChatMessage");
    const chunks = [...frames.map((frame) => frameConnectStream(frame, false)), Buffer.from([2, 0, 0, 0, 2, 123, 125])];
    return new Response(new ReadableStream({
      async pull(controller) {
        await setImmediate();
        if (chunks.length) controller.enqueue(chunks.shift());
        else controller.close();
      },
    }));
  });
  const stream = streamDevin(model, { messages: [] }, {
    apiKey: "synthetic-test-key", env: { DEVIN_API_SERVER_URL: "https://devin.invalid" },
  });
  const events = [];
  for await (const event of stream) events.push(structuredClone(event));
  const result = await stream.result();
  assert.notEqual(result.stopReason, "error", result.errorMessage);
  return { events, result };
}

function usage(metrics) {
  return encodeMessage(28, Buffer.concat(Object.entries(metrics).map(([name, value]) => {
    const float = Buffer.alloc(4);
    float.writeFloatLE(value);
    return encodeMessage(2, Buffer.concat([
      encodeString(5, name), encodeMessage(4, Buffer.concat([encodeTag(2, 5), float])),
    ]));
  })));
}

const metrics = { input_tokens: 461, output_tokens: 262, cached_input_tokens: 228_162, cache_creation_input_tokens: 17 };

test("counts all four usage components and preserves them after an empty frame (#3, #4)", async (t) => {
  const { result } = await run(t, [usage(metrics), usage(Object.fromEntries(Object.keys(metrics).map((key) => [key, 0])))]);
  assert.deepEqual({ ...result.usage, cost: undefined }, {
    input: 461, output: 262, cacheRead: 228_162, cacheWrite: 17, totalTokens: 228_902, cost: undefined,
  });
  assert.equal(result.usage.cost.input, 461 / 1_000_000);
  assert.equal(result.usage.cost.cacheRead, 228_162 * 0.1 / 1_000_000);
});

test("accepts cache-only usage and combines separate component snapshots without double counting", async (t) => {
  const { result } = await run(t, [
    usage({ cached_input_tokens: 58_905 }), usage({ input_tokens: 280, output_tokens: 118 }),
    usage({ input_tokens: 280, output_tokens: 118 }), usage({ cache_creation_input_tokens: 17 }),
  ]);
  assert.equal(result.usage.totalTokens, 59_320);
  assert.equal(result.usage.cacheRead, 58_905);
  assert.equal(result.usage.cacheWrite, 17);
});

test("keeps growing tool arguments across split keys, strings, and Unicode escapes (#2)", async (t) => {
  const fragments = ['{"path":"file.txt","cont', 'ent":"hello ', '\\u26', '3a"}'];
  const frames = fragments.map((part, index) => encodeMessage(6, Buffer.concat([
    ...(index === 0 ? [encodeString(1, "call-1"), encodeString(2, "write")] : []), encodeString(3, part),
  ])));
  frames.push(encodeVarintField(5, 10));
  const { events, result } = await run(t, frames);
  const deltas = events.filter((event) => event.type === "toolcall_delta");
  assert.equal(deltas.length, 4);
  for (const event of deltas) assert.equal(event.partial.content[0].arguments.path, "file.txt");
  assert.equal(deltas[1].partial.content[0].arguments.content, "hello");
  assert.deepEqual(result.content[0].arguments, { path: "file.txt", content: "hello ☺" });
  assert.equal(result.stopReason, "toolUse");
});