feat: extract text from fetched PDFs

server/tests/bounded-response.test.ts

@@ -0,0 +1,26 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { readBoundedResponseBytes } from "../src/llm/bounded-response.js";
+
+const oneMiB = 1024 * 1024;
+
+test("reads response bytes within the declared size limit", async () => {
+  const bytes = await readBoundedResponseBytes(new Response("small"), oneMiB);
+  assert.equal(new TextDecoder().decode(bytes), "small");
+});
+
+test("rejects oversized responses from Content-Length before buffering", async () => {
+  const response = new Response("x", { headers: { "content-length": String(oneMiB + 1) } });
+  await assert.rejects(readBoundedResponseBytes(response, oneMiB), /exceeds the 1 MiB download limit/);
+});
+
+test("rejects streamed responses when Content-Length is absent", async () => {
+  const response = new Response(new ReadableStream({
+    start(controller) {
+      controller.enqueue(new Uint8Array(oneMiB));
+      controller.enqueue(new Uint8Array(1));
+      controller.close();
+    },
+  }));
+  await assert.rejects(readBoundedResponseBytes(response, oneMiB), /exceeds the 1 MiB download limit/);
+});

server/tests/chat-tools-streaming.test.ts

@@ -4,6 +4,7 @@ import { type ToolAwareStreamingEvent } from "../src/llm/chat-tools.js";
 import { completeWithChatCompletionsApi, streamWithChatCompletionsApi } from "../src/llm/protocols/chat-completions-api.js";
 import { completeWithMessagesApi, streamWithMessagesApi } from "../src/llm/protocols/messages-api.js";
 import { streamWithResponsesApi } from "../src/llm/protocols/responses-api.js";
+import { createPdf } from "./pdf-fixture.js";
 
 async function* streamFrom(events: any[]) {
   for (const event of events) {
@@ -241,10 +242,12 @@ test("fetch_url sends browser-like navigation headers", async () => {
 
   try {
     let requestCount = 0;
+    const requestBodies: any[] = [];
     const client = {
       chat: {
         completions: {
-          create: async () => {
+          create: async (body: any) => {
+            requestBodies.push(body);
             requestCount += 1;
             if (requestCount === 1) {
               return {
@@ -295,12 +298,108 @@ test("fetch_url sends browser-like navigation headers", async () => {
       "Sec-Fetch-Site": "none",
       "Sec-Fetch-User": "?1",
     });
+    const toolResult = requestBodies[1]?.messages?.find((message: any) => message.role === "tool");
+    const fetched = JSON.parse(toolResult?.content ?? "{}");
+    assert.match(fetched.text, /Consumer price index/);
+    assert.equal(fetched.title, "CPI");
     assert.equal(result.toolEvents[0]?.status, "completed");
   } finally {
     globalThis.fetch = originalFetch;
   }
 });
 
+test("fetch_url extracts a mislabeled PDF and returns page handles", async () => {
+  const originalFetch = globalThis.fetch;
+  const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
+  const pdf = createPdf(["Revenue is 42.", "Costs are 18."]);
+  globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+    fetchCalls.push({ input, init });
+    return new Response(pdf, { status: 200, headers: { "content-type": "text/plain" } });
+  }) as typeof fetch;
+
+  try {
+    const requestBodies: any[] = [];
+    const client = {
+      messages: {
+        create: async (body: any) => {
+          requestBodies.push(body);
+          if (requestBodies.length === 1) {
+            return {
+              content: [{
+                type: "tool_use",
+                id: "pdf_call",
+                name: "fetch_url",
+                input: { url: "https://example.com/report.pdf", pages: [2] },
+              }],
+              usage: { input_tokens: 1, output_tokens: 1 },
+            };
+          }
+          return { content: [{ type: "text", text: "Read page two" }], usage: { input_tokens: 1, output_tokens: 1 } };
+        },
+      },
+    };
+
+    const result = await completeWithMessagesApi({
+      client: client as any,
+      model: "claude-test",
+      messages: [{ role: "user", content: "Read the report" }],
+    });
+    const toolResult = requestBodies[1]?.messages?.at(-1)?.content?.[0];
+    const fetched = JSON.parse(toolResult?.content ?? "{}");
+    assert.equal(result.text, "Read page two");
+    assert.equal(fetchCalls.length, 1);
+    assert.equal(fetched.ok, true);
+    assert.equal(fetched.pdfStatus, "ok");
+    assert.equal(fetched.pageCount, 2);
+    assert.deepEqual(fetched.pages, [2]);
+    assert.equal(fetched.pagesTruncated, true);
+    assert.match(fetched.text, /--- Page 2 ---/);
+    assert.match(fetched.text, /Costs are 18/);
+    assert.doesNotMatch(fetched.text, /Revenue is 42/);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test("fetch_url marks encrypted and text-free PDFs as unavailable", async () => {
+  const originalFetch = globalThis.fetch;
+  let responseBytes = createPdf([""]);
+  globalThis.fetch = (async () => new Response(responseBytes, {
+    status: 200,
+    headers: { "content-type": "application/pdf" },
+  })) as typeof fetch;
+
+  try {
+    for (const [pdf, expectedStatus] of [[createPdf([""]), "no_text"], [createPdf(["Secret"], true), "encrypted"]] as const) {
+      responseBytes = pdf;
+      const requestBodies: any[] = [];
+      const client = {
+        messages: {
+          create: async (body: any) => {
+            requestBodies.push(body);
+            if (requestBodies.length === 1) {
+              return {
+                content: [{ type: "tool_use", id: "pdf_status_call", name: "fetch_url", input: { url: "https://example.com/report.pdf" } }],
+                usage: { input_tokens: 1, output_tokens: 1 },
+              };
+            }
+            return { content: [{ type: "text", text: "Unavailable" }], usage: { input_tokens: 1, output_tokens: 1 } };
+          },
+        },
+      };
+      await completeWithMessagesApi({ client: client as any, model: "claude-test", messages: [{ role: "user", content: "Read PDF" }] });
+      const toolResult = requestBodies[1]?.messages?.at(-1)?.content?.[0];
+      const fetched = JSON.parse(toolResult?.content ?? "{}");
+      assert.equal(fetched.ok, false);
+      assert.equal(fetched.pdfStatus, expectedStatus);
+      assert.equal(toolResult?.is_error, true);
+      assert.ok(fetched.text.length > 0);
+    }
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
 test("Messages API executes tool_use blocks and sends tool_result follow-up", async () => {
   const originalFetch = globalThis.fetch;
   const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];

server/tests/pdf-fixture.ts

@@ -0,0 +1,37 @@
+export function createPdf(pages: string[], encrypted = false): Uint8Array {
+  const fontId = 3 + pages.length * 2;
+  const objects = [
+    "<< /Type /Catalog /Pages 2 0 R >>",
+    `<< /Type /Pages /Kids [${pages.map((_, index) => `${3 + index * 2} 0 R`).join(" ")}] /Count ${pages.length} >>`,
+  ];
+  for (let index = 0; index < pages.length; index += 1) {
+    const pageId = 3 + index * 2;
+    const contentId = pageId + 1;
+    const text = pages[index]!;
+    const operators = Array.from({ length: Math.ceil(text.length / 100) }, (_, partIndex) => {
+      const part = text.slice(partIndex * 100, (partIndex + 1) * 100);
+      const escaped = part.replaceAll("\\", "\\\\").replaceAll("(", "\\(").replaceAll(")", "\\)");
+      return `(${escaped}) Tj`;
+    });
+    const content = text ? `BT /F1 12 Tf 72 720 Td ${operators.join(" ")} ET` : "";
+    objects.push(
+      `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontId} 0 R >> >> /Contents ${contentId} 0 R >>`,
+      `<< /Length ${Buffer.byteLength(content)} >>\nstream\n${content}\nendstream`,
+    );
+  }
+  objects.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>");
+  const encryptionId = objects.length + 1;
+  if (encrypted) objects.push(`<< /Filter /Standard /V 1 /R 2 /O <${"00".repeat(32)}> /U <${"00".repeat(32)}> /P -4 >>`);
+
+  let output = "%PDF-1.4\n";
+  const offsets = [0];
+  objects.forEach((object, index) => {
+    offsets.push(Buffer.byteLength(output));
+    output += `${index + 1} 0 obj\n${object}\nendobj\n`;
+  });
+  const xrefOffset = Buffer.byteLength(output);
+  output += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
+  for (const offset of offsets.slice(1)) output += `${String(offset).padStart(10, "0")} 00000 n \n`;
+  output += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R${encrypted ? ` /Encrypt ${encryptionId} 0 R` : ""} >>\nstartxref\n${xrefOffset}\n%%EOF`;
+  return new TextEncoder().encode(output);
+}

server/tests/pdf-text.test.ts

@@ -0,0 +1,63 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { extractPdfText, hasPdfMagic } from "../src/llm/pdf-text.js";
+import { createPdf } from "./pdf-fixture.js";
+
+test("extracts selectable text with page markers and selected-page support", async () => {
+  const bytes = createPdf(["Quarterly revenue was 42.", "Costs were 18."]);
+  const result = await extractPdfText(bytes, [2]);
+  assert.equal(result.status, "ok");
+  assert.equal(result.pageCount, 2);
+  assert.deepEqual(result.pages, [2]);
+  assert.match(result.text, /--- Page 2 ---/);
+  assert.match(result.text, /Costs were 18\./);
+  assert.doesNotMatch(result.text, /Quarterly revenue/);
+});
+
+test("reports PDFs with no selectable text clearly", async () => {
+  const result = await extractPdfText(createPdf([""]));
+  assert.equal(result.status, "no_text");
+  assert.match(result.text, /No extractable text/);
+});
+
+test("reports encrypted PDFs clearly", async () => {
+  const result = await extractPdfText(createPdf(["Secret"], true));
+  assert.equal(result.status, "encrypted");
+  assert.match(result.text, /encrypted and requires a password/);
+});
+
+test("reports page-free PDFs and rejects invalid requested pages", async () => {
+  const empty = await extractPdfText(createPdf([]));
+  assert.equal(empty.status, "no_pages");
+  assert.match(empty.text, /no pages/);
+  const outOfRange = await extractPdfText(createPdf(["Only page"]), [2]);
+  assert.equal(outOfRange.status, "error");
+  assert.match(outOfRange.text, /has 1 pages/);
+});
+
+test("limits each extraction to 30 pages and preserves the full page count", async () => {
+  const result = await extractPdfText(createPdf(Array.from({ length: 31 }, (_, index) => `Text on page ${index + 1}.`)));
+  assert.equal(result.status, "ok");
+  assert.equal(result.pageCount, 31);
+  assert.equal(result.pages.length, 30);
+  assert.equal(result.pages.at(-1), 30);
+});
+
+test("caps accumulated extracted text at the caller limit", async () => {
+  const result = await extractPdfText(createPdf(["Long sentence. ".repeat(200)]), undefined, 50);
+  assert.equal(result.status, "ok");
+  assert.equal(result.truncated, true);
+  assert.ok(result.text.length <= 50);
+});
+
+test("terminates an over-time worker and returns an explicit timeout result", async () => {
+  const result = await extractPdfText(createPdf(["Small document"]), undefined, 12_000, 1);
+  assert.equal(result.status, "error");
+  assert.match(result.text, /time limit/);
+});
+
+test("recognizes PDF magic in a mislabeled or prefixed response", () => {
+  assert.equal(hasPdfMagic(new TextEncoder().encode("%PDF-1.4")), true);
+  assert.equal(hasPdfMagic(new TextEncoder().encode(`${" ".repeat(500)}%PDF-1.7`)), true);
+  assert.equal(hasPdfMagic(new TextEncoder().encode("<html>not a PDF</html>")), false);
+});