feat: extract text from fetched PDFs

2026-11-01 14:17:40 -08:00
13 changed files with 768 additions and 43 deletions
@@ -31,6 +31,7 @@ RUN apt-get update \
 RUN npm ci --omit=dev --no-audit --no-fund

 COPY --from=server-build /app/server/dist ./dist
+COPY server/src/llm/pdf-text-worker.mjs ./dist/llm/pdf-text-worker.mjs

 ENV NODE_ENV=production
 ENV HOST=0.0.0.0
@@ -359,7 +359,7 @@ Behavior notes:
 - Available Sybil-managed tool calls for `openai`, `anthropic`, `xai`, and `gemini`: `web_search` and `fetch_url`. When `CHAT_CODEX_TOOL_ENABLED=true`, `codex_exec` is also available. When `CHAT_SHELL_TOOL_ENABLED=true`, `shell_exec` is also available.
 - `web_search` returns ranked results with per-result summaries/snippets. Its backend engine is selected by `CHAT_WEB_SEARCH_ENGINE`: `exa` (default), `brave` (requires `BRAVE_SEARCH_API_KEY`), or `searxng` (requires `SEARXNG_BASE_URL`; the instance must allow `format=json`).
 - Brave searches are queued and evenly paced according to the shortest window in Brave's `X-RateLimit-Policy` response header. The backend also honors `X-RateLimit-Remaining`/`X-RateLimit-Reset` and retries `429` responses up to three times with reset-aware exponential backoff; quota resets beyond the bounded retry window fail immediately.
- `fetch_url` fetches a URL with browser-like navigation headers and returns plaintext page content (HTML converted to text server-side).
+- `fetch_url` fetches an HTTP(S) URL with browser-like navigation headers and returns plaintext page content. HTML is converted to text. PDF files are detected by media type or PDF signature and have selectable text extracted with page markers (no OCR); at most 10 MiB and 30 pages are read per call. The optional `pages` array selects up to 30 page numbers for follow-up reads, and `maxCharacters` bounds PDF text collection; a brief truncation hint may follow the extracted text. PDF parsing runs in a worker terminated after 8 seconds. Responses identify page count, selected pages, truncation, encrypted PDFs, empty PDFs, and PDFs without extractable text; unavailable extraction returns `ok:false` with `pdfStatus`. Other URL responses also have a 10 MiB download limit.
 - `codex_exec` delegates coding, shell, repository inspection, and other complex software tasks to a persistent remote Codex CLI workspace over SSH. The server runs `codex exec --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check <non-interactive wrapped prompt>` on the configured devbox inside `CHAT_CODEX_REMOTE_WORKDIR`, with SSH stdin closed.
 - `shell_exec` runs arbitrary non-interactive shell commands on the same configured devbox, starting in `CHAT_CODEX_REMOTE_WORKDIR`. It uses `bash -lc` when bash exists, otherwise `sh -lc`, closes SSH stdin, and does not run inside the Sybil server container.
 - Devbox tool configuration:
@@ -213,7 +213,7 @@ Terminal tool-call event:
 - `anthropic`: backend uses Anthropic's Messages API and may execute the same internal tools with `tool_use`/`tool_result` content blocks before producing final text.
 - `xai`: backend uses xAI's OpenAI-compatible Chat Completions API and may execute the same internal tool calls before producing final text.
 - `gemini`: backend uses Google's native Gemini `streamGenerateContent` API and may execute the same internal tool calls before producing final text.
- `fetch_url` sends browser-like navigation headers for outbound URL requests to reduce false 403s from sites that reject generic server clients.
+- `fetch_url` sends browser-like navigation headers for outbound URL requests to reduce false 403s from sites that reject generic server clients. It converts HTML to text and extracts selectable PDF text page by page (without OCR), including when the response media type is inaccurate but the PDF signature is present. URL downloads are limited to 10 MiB, PDF extraction to 30 pages per call and an isolated worker terminated after 8 seconds, and accumulated PDF text to `maxCharacters` (default 12,000; maximum 50,000), plus a brief truncation hint. PDF tool results include `pdfStatus`, `pageCount`, and selected `pages`; call with `pages:[...]` to retrieve specific pages. Encrypted, page-free, and text-free PDFs return explicit status and text with `ok:false`. Oversized responses fail with a size-limit error.
 - `hermes-agent`: backend uses the configured Hermes Agent OpenAI-compatible Chat Completions API. Sybil does not add its own tool definitions for this provider; Hermes Agent handles its own tools server-side. Custom Hermes stream events are normalized away unless they produce text deltas in this SSE contract.
 - `openai`: image attachments are sent as Responses `input_image` items; text attachments are sent as `input_text` items.
 - `gemini`: image attachments are sent as native Gemini `inlineData` parts; text attachments are inlined as text parts.
@@ -21,6 +21,7 @@
        "html-to-text": "^9.0.5",
        "marked": "17.0.2",
        "openai": "^6.16.0",
+        "pdfjs-dist": "5.4.394",
        "pino-pretty": "^13.1.3",
        "prisma": "^6.6.0",
        "web-push": "^3.6.7",
@@ -775,6 +776,256 @@
        "node": ">=8"
      }
    },
+    "node_modules/@napi-rs/canvas": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.100.tgz",
+      "integrity": "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==",
+      "license": "MIT",
+      "optional": true,
+      "workspaces": [
+        "e2e/*"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      },
+      "optionalDependencies": {
+        "@napi-rs/canvas-android-arm64": "0.1.100",
+        "@napi-rs/canvas-darwin-arm64": "0.1.100",
+        "@napi-rs/canvas-darwin-x64": "0.1.100",
+        "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100",
+        "@napi-rs/canvas-linux-arm64-gnu": "0.1.100",
+        "@napi-rs/canvas-linux-arm64-musl": "0.1.100",
+        "@napi-rs/canvas-linux-riscv64-gnu": "0.1.100",
+        "@napi-rs/canvas-linux-x64-gnu": "0.1.100",
+        "@napi-rs/canvas-linux-x64-musl": "0.1.100",
+        "@napi-rs/canvas-win32-arm64-msvc": "0.1.100",
+        "@napi-rs/canvas-win32-x64-msvc": "0.1.100"
+      }
+    },
+    "node_modules/@napi-rs/canvas-android-arm64": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.100.tgz",
+      "integrity": "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-darwin-arm64": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.100.tgz",
+      "integrity": "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-darwin-x64": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.100.tgz",
+      "integrity": "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.100.tgz",
+      "integrity": "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-arm64-gnu": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.100.tgz",
+      "integrity": "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-arm64-musl": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.100.tgz",
+      "integrity": "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.100.tgz",
+      "integrity": "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==",
+      "cpu": [
+        "riscv64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-x64-gnu": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.100.tgz",
+      "integrity": "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-x64-musl": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.100.tgz",
+      "integrity": "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-win32-arm64-msvc": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.100.tgz",
+      "integrity": "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-win32-x64-msvc": {
+      "version": "0.1.100",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.100.tgz",
+      "integrity": "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
    "node_modules/@pinojs/redact": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
@@ -1939,6 +2190,18 @@
        "url": "https://github.com/sponsors/isaacs"
      }
    },
+    "node_modules/pdfjs-dist": {
+      "version": "5.4.394",
+      "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.394.tgz",
+      "integrity": "sha512-9ariAYGqUJzx+V/1W4jHyiyCep6IZALmDzoaTLZ6VNu8q9LWi1/ukhzHgE2Xsx96AZi0mbZuK4/ttIbqSbLypg==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=20.16.0 || >=22.3.0"
+      },
+      "optionalDependencies": {
+        "@napi-rs/canvas": "^0.1.81"
+      }
+    },
    "node_modules/peberminta": {
      "version": "0.9.0",
      "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz",
@@ -30,6 +30,7 @@
    "html-to-text": "^9.0.5",
    "marked": "17.0.2",
    "openai": "^6.16.0",
+    "pdfjs-dist": "5.4.394",
    "pino-pretty": "^13.1.3",
    "prisma": "^6.6.0",
    "web-push": "^3.6.7",
@@ -0,0 +1,36 @@
+export const MAX_FETCH_RESPONSE_BYTES = 10 * 1024 * 1024;
+
+export async function readBoundedResponseBytes(response: Response, maxBytes: number) {
+  const contentLength = Number(response.headers.get("content-length"));
+  if (Number.isFinite(contentLength) && contentLength > maxBytes) {
+    await response.body?.cancel();
+    throw new Error(`URL response exceeds the ${Math.floor(maxBytes / (1024 * 1024))} MiB download limit.`);
+  }
+
+  const reader = response.body?.getReader();
+  if (!reader) return new Uint8Array();
+  const chunks: Uint8Array[] = [];
+  let totalBytes = 0;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      totalBytes += value.byteLength;
+      if (totalBytes > maxBytes) {
+        await reader.cancel();
+        throw new Error(`URL response exceeds the ${Math.floor(maxBytes / (1024 * 1024))} MiB download limit.`);
+      }
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const bytes = new Uint8Array(totalBytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return bytes;
+}
@@ -6,10 +6,12 @@ import { promisify } from "node:util";
 import { convert as htmlToText } from "html-to-text";
 import { z } from "zod";
 import { buildBrowserLikeNavigationHeaders } from "../browser-fetch-headers.js";
+import { MAX_FETCH_RESPONSE_BYTES, readBoundedResponseBytes } from "./bounded-response.js";
 import { env } from "../env.js";
 import { searchBrave } from "../search/brave.js";
 import { exaClient } from "../search/exa.js";
 import { searchSearxng } from "../search/searxng.js";
+import { extractPdfText, hasPdfMagic, MAX_PDF_PAGES_PER_FETCH } from "./pdf-text.js";
 import type { ChatMessage } from "./types.js";

 export const MAX_TOOL_ROUNDS = env.CHAT_MAX_TOOL_ROUNDS;
@@ -45,6 +47,7 @@ const FetchUrlArgsSchema = z
  .object({
    url: z.string().trim().url(),
    maxCharacters: z.coerce.number().int().min(500).max(MAX_FETCH_MAX_CHARACTERS).optional(),
+    pages: z.array(z.coerce.number().int().min(1).max(1_000_000)).min(1).max(MAX_PDF_PAGES_PER_FETCH).optional(),
  })
  .strict();

@@ -163,7 +166,7 @@ const BASE_CHAT_TOOLS: any[] = [
    function: {
      name: "fetch_url",
      description:
-        "Fetch a webpage by URL and return readable plaintext content extracted from the page for deeper inspection.",
+        "Fetch a webpage by URL and return readable plaintext content. Text PDFs are extracted page by page; use pages:[3,4] to retrieve selected pages after a truncated result. Scanned PDFs have no extractable text.",
      parameters: {
        type: "object",
        properties: {
@@ -174,6 +177,12 @@ const BASE_CHAT_TOOLS: any[] = [
            maximum: MAX_FETCH_MAX_CHARACTERS,
            description: "Maximum response text characters returned (default 12000).",
          },
+          pages: {
+            type: "array",
+            items: { type: "integer", minimum: 1, maximum: 1000000 },
+            maxItems: MAX_PDF_PAGES_PER_FETCH,
+            description: "For PDFs, extract these page numbers only (up to 30 per fetch).",
+          },
        },
        required: ["url"],
        additionalProperties: false,
@@ -561,48 +570,69 @@ async function runFetchUrlTool(input: unknown): Promise<ToolRunOutcome> {
      signal: controller.signal,
      headers: buildBrowserLikeNavigationHeaders(),
    });
+    if (!response.ok) {
+      await response.body?.cancel();
+      throw new Error(`Fetch failed with status ${response.status}.`);
+    }
+
+    const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
+    const bytes = await readBoundedResponseBytes(response, MAX_FETCH_RESPONSE_BYTES);
+    const url = response.url || parsed.toString();
+    if (contentType.includes("application/pdf") || hasPdfMagic(bytes)) {
+      if (!hasPdfMagic(bytes)) {
+        return { ok: false, url, status: response.status, contentType: contentType || null, pdfStatus: "invalid", text: "The server declared a PDF, but the downloaded file does not have a valid PDF header." };
+      }
+      const result = await extractPdfText(bytes, args.pages, maxCharacters);
+      const text = result.truncated
+        ? `${result.text}\n\n[truncated; re-fetch selected pages with pages:[N,...]]`
+        : result.text;
+      return {
+        ok: result.status === "ok",
+        url,
+        status: response.status,
+        contentType: contentType || "application/pdf",
+        pdfStatus: result.status,
+        pageCount: result.pageCount,
+        pages: result.pages,
+        pagesTruncated: result.pageCount !== null && result.pageCount > result.pages.length,
+        truncated: result.truncated,
+        text,
+      };
+    }
+
+    const body = new TextDecoder().decode(bytes);
+    const isHtml = contentType.includes("text/html") || /<!doctype html|<html[\s>]/i.test(body);
+    const extracted = isHtml
+      ? htmlToText(body, {
+          wordwrap: false,
+          preserveNewlines: true,
+          selectors: [
+            { selector: "img", format: "skip" },
+            { selector: "script", format: "skip" },
+            { selector: "style", format: "skip" },
+            { selector: "noscript", format: "skip" },
+            { selector: "a", options: { ignoreHref: true } },
+          ],
+        })
+      : body;
+    const normalized = compactWhitespace(extracted);
+    const truncated = normalized.length > maxCharacters;
+    const text = truncated
+      ? `${normalized.slice(0, maxCharacters)}\n\n[truncated ${normalized.length - maxCharacters} characters]`
+      : normalized;
+
+    return {
+      ok: true,
+      url,
+      status: response.status,
+      contentType: contentType || null,
+      title: isHtml ? extractHtmlTitle(body) : null,
+      truncated,
+      text,
+    };
  } finally {
    clearTimeout(timeout);
  }
-
-  if (!response.ok) {
-    throw new Error(`Fetch failed with status ${response.status}.`);
-  }
-
-  const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
-  const body = await response.text();
-  const isHtml = contentType.includes("text/html") || /<!doctype html|<html[\s>]/i.test(body);
-
-  let extracted = body;
-  if (isHtml) {
-    extracted = htmlToText(body, {
-      wordwrap: false,
-      preserveNewlines: true,
-      selectors: [
-        { selector: "img", format: "skip" },
-        { selector: "script", format: "skip" },
-        { selector: "style", format: "skip" },
-        { selector: "noscript", format: "skip" },
-        { selector: "a", options: { ignoreHref: true } },
-      ],
-    });
-  }
-
-  const normalized = compactWhitespace(extracted);
-  const truncated = normalized.length > maxCharacters;
-  const text = truncated
-    ? `${normalized.slice(0, maxCharacters)}\n\n[truncated ${normalized.length - maxCharacters} characters]`
-    : normalized;
-
-  return {
-    ok: true,
-    url: response.url || parsed.toString(),
-    status: response.status,
-    contentType: contentType || null,
-    title: isHtml ? extractHtmlTitle(body) : null,
-    truncated,
-    text,
-  };
 }

 function shellQuote(value: string) {
@@ -0,0 +1,100 @@
+import { getDocument } from "pdfjs-dist/legacy/build/pdf.mjs";
+import { parentPort, workerData } from "node:worker_threads";
+
+const { bytes, requestedPages, maxCharacters, maxPages } = workerData;
+
+function isPasswordError(error) {
+  return Boolean(error && typeof error === "object" && error.name === "PasswordException");
+}
+
+async function extract() {
+  let loadingTask;
+  try {
+    loadingTask = getDocument({
+      data: bytes,
+      standardFontDataUrl: new URL("../../node_modules/pdfjs-dist/standard_fonts/", import.meta.url).href,
+      isEvalSupported: false,
+      disableFontFace: true,
+      stopAtErrors: true,
+    });
+    const document = await loadingTask.promise;
+    if (document.numPages === 0) {
+      return { status: "no_pages", pageCount: 0, pages: [], text: "This PDF has no pages.", truncated: false };
+    }
+
+    const pages = requestedPages?.length
+      ? [...new Set(requestedPages)].slice(0, maxPages)
+      : Array.from({ length: Math.min(document.numPages, maxPages) }, (_, index) => index + 1);
+    const invalidPage = pages.find((page) => page > document.numPages);
+    if (invalidPage) {
+      return {
+        status: "error",
+        pageCount: document.numPages,
+        pages: [],
+        text: `Requested page ${invalidPage}, but this PDF has ${document.numPages} pages.`,
+        truncated: false,
+      };
+    }
+
+    const chunks = [];
+    let characterCount = 0;
+    let foundText = false;
+    let truncated = false;
+    const append = (text) => {
+      const remaining = maxCharacters - characterCount;
+      if (remaining <= 0) {
+        truncated = true;
+        return;
+      }
+      const part = text.slice(0, remaining);
+      chunks.push(part);
+      characterCount += part.length;
+      if (part.length < text.length) truncated = true;
+    };
+
+    for (const pageNumber of pages) {
+      append(`${chunks.length ? "\n\n" : ""}--- Page ${pageNumber} ---\n`);
+      if (truncated) break;
+      const page = await document.getPage(pageNumber);
+      const reader = page.streamTextContent().getReader();
+      try {
+        while (!truncated) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          for (const item of value.items) {
+            if (!("str" in item) || !item.str) continue;
+            foundText = true;
+            append(item.str + (item.hasEOL ? "\n" : " "));
+            if (truncated) break;
+          }
+        }
+      } finally {
+        if (truncated) await reader.cancel().catch(() => undefined);
+        reader.releaseLock();
+      }
+      if (truncated) break;
+    }
+
+    const text = chunks.join("").trimEnd();
+    if (!foundText) {
+      return {
+        status: "no_text",
+        pageCount: document.numPages,
+        pages,
+        text: "No extractable text was found in the selected pages.",
+        truncated: false,
+      };
+    }
+    return { status: "ok", pageCount: document.numPages, pages, text, truncated };
+  } catch (error) {
+    if (isPasswordError(error)) {
+      return { status: "encrypted", pageCount: null, pages: [], text: "This PDF is encrypted and requires a password.", truncated: false };
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    return { status: "error", pageCount: null, pages: [], text: `PDF reading failed: ${message}`, truncated: false };
+  } finally {
+    await loadingTask?.destroy().catch(() => undefined);
+  }
+}
+
+parentPort?.postMessage(await extract());
@@ -0,0 +1,69 @@
+import { Worker } from "node:worker_threads";
+import { fileURLToPath } from "node:url";
+
+export const MAX_PDF_PAGES_PER_FETCH = 30;
+export const PDF_PARSE_TIMEOUT_MS = 8_000;
+const DEFAULT_PDF_TEXT_CHARACTERS = 12_000;
+
+export type PdfTextResult =
+  | { status: "ok"; pageCount: number; pages: number[]; text: string; truncated: boolean }
+  | { status: "no_pages"; pageCount: 0; pages: []; text: "This PDF has no pages."; truncated: false }
+  | { status: "no_text"; pageCount: number; pages: number[]; text: "No extractable text was found in the selected pages."; truncated: false }
+  | { status: "encrypted"; pageCount: null; pages: []; text: "This PDF is encrypted and requires a password."; truncated: false }
+  | { status: "error"; pageCount: number | null; pages: number[]; text: string; truncated: false };
+
+export function hasPdfMagic(bytes: Uint8Array) {
+  const headerEnd = Math.min(bytes.length, 1024);
+  return Buffer.from(bytes.subarray(0, headerEnd)).includes(Buffer.from("%PDF-"));
+}
+
+export function extractPdfText(
+  bytes: Uint8Array,
+  requestedPages?: number[],
+  maxCharacters = DEFAULT_PDF_TEXT_CHARACTERS,
+  timeoutMs = PDF_PARSE_TIMEOUT_MS,
+): Promise<PdfTextResult> {
+  return new Promise((resolve) => {
+    let settled = false;
+    let timeout: ReturnType<typeof setTimeout>;
+    let worker: Worker;
+    const finish = (result: PdfTextResult) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      void worker.terminate().catch(() => undefined);
+      resolve(result);
+    };
+
+    try {
+      worker = new Worker(fileURLToPath(new URL("./pdf-text-worker.mjs", import.meta.url)), {
+        workerData: { bytes: new Uint8Array(bytes), requestedPages, maxCharacters, maxPages: MAX_PDF_PAGES_PER_FETCH },
+        execArgv: [],
+      });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      resolve({ status: "error", pageCount: null, pages: [], text: `PDF reader could not start: ${message}`, truncated: false });
+      return;
+    }
+
+    timeout = setTimeout(() => {
+      finish({
+        status: "error",
+        pageCount: null,
+        pages: [],
+        text: `PDF reading exceeded its time limit (${timeoutMs} ms).`,
+        truncated: false,
+      });
+    }, timeoutMs);
+    worker.once("message", (result: PdfTextResult) => finish(result));
+    worker.once("error", (error) => {
+      const message = error instanceof Error ? error.message : String(error);
+      finish({ status: "error", pageCount: null, pages: [], text: `PDF reading failed: ${message}`, truncated: false });
+    });
+    worker.once("exit", (code) => {
+      if (!settled) {
+        finish({ status: "error", pageCount: null, pages: [], text: `PDF reader exited unexpectedly (${code}).`, truncated: false });
+      }
+    });
+  });
+}
@@ -0,0 +1,26 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { readBoundedResponseBytes } from "../src/llm/bounded-response.js";
+
+const oneMiB = 1024 * 1024;
+
+test("reads response bytes within the declared size limit", async () => {
+  const bytes = await readBoundedResponseBytes(new Response("small"), oneMiB);
+  assert.equal(new TextDecoder().decode(bytes), "small");
+});
+
+test("rejects oversized responses from Content-Length before buffering", async () => {
+  const response = new Response("x", { headers: { "content-length": String(oneMiB + 1) } });
+  await assert.rejects(readBoundedResponseBytes(response, oneMiB), /exceeds the 1 MiB download limit/);
+});
+
+test("rejects streamed responses when Content-Length is absent", async () => {
+  const response = new Response(new ReadableStream({
+    start(controller) {
+      controller.enqueue(new Uint8Array(oneMiB));
+      controller.enqueue(new Uint8Array(1));
+      controller.close();
+    },
+  }));
+  await assert.rejects(readBoundedResponseBytes(response, oneMiB), /exceeds the 1 MiB download limit/);
+});
@@ -4,6 +4,7 @@ import { type ToolAwareStreamingEvent } from "../src/llm/chat-tools.js";
 import { completeWithChatCompletionsApi, streamWithChatCompletionsApi } from "../src/llm/protocols/chat-completions-api.js";
 import { completeWithMessagesApi, streamWithMessagesApi } from "../src/llm/protocols/messages-api.js";
 import { streamWithResponsesApi } from "../src/llm/protocols/responses-api.js";
+import { createPdf } from "./pdf-fixture.js";

 async function* streamFrom(events: any[]) {
  for (const event of events) {
@@ -241,10 +242,12 @@ test("fetch_url sends browser-like navigation headers", async () => {

  try {
    let requestCount = 0;
+    const requestBodies: any[] = [];
    const client = {
      chat: {
        completions: {
-          create: async () => {
+          create: async (body: any) => {
+            requestBodies.push(body);
            requestCount += 1;
            if (requestCount === 1) {
              return {
@@ -295,12 +298,108 @@ test("fetch_url sends browser-like navigation headers", async () => {
      "Sec-Fetch-Site": "none",
      "Sec-Fetch-User": "?1",
    });
+    const toolResult = requestBodies[1]?.messages?.find((message: any) => message.role === "tool");
+    const fetched = JSON.parse(toolResult?.content ?? "{}");
+    assert.match(fetched.text, /Consumer price index/);
+    assert.equal(fetched.title, "CPI");
    assert.equal(result.toolEvents[0]?.status, "completed");
  } finally {
    globalThis.fetch = originalFetch;
  }
 });

+test("fetch_url extracts a mislabeled PDF and returns page handles", async () => {
+  const originalFetch = globalThis.fetch;
+  const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
+  const pdf = createPdf(["Revenue is 42.", "Costs are 18."]);
+  globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+    fetchCalls.push({ input, init });
+    return new Response(pdf, { status: 200, headers: { "content-type": "text/plain" } });
+  }) as typeof fetch;
+
+  try {
+    const requestBodies: any[] = [];
+    const client = {
+      messages: {
+        create: async (body: any) => {
+          requestBodies.push(body);
+          if (requestBodies.length === 1) {
+            return {
+              content: [{
+                type: "tool_use",
+                id: "pdf_call",
+                name: "fetch_url",
+                input: { url: "https://example.com/report.pdf", pages: [2] },
+              }],
+              usage: { input_tokens: 1, output_tokens: 1 },
+            };
+          }
+          return { content: [{ type: "text", text: "Read page two" }], usage: { input_tokens: 1, output_tokens: 1 } };
+        },
+      },
+    };
+
+    const result = await completeWithMessagesApi({
+      client: client as any,
+      model: "claude-test",
+      messages: [{ role: "user", content: "Read the report" }],
+    });
+    const toolResult = requestBodies[1]?.messages?.at(-1)?.content?.[0];
+    const fetched = JSON.parse(toolResult?.content ?? "{}");
+    assert.equal(result.text, "Read page two");
+    assert.equal(fetchCalls.length, 1);
+    assert.equal(fetched.ok, true);
+    assert.equal(fetched.pdfStatus, "ok");
+    assert.equal(fetched.pageCount, 2);
+    assert.deepEqual(fetched.pages, [2]);
+    assert.equal(fetched.pagesTruncated, true);
+    assert.match(fetched.text, /--- Page 2 ---/);
+    assert.match(fetched.text, /Costs are 18/);
+    assert.doesNotMatch(fetched.text, /Revenue is 42/);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test("fetch_url marks encrypted and text-free PDFs as unavailable", async () => {
+  const originalFetch = globalThis.fetch;
+  let responseBytes = createPdf([""]);
+  globalThis.fetch = (async () => new Response(responseBytes, {
+    status: 200,
+    headers: { "content-type": "application/pdf" },
+  })) as typeof fetch;
+
+  try {
+    for (const [pdf, expectedStatus] of [[createPdf([""]), "no_text"], [createPdf(["Secret"], true), "encrypted"]] as const) {
+      responseBytes = pdf;
+      const requestBodies: any[] = [];
+      const client = {
+        messages: {
+          create: async (body: any) => {
+            requestBodies.push(body);
+            if (requestBodies.length === 1) {
+              return {
+                content: [{ type: "tool_use", id: "pdf_status_call", name: "fetch_url", input: { url: "https://example.com/report.pdf" } }],
+                usage: { input_tokens: 1, output_tokens: 1 },
+              };
+            }
+            return { content: [{ type: "text", text: "Unavailable" }], usage: { input_tokens: 1, output_tokens: 1 } };
+          },
+        },
+      };
+      await completeWithMessagesApi({ client: client as any, model: "claude-test", messages: [{ role: "user", content: "Read PDF" }] });
+      const toolResult = requestBodies[1]?.messages?.at(-1)?.content?.[0];
+      const fetched = JSON.parse(toolResult?.content ?? "{}");
+      assert.equal(fetched.ok, false);
+      assert.equal(fetched.pdfStatus, expectedStatus);
+      assert.equal(toolResult?.is_error, true);
+      assert.ok(fetched.text.length > 0);
+    }
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
 test("Messages API executes tool_use blocks and sends tool_result follow-up", async () => {
  const originalFetch = globalThis.fetch;
  const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
@@ -0,0 +1,37 @@
+export function createPdf(pages: string[], encrypted = false): Uint8Array {
+  const fontId = 3 + pages.length * 2;
+  const objects = [
+    "<< /Type /Catalog /Pages 2 0 R >>",
+    `<< /Type /Pages /Kids [${pages.map((_, index) => `${3 + index * 2} 0 R`).join(" ")}] /Count ${pages.length} >>`,
+  ];
+  for (let index = 0; index < pages.length; index += 1) {
+    const pageId = 3 + index * 2;
+    const contentId = pageId + 1;
+    const text = pages[index]!;
+    const operators = Array.from({ length: Math.ceil(text.length / 100) }, (_, partIndex) => {
+      const part = text.slice(partIndex * 100, (partIndex + 1) * 100);
+      const escaped = part.replaceAll("\\", "\\\\").replaceAll("(", "\\(").replaceAll(")", "\\)");
+      return `(${escaped}) Tj`;
+    });
+    const content = text ? `BT /F1 12 Tf 72 720 Td ${operators.join(" ")} ET` : "";
+    objects.push(
+      `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontId} 0 R >> >> /Contents ${contentId} 0 R >>`,
+      `<< /Length ${Buffer.byteLength(content)} >>\nstream\n${content}\nendstream`,
+    );
+  }
+  objects.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>");
+  const encryptionId = objects.length + 1;
+  if (encrypted) objects.push(`<< /Filter /Standard /V 1 /R 2 /O <${"00".repeat(32)}> /U <${"00".repeat(32)}> /P -4 >>`);
+
+  let output = "%PDF-1.4\n";
+  const offsets = [0];
+  objects.forEach((object, index) => {
+    offsets.push(Buffer.byteLength(output));
+    output += `${index + 1} 0 obj\n${object}\nendobj\n`;
+  });
+  const xrefOffset = Buffer.byteLength(output);
+  output += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
+  for (const offset of offsets.slice(1)) output += `${String(offset).padStart(10, "0")} 00000 n \n`;
+  output += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R${encrypted ? ` /Encrypt ${encryptionId} 0 R` : ""} >>\nstartxref\n${xrefOffset}\n%%EOF`;
+  return new TextEncoder().encode(output);
+}
@@ -0,0 +1,63 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { extractPdfText, hasPdfMagic } from "../src/llm/pdf-text.js";
+import { createPdf } from "./pdf-fixture.js";
+
+test("extracts selectable text with page markers and selected-page support", async () => {
+  const bytes = createPdf(["Quarterly revenue was 42.", "Costs were 18."]);
+  const result = await extractPdfText(bytes, [2]);
+  assert.equal(result.status, "ok");
+  assert.equal(result.pageCount, 2);
+  assert.deepEqual(result.pages, [2]);
+  assert.match(result.text, /--- Page 2 ---/);
+  assert.match(result.text, /Costs were 18\./);
+  assert.doesNotMatch(result.text, /Quarterly revenue/);
+});
+
+test("reports PDFs with no selectable text clearly", async () => {
+  const result = await extractPdfText(createPdf([""]));
+  assert.equal(result.status, "no_text");
+  assert.match(result.text, /No extractable text/);
+});
+
+test("reports encrypted PDFs clearly", async () => {
+  const result = await extractPdfText(createPdf(["Secret"], true));
+  assert.equal(result.status, "encrypted");
+  assert.match(result.text, /encrypted and requires a password/);
+});
+
+test("reports page-free PDFs and rejects invalid requested pages", async () => {
+  const empty = await extractPdfText(createPdf([]));
+  assert.equal(empty.status, "no_pages");
+  assert.match(empty.text, /no pages/);
+  const outOfRange = await extractPdfText(createPdf(["Only page"]), [2]);
+  assert.equal(outOfRange.status, "error");
+  assert.match(outOfRange.text, /has 1 pages/);
+});
+
+test("limits each extraction to 30 pages and preserves the full page count", async () => {
+  const result = await extractPdfText(createPdf(Array.from({ length: 31 }, (_, index) => `Text on page ${index + 1}.`)));
+  assert.equal(result.status, "ok");
+  assert.equal(result.pageCount, 31);
+  assert.equal(result.pages.length, 30);
+  assert.equal(result.pages.at(-1), 30);
+});
+
+test("caps accumulated extracted text at the caller limit", async () => {
+  const result = await extractPdfText(createPdf(["Long sentence. ".repeat(200)]), undefined, 50);
+  assert.equal(result.status, "ok");
+  assert.equal(result.truncated, true);
+  assert.ok(result.text.length <= 50);
+});
+
+test("terminates an over-time worker and returns an explicit timeout result", async () => {
+  const result = await extractPdfText(createPdf(["Small document"]), undefined, 12_000, 1);
+  assert.equal(result.status, "error");
+  assert.match(result.text, /time limit/);
+});
+
+test("recognizes PDF magic in a mislabeled or prefixed response", () => {
+  assert.equal(hasPdfMagic(new TextEncoder().encode("%PDF-1.4")), true);
+  assert.equal(hasPdfMagic(new TextEncoder().encode(`${" ".repeat(500)}%PDF-1.7`)), true);
+  assert.equal(hasPdfMagic(new TextEncoder().encode("<html>not a PDF</html>")), false);
+});