feat: extract text from fetched PDFs

server/src/llm/bounded-response.ts

@@ -0,0 +1,36 @@
+export const MAX_FETCH_RESPONSE_BYTES = 10 * 1024 * 1024;
+
+export async function readBoundedResponseBytes(response: Response, maxBytes: number) {
+  const contentLength = Number(response.headers.get("content-length"));
+  if (Number.isFinite(contentLength) && contentLength > maxBytes) {
+    await response.body?.cancel();
+    throw new Error(`URL response exceeds the ${Math.floor(maxBytes / (1024 * 1024))} MiB download limit.`);
+  }
+
+  const reader = response.body?.getReader();
+  if (!reader) return new Uint8Array();
+  const chunks: Uint8Array[] = [];
+  let totalBytes = 0;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      totalBytes += value.byteLength;
+      if (totalBytes > maxBytes) {
+        await reader.cancel();
+        throw new Error(`URL response exceeds the ${Math.floor(maxBytes / (1024 * 1024))} MiB download limit.`);
+      }
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const bytes = new Uint8Array(totalBytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return bytes;
+}

server/src/llm/chat-tools.ts

@@ -6,10 +6,12 @@ import { promisify } from "node:util";
 import { convert as htmlToText } from "html-to-text";
 import { z } from "zod";
 import { buildBrowserLikeNavigationHeaders } from "../browser-fetch-headers.js";
+import { MAX_FETCH_RESPONSE_BYTES, readBoundedResponseBytes } from "./bounded-response.js";
 import { env } from "../env.js";
 import { searchBrave } from "../search/brave.js";
 import { exaClient } from "../search/exa.js";
 import { searchSearxng } from "../search/searxng.js";
+import { extractPdfText, hasPdfMagic, MAX_PDF_PAGES_PER_FETCH } from "./pdf-text.js";
 import type { ChatMessage } from "./types.js";
 
 export const MAX_TOOL_ROUNDS = env.CHAT_MAX_TOOL_ROUNDS;
@@ -45,6 +47,7 @@ const FetchUrlArgsSchema = z
   .object({
     url: z.string().trim().url(),
     maxCharacters: z.coerce.number().int().min(500).max(MAX_FETCH_MAX_CHARACTERS).optional(),
+    pages: z.array(z.coerce.number().int().min(1).max(1_000_000)).min(1).max(MAX_PDF_PAGES_PER_FETCH).optional(),
   })
   .strict();
 
@@ -163,7 +166,7 @@ const BASE_CHAT_TOOLS: any[] = [
     function: {
       name: "fetch_url",
       description:
-        "Fetch a webpage by URL and return readable plaintext content extracted from the page for deeper inspection.",
+        "Fetch a webpage by URL and return readable plaintext content. Text PDFs are extracted page by page; use pages:[3,4] to retrieve selected pages after a truncated result. Scanned PDFs have no extractable text.",
       parameters: {
         type: "object",
         properties: {
@@ -174,6 +177,12 @@ const BASE_CHAT_TOOLS: any[] = [
             maximum: MAX_FETCH_MAX_CHARACTERS,
             description: "Maximum response text characters returned (default 12000).",
           },
+          pages: {
+            type: "array",
+            items: { type: "integer", minimum: 1, maximum: 1000000 },
+            maxItems: MAX_PDF_PAGES_PER_FETCH,
+            description: "For PDFs, extract these page numbers only (up to 30 per fetch).",
+          },
         },
         required: ["url"],
         additionalProperties: false,
@@ -561,48 +570,69 @@ async function runFetchUrlTool(input: unknown): Promise<ToolRunOutcome> {
       signal: controller.signal,
       headers: buildBrowserLikeNavigationHeaders(),
     });
+    if (!response.ok) {
+      await response.body?.cancel();
+      throw new Error(`Fetch failed with status ${response.status}.`);
+    }
+
+    const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
+    const bytes = await readBoundedResponseBytes(response, MAX_FETCH_RESPONSE_BYTES);
+    const url = response.url || parsed.toString();
+    if (contentType.includes("application/pdf") || hasPdfMagic(bytes)) {
+      if (!hasPdfMagic(bytes)) {
+        return { ok: false, url, status: response.status, contentType: contentType || null, pdfStatus: "invalid", text: "The server declared a PDF, but the downloaded file does not have a valid PDF header." };
+      }
+      const result = await extractPdfText(bytes, args.pages, maxCharacters);
+      const text = result.truncated
+        ? `${result.text}\n\n[truncated; re-fetch selected pages with pages:[N,...]]`
+        : result.text;
+      return {
+        ok: result.status === "ok",
+        url,
+        status: response.status,
+        contentType: contentType || "application/pdf",
+        pdfStatus: result.status,
+        pageCount: result.pageCount,
+        pages: result.pages,
+        pagesTruncated: result.pageCount !== null && result.pageCount > result.pages.length,
+        truncated: result.truncated,
+        text,
+      };
+    }
+
+    const body = new TextDecoder().decode(bytes);
+    const isHtml = contentType.includes("text/html") || /<!doctype html|<html[\s>]/i.test(body);
+    const extracted = isHtml
+      ? htmlToText(body, {
+          wordwrap: false,
+          preserveNewlines: true,
+          selectors: [
+            { selector: "img", format: "skip" },
+            { selector: "script", format: "skip" },
+            { selector: "style", format: "skip" },
+            { selector: "noscript", format: "skip" },
+            { selector: "a", options: { ignoreHref: true } },
+          ],
+        })
+      : body;
+    const normalized = compactWhitespace(extracted);
+    const truncated = normalized.length > maxCharacters;
+    const text = truncated
+      ? `${normalized.slice(0, maxCharacters)}\n\n[truncated ${normalized.length - maxCharacters} characters]`
+      : normalized;
+
+    return {
+      ok: true,
+      url,
+      status: response.status,
+      contentType: contentType || null,
+      title: isHtml ? extractHtmlTitle(body) : null,
+      truncated,
+      text,
+    };
   } finally {
     clearTimeout(timeout);
   }
-
-  if (!response.ok) {
-    throw new Error(`Fetch failed with status ${response.status}.`);
-  }
-
-  const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
-  const body = await response.text();
-  const isHtml = contentType.includes("text/html") || /<!doctype html|<html[\s>]/i.test(body);
-
-  let extracted = body;
-  if (isHtml) {
-    extracted = htmlToText(body, {
-      wordwrap: false,
-      preserveNewlines: true,
-      selectors: [
-        { selector: "img", format: "skip" },
-        { selector: "script", format: "skip" },
-        { selector: "style", format: "skip" },
-        { selector: "noscript", format: "skip" },
-        { selector: "a", options: { ignoreHref: true } },
-      ],
-    });
-  }
-
-  const normalized = compactWhitespace(extracted);
-  const truncated = normalized.length > maxCharacters;
-  const text = truncated
-    ? `${normalized.slice(0, maxCharacters)}\n\n[truncated ${normalized.length - maxCharacters} characters]`
-    : normalized;
-
-  return {
-    ok: true,
-    url: response.url || parsed.toString(),
-    status: response.status,
-    contentType: contentType || null,
-    title: isHtml ? extractHtmlTitle(body) : null,
-    truncated,
-    text,
-  };
 }
 
 function shellQuote(value: string) {

server/src/llm/pdf-text-worker.mjs

@@ -0,0 +1,100 @@
+import { getDocument } from "pdfjs-dist/legacy/build/pdf.mjs";
+import { parentPort, workerData } from "node:worker_threads";
+
+const { bytes, requestedPages, maxCharacters, maxPages } = workerData;
+
+function isPasswordError(error) {
+  return Boolean(error && typeof error === "object" && error.name === "PasswordException");
+}
+
+async function extract() {
+  let loadingTask;
+  try {
+    loadingTask = getDocument({
+      data: bytes,
+      standardFontDataUrl: new URL("../../node_modules/pdfjs-dist/standard_fonts/", import.meta.url).href,
+      isEvalSupported: false,
+      disableFontFace: true,
+      stopAtErrors: true,
+    });
+    const document = await loadingTask.promise;
+    if (document.numPages === 0) {
+      return { status: "no_pages", pageCount: 0, pages: [], text: "This PDF has no pages.", truncated: false };
+    }
+
+    const pages = requestedPages?.length
+      ? [...new Set(requestedPages)].slice(0, maxPages)
+      : Array.from({ length: Math.min(document.numPages, maxPages) }, (_, index) => index + 1);
+    const invalidPage = pages.find((page) => page > document.numPages);
+    if (invalidPage) {
+      return {
+        status: "error",
+        pageCount: document.numPages,
+        pages: [],
+        text: `Requested page ${invalidPage}, but this PDF has ${document.numPages} pages.`,
+        truncated: false,
+      };
+    }
+
+    const chunks = [];
+    let characterCount = 0;
+    let foundText = false;
+    let truncated = false;
+    const append = (text) => {
+      const remaining = maxCharacters - characterCount;
+      if (remaining <= 0) {
+        truncated = true;
+        return;
+      }
+      const part = text.slice(0, remaining);
+      chunks.push(part);
+      characterCount += part.length;
+      if (part.length < text.length) truncated = true;
+    };
+
+    for (const pageNumber of pages) {
+      append(`${chunks.length ? "\n\n" : ""}--- Page ${pageNumber} ---\n`);
+      if (truncated) break;
+      const page = await document.getPage(pageNumber);
+      const reader = page.streamTextContent().getReader();
+      try {
+        while (!truncated) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          for (const item of value.items) {
+            if (!("str" in item) || !item.str) continue;
+            foundText = true;
+            append(item.str + (item.hasEOL ? "\n" : " "));
+            if (truncated) break;
+          }
+        }
+      } finally {
+        if (truncated) await reader.cancel().catch(() => undefined);
+        reader.releaseLock();
+      }
+      if (truncated) break;
+    }
+
+    const text = chunks.join("").trimEnd();
+    if (!foundText) {
+      return {
+        status: "no_text",
+        pageCount: document.numPages,
+        pages,
+        text: "No extractable text was found in the selected pages.",
+        truncated: false,
+      };
+    }
+    return { status: "ok", pageCount: document.numPages, pages, text, truncated };
+  } catch (error) {
+    if (isPasswordError(error)) {
+      return { status: "encrypted", pageCount: null, pages: [], text: "This PDF is encrypted and requires a password.", truncated: false };
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    return { status: "error", pageCount: null, pages: [], text: `PDF reading failed: ${message}`, truncated: false };
+  } finally {
+    await loadingTask?.destroy().catch(() => undefined);
+  }
+}
+
+parentPort?.postMessage(await extract());

server/src/llm/pdf-text.ts

@@ -0,0 +1,69 @@
+import { Worker } from "node:worker_threads";
+import { fileURLToPath } from "node:url";
+
+export const MAX_PDF_PAGES_PER_FETCH = 30;
+export const PDF_PARSE_TIMEOUT_MS = 8_000;
+const DEFAULT_PDF_TEXT_CHARACTERS = 12_000;
+
+export type PdfTextResult =
+  | { status: "ok"; pageCount: number; pages: number[]; text: string; truncated: boolean }
+  | { status: "no_pages"; pageCount: 0; pages: []; text: "This PDF has no pages."; truncated: false }
+  | { status: "no_text"; pageCount: number; pages: number[]; text: "No extractable text was found in the selected pages."; truncated: false }
+  | { status: "encrypted"; pageCount: null; pages: []; text: "This PDF is encrypted and requires a password."; truncated: false }
+  | { status: "error"; pageCount: number | null; pages: number[]; text: string; truncated: false };
+
+export function hasPdfMagic(bytes: Uint8Array) {
+  const headerEnd = Math.min(bytes.length, 1024);
+  return Buffer.from(bytes.subarray(0, headerEnd)).includes(Buffer.from("%PDF-"));
+}
+
+export function extractPdfText(
+  bytes: Uint8Array,
+  requestedPages?: number[],
+  maxCharacters = DEFAULT_PDF_TEXT_CHARACTERS,
+  timeoutMs = PDF_PARSE_TIMEOUT_MS,
+): Promise<PdfTextResult> {
+  return new Promise((resolve) => {
+    let settled = false;
+    let timeout: ReturnType<typeof setTimeout>;
+    let worker: Worker;
+    const finish = (result: PdfTextResult) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      void worker.terminate().catch(() => undefined);
+      resolve(result);
+    };
+
+    try {
+      worker = new Worker(fileURLToPath(new URL("./pdf-text-worker.mjs", import.meta.url)), {
+        workerData: { bytes: new Uint8Array(bytes), requestedPages, maxCharacters, maxPages: MAX_PDF_PAGES_PER_FETCH },
+        execArgv: [],
+      });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      resolve({ status: "error", pageCount: null, pages: [], text: `PDF reader could not start: ${message}`, truncated: false });
+      return;
+    }
+
+    timeout = setTimeout(() => {
+      finish({
+        status: "error",
+        pageCount: null,
+        pages: [],
+        text: `PDF reading exceeded its time limit (${timeoutMs} ms).`,
+        truncated: false,
+      });
+    }, timeoutMs);
+    worker.once("message", (result: PdfTextResult) => finish(result));
+    worker.once("error", (error) => {
+      const message = error instanceof Error ? error.message : String(error);
+      finish({ status: "error", pageCount: null, pages: [], text: `PDF reading failed: ${message}`, truncated: false });
+    });
+    worker.once("exit", (code) => {
+      if (!settled) {
+        finish({ status: "error", pageCount: null, pages: [], text: `PDF reader exited unexpectedly (${code}).`, truncated: false });
+      }
+    });
+  });
+}