feat: extract text from fetched PDFs
This commit is contained in:
Generated
+263
@@ -21,6 +21,7 @@
|
||||
"html-to-text": "^9.0.5",
|
||||
"marked": "17.0.2",
|
||||
"openai": "^6.16.0",
|
||||
"pdfjs-dist": "5.4.394",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"prisma": "^6.6.0",
|
||||
"web-push": "^3.6.7",
|
||||
@@ -775,6 +776,256 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.100.tgz",
|
||||
"integrity": "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"workspaces": [
|
||||
"e2e/*"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas-android-arm64": "0.1.100",
|
||||
"@napi-rs/canvas-darwin-arm64": "0.1.100",
|
||||
"@napi-rs/canvas-darwin-x64": "0.1.100",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "0.1.100",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "0.1.100",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.100",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "0.1.100",
|
||||
"@napi-rs/canvas-linux-x64-musl": "0.1.100",
|
||||
"@napi-rs/canvas-win32-arm64-msvc": "0.1.100",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "0.1.100"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.100.tgz",
|
||||
"integrity": "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.100.tgz",
|
||||
"integrity": "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.100.tgz",
|
||||
"integrity": "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.100.tgz",
|
||||
"integrity": "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.100.tgz",
|
||||
"integrity": "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.100.tgz",
|
||||
"integrity": "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.100.tgz",
|
||||
"integrity": "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.100.tgz",
|
||||
"integrity": "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.100.tgz",
|
||||
"integrity": "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-arm64-msvc": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.100.tgz",
|
||||
"integrity": "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.100.tgz",
|
||||
"integrity": "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@pinojs/redact": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
|
||||
@@ -1939,6 +2190,18 @@
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/pdfjs-dist": {
|
||||
"version": "5.4.394",
|
||||
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.394.tgz",
|
||||
"integrity": "sha512-9ariAYGqUJzx+V/1W4jHyiyCep6IZALmDzoaTLZ6VNu8q9LWi1/ukhzHgE2Xsx96AZi0mbZuK4/ttIbqSbLypg==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=20.16.0 || >=22.3.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas": "^0.1.81"
|
||||
}
|
||||
},
|
||||
"node_modules/peberminta": {
|
||||
"version": "0.9.0",
|
||||
"resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz",
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
"html-to-text": "^9.0.5",
|
||||
"marked": "17.0.2",
|
||||
"openai": "^6.16.0",
|
||||
"pdfjs-dist": "5.4.394",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"prisma": "^6.6.0",
|
||||
"web-push": "^3.6.7",
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
export const MAX_FETCH_RESPONSE_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
export async function readBoundedResponseBytes(response: Response, maxBytes: number) {
|
||||
const contentLength = Number(response.headers.get("content-length"));
|
||||
if (Number.isFinite(contentLength) && contentLength > maxBytes) {
|
||||
await response.body?.cancel();
|
||||
throw new Error(`URL response exceeds the ${Math.floor(maxBytes / (1024 * 1024))} MiB download limit.`);
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) return new Uint8Array();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let totalBytes = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
totalBytes += value.byteLength;
|
||||
if (totalBytes > maxBytes) {
|
||||
await reader.cancel();
|
||||
throw new Error(`URL response exceeds the ${Math.floor(maxBytes / (1024 * 1024))} MiB download limit.`);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
const bytes = new Uint8Array(totalBytes);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
bytes.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
@@ -6,10 +6,12 @@ import { promisify } from "node:util";
|
||||
import { convert as htmlToText } from "html-to-text";
|
||||
import { z } from "zod";
|
||||
import { buildBrowserLikeNavigationHeaders } from "../browser-fetch-headers.js";
|
||||
import { MAX_FETCH_RESPONSE_BYTES, readBoundedResponseBytes } from "./bounded-response.js";
|
||||
import { env } from "../env.js";
|
||||
import { searchBrave } from "../search/brave.js";
|
||||
import { exaClient } from "../search/exa.js";
|
||||
import { searchSearxng } from "../search/searxng.js";
|
||||
import { extractPdfText, hasPdfMagic, MAX_PDF_PAGES_PER_FETCH } from "./pdf-text.js";
|
||||
import type { ChatMessage } from "./types.js";
|
||||
|
||||
export const MAX_TOOL_ROUNDS = env.CHAT_MAX_TOOL_ROUNDS;
|
||||
@@ -45,6 +47,7 @@ const FetchUrlArgsSchema = z
|
||||
.object({
|
||||
url: z.string().trim().url(),
|
||||
maxCharacters: z.coerce.number().int().min(500).max(MAX_FETCH_MAX_CHARACTERS).optional(),
|
||||
pages: z.array(z.coerce.number().int().min(1).max(1_000_000)).min(1).max(MAX_PDF_PAGES_PER_FETCH).optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
@@ -163,7 +166,7 @@ const BASE_CHAT_TOOLS: any[] = [
|
||||
function: {
|
||||
name: "fetch_url",
|
||||
description:
|
||||
"Fetch a webpage by URL and return readable plaintext content extracted from the page for deeper inspection.",
|
||||
"Fetch a webpage by URL and return readable plaintext content. Text PDFs are extracted page by page; use pages:[3,4] to retrieve selected pages after a truncated result. Scanned PDFs have no extractable text.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
@@ -174,6 +177,12 @@ const BASE_CHAT_TOOLS: any[] = [
|
||||
maximum: MAX_FETCH_MAX_CHARACTERS,
|
||||
description: "Maximum response text characters returned (default 12000).",
|
||||
},
|
||||
pages: {
|
||||
type: "array",
|
||||
items: { type: "integer", minimum: 1, maximum: 1000000 },
|
||||
maxItems: MAX_PDF_PAGES_PER_FETCH,
|
||||
description: "For PDFs, extract these page numbers only (up to 30 per fetch).",
|
||||
},
|
||||
},
|
||||
required: ["url"],
|
||||
additionalProperties: false,
|
||||
@@ -561,48 +570,69 @@ async function runFetchUrlTool(input: unknown): Promise<ToolRunOutcome> {
|
||||
signal: controller.signal,
|
||||
headers: buildBrowserLikeNavigationHeaders(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error(`Fetch failed with status ${response.status}.`);
|
||||
}
|
||||
|
||||
const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
|
||||
const bytes = await readBoundedResponseBytes(response, MAX_FETCH_RESPONSE_BYTES);
|
||||
const url = response.url || parsed.toString();
|
||||
if (contentType.includes("application/pdf") || hasPdfMagic(bytes)) {
|
||||
if (!hasPdfMagic(bytes)) {
|
||||
return { ok: false, url, status: response.status, contentType: contentType || null, pdfStatus: "invalid", text: "The server declared a PDF, but the downloaded file does not have a valid PDF header." };
|
||||
}
|
||||
const result = await extractPdfText(bytes, args.pages, maxCharacters);
|
||||
const text = result.truncated
|
||||
? `${result.text}\n\n[truncated; re-fetch selected pages with pages:[N,...]]`
|
||||
: result.text;
|
||||
return {
|
||||
ok: result.status === "ok",
|
||||
url,
|
||||
status: response.status,
|
||||
contentType: contentType || "application/pdf",
|
||||
pdfStatus: result.status,
|
||||
pageCount: result.pageCount,
|
||||
pages: result.pages,
|
||||
pagesTruncated: result.pageCount !== null && result.pageCount > result.pages.length,
|
||||
truncated: result.truncated,
|
||||
text,
|
||||
};
|
||||
}
|
||||
|
||||
const body = new TextDecoder().decode(bytes);
|
||||
const isHtml = contentType.includes("text/html") || /<!doctype html|<html[\s>]/i.test(body);
|
||||
const extracted = isHtml
|
||||
? htmlToText(body, {
|
||||
wordwrap: false,
|
||||
preserveNewlines: true,
|
||||
selectors: [
|
||||
{ selector: "img", format: "skip" },
|
||||
{ selector: "script", format: "skip" },
|
||||
{ selector: "style", format: "skip" },
|
||||
{ selector: "noscript", format: "skip" },
|
||||
{ selector: "a", options: { ignoreHref: true } },
|
||||
],
|
||||
})
|
||||
: body;
|
||||
const normalized = compactWhitespace(extracted);
|
||||
const truncated = normalized.length > maxCharacters;
|
||||
const text = truncated
|
||||
? `${normalized.slice(0, maxCharacters)}\n\n[truncated ${normalized.length - maxCharacters} characters]`
|
||||
: normalized;
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
url,
|
||||
status: response.status,
|
||||
contentType: contentType || null,
|
||||
title: isHtml ? extractHtmlTitle(body) : null,
|
||||
truncated,
|
||||
text,
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed with status ${response.status}.`);
|
||||
}
|
||||
|
||||
const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
|
||||
const body = await response.text();
|
||||
const isHtml = contentType.includes("text/html") || /<!doctype html|<html[\s>]/i.test(body);
|
||||
|
||||
let extracted = body;
|
||||
if (isHtml) {
|
||||
extracted = htmlToText(body, {
|
||||
wordwrap: false,
|
||||
preserveNewlines: true,
|
||||
selectors: [
|
||||
{ selector: "img", format: "skip" },
|
||||
{ selector: "script", format: "skip" },
|
||||
{ selector: "style", format: "skip" },
|
||||
{ selector: "noscript", format: "skip" },
|
||||
{ selector: "a", options: { ignoreHref: true } },
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
const normalized = compactWhitespace(extracted);
|
||||
const truncated = normalized.length > maxCharacters;
|
||||
const text = truncated
|
||||
? `${normalized.slice(0, maxCharacters)}\n\n[truncated ${normalized.length - maxCharacters} characters]`
|
||||
: normalized;
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
url: response.url || parsed.toString(),
|
||||
status: response.status,
|
||||
contentType: contentType || null,
|
||||
title: isHtml ? extractHtmlTitle(body) : null,
|
||||
truncated,
|
||||
text,
|
||||
};
|
||||
}
|
||||
|
||||
function shellQuote(value: string) {
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import { getDocument } from "pdfjs-dist/legacy/build/pdf.mjs";
|
||||
import { parentPort, workerData } from "node:worker_threads";
|
||||
|
||||
const { bytes, requestedPages, maxCharacters, maxPages } = workerData;
|
||||
|
||||
function isPasswordError(error) {
|
||||
return Boolean(error && typeof error === "object" && error.name === "PasswordException");
|
||||
}
|
||||
|
||||
async function extract() {
|
||||
let loadingTask;
|
||||
try {
|
||||
loadingTask = getDocument({
|
||||
data: bytes,
|
||||
standardFontDataUrl: new URL("../../node_modules/pdfjs-dist/standard_fonts/", import.meta.url).href,
|
||||
isEvalSupported: false,
|
||||
disableFontFace: true,
|
||||
stopAtErrors: true,
|
||||
});
|
||||
const document = await loadingTask.promise;
|
||||
if (document.numPages === 0) {
|
||||
return { status: "no_pages", pageCount: 0, pages: [], text: "This PDF has no pages.", truncated: false };
|
||||
}
|
||||
|
||||
const pages = requestedPages?.length
|
||||
? [...new Set(requestedPages)].slice(0, maxPages)
|
||||
: Array.from({ length: Math.min(document.numPages, maxPages) }, (_, index) => index + 1);
|
||||
const invalidPage = pages.find((page) => page > document.numPages);
|
||||
if (invalidPage) {
|
||||
return {
|
||||
status: "error",
|
||||
pageCount: document.numPages,
|
||||
pages: [],
|
||||
text: `Requested page ${invalidPage}, but this PDF has ${document.numPages} pages.`,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
const chunks = [];
|
||||
let characterCount = 0;
|
||||
let foundText = false;
|
||||
let truncated = false;
|
||||
const append = (text) => {
|
||||
const remaining = maxCharacters - characterCount;
|
||||
if (remaining <= 0) {
|
||||
truncated = true;
|
||||
return;
|
||||
}
|
||||
const part = text.slice(0, remaining);
|
||||
chunks.push(part);
|
||||
characterCount += part.length;
|
||||
if (part.length < text.length) truncated = true;
|
||||
};
|
||||
|
||||
for (const pageNumber of pages) {
|
||||
append(`${chunks.length ? "\n\n" : ""}--- Page ${pageNumber} ---\n`);
|
||||
if (truncated) break;
|
||||
const page = await document.getPage(pageNumber);
|
||||
const reader = page.streamTextContent().getReader();
|
||||
try {
|
||||
while (!truncated) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
for (const item of value.items) {
|
||||
if (!("str" in item) || !item.str) continue;
|
||||
foundText = true;
|
||||
append(item.str + (item.hasEOL ? "\n" : " "));
|
||||
if (truncated) break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (truncated) await reader.cancel().catch(() => undefined);
|
||||
reader.releaseLock();
|
||||
}
|
||||
if (truncated) break;
|
||||
}
|
||||
|
||||
const text = chunks.join("").trimEnd();
|
||||
if (!foundText) {
|
||||
return {
|
||||
status: "no_text",
|
||||
pageCount: document.numPages,
|
||||
pages,
|
||||
text: "No extractable text was found in the selected pages.",
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
return { status: "ok", pageCount: document.numPages, pages, text, truncated };
|
||||
} catch (error) {
|
||||
if (isPasswordError(error)) {
|
||||
return { status: "encrypted", pageCount: null, pages: [], text: "This PDF is encrypted and requires a password.", truncated: false };
|
||||
}
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return { status: "error", pageCount: null, pages: [], text: `PDF reading failed: ${message}`, truncated: false };
|
||||
} finally {
|
||||
await loadingTask?.destroy().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
parentPort?.postMessage(await extract());
|
||||
@@ -0,0 +1,69 @@
|
||||
import { Worker } from "node:worker_threads";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const MAX_PDF_PAGES_PER_FETCH = 30;
|
||||
export const PDF_PARSE_TIMEOUT_MS = 8_000;
|
||||
const DEFAULT_PDF_TEXT_CHARACTERS = 12_000;
|
||||
|
||||
export type PdfTextResult =
|
||||
| { status: "ok"; pageCount: number; pages: number[]; text: string; truncated: boolean }
|
||||
| { status: "no_pages"; pageCount: 0; pages: []; text: "This PDF has no pages."; truncated: false }
|
||||
| { status: "no_text"; pageCount: number; pages: number[]; text: "No extractable text was found in the selected pages."; truncated: false }
|
||||
| { status: "encrypted"; pageCount: null; pages: []; text: "This PDF is encrypted and requires a password."; truncated: false }
|
||||
| { status: "error"; pageCount: number | null; pages: number[]; text: string; truncated: false };
|
||||
|
||||
export function hasPdfMagic(bytes: Uint8Array) {
|
||||
const headerEnd = Math.min(bytes.length, 1024);
|
||||
return Buffer.from(bytes.subarray(0, headerEnd)).includes(Buffer.from("%PDF-"));
|
||||
}
|
||||
|
||||
export function extractPdfText(
|
||||
bytes: Uint8Array,
|
||||
requestedPages?: number[],
|
||||
maxCharacters = DEFAULT_PDF_TEXT_CHARACTERS,
|
||||
timeoutMs = PDF_PARSE_TIMEOUT_MS,
|
||||
): Promise<PdfTextResult> {
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
let timeout: ReturnType<typeof setTimeout>;
|
||||
let worker: Worker;
|
||||
const finish = (result: PdfTextResult) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
void worker.terminate().catch(() => undefined);
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
try {
|
||||
worker = new Worker(fileURLToPath(new URL("./pdf-text-worker.mjs", import.meta.url)), {
|
||||
workerData: { bytes: new Uint8Array(bytes), requestedPages, maxCharacters, maxPages: MAX_PDF_PAGES_PER_FETCH },
|
||||
execArgv: [],
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
resolve({ status: "error", pageCount: null, pages: [], text: `PDF reader could not start: ${message}`, truncated: false });
|
||||
return;
|
||||
}
|
||||
|
||||
timeout = setTimeout(() => {
|
||||
finish({
|
||||
status: "error",
|
||||
pageCount: null,
|
||||
pages: [],
|
||||
text: `PDF reading exceeded its time limit (${timeoutMs} ms).`,
|
||||
truncated: false,
|
||||
});
|
||||
}, timeoutMs);
|
||||
worker.once("message", (result: PdfTextResult) => finish(result));
|
||||
worker.once("error", (error) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
finish({ status: "error", pageCount: null, pages: [], text: `PDF reading failed: ${message}`, truncated: false });
|
||||
});
|
||||
worker.once("exit", (code) => {
|
||||
if (!settled) {
|
||||
finish({ status: "error", pageCount: null, pages: [], text: `PDF reader exited unexpectedly (${code}).`, truncated: false });
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { readBoundedResponseBytes } from "../src/llm/bounded-response.js";
|
||||
|
||||
const oneMiB = 1024 * 1024;
|
||||
|
||||
test("reads response bytes within the declared size limit", async () => {
|
||||
const bytes = await readBoundedResponseBytes(new Response("small"), oneMiB);
|
||||
assert.equal(new TextDecoder().decode(bytes), "small");
|
||||
});
|
||||
|
||||
test("rejects oversized responses from Content-Length before buffering", async () => {
|
||||
const response = new Response("x", { headers: { "content-length": String(oneMiB + 1) } });
|
||||
await assert.rejects(readBoundedResponseBytes(response, oneMiB), /exceeds the 1 MiB download limit/);
|
||||
});
|
||||
|
||||
test("rejects streamed responses when Content-Length is absent", async () => {
|
||||
const response = new Response(new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(oneMiB));
|
||||
controller.enqueue(new Uint8Array(1));
|
||||
controller.close();
|
||||
},
|
||||
}));
|
||||
await assert.rejects(readBoundedResponseBytes(response, oneMiB), /exceeds the 1 MiB download limit/);
|
||||
});
|
||||
@@ -4,6 +4,7 @@ import { type ToolAwareStreamingEvent } from "../src/llm/chat-tools.js";
|
||||
import { completeWithChatCompletionsApi, streamWithChatCompletionsApi } from "../src/llm/protocols/chat-completions-api.js";
|
||||
import { completeWithMessagesApi, streamWithMessagesApi } from "../src/llm/protocols/messages-api.js";
|
||||
import { streamWithResponsesApi } from "../src/llm/protocols/responses-api.js";
|
||||
import { createPdf } from "./pdf-fixture.js";
|
||||
|
||||
async function* streamFrom(events: any[]) {
|
||||
for (const event of events) {
|
||||
@@ -241,10 +242,12 @@ test("fetch_url sends browser-like navigation headers", async () => {
|
||||
|
||||
try {
|
||||
let requestCount = 0;
|
||||
const requestBodies: any[] = [];
|
||||
const client = {
|
||||
chat: {
|
||||
completions: {
|
||||
create: async () => {
|
||||
create: async (body: any) => {
|
||||
requestBodies.push(body);
|
||||
requestCount += 1;
|
||||
if (requestCount === 1) {
|
||||
return {
|
||||
@@ -295,12 +298,108 @@ test("fetch_url sends browser-like navigation headers", async () => {
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
});
|
||||
const toolResult = requestBodies[1]?.messages?.find((message: any) => message.role === "tool");
|
||||
const fetched = JSON.parse(toolResult?.content ?? "{}");
|
||||
assert.match(fetched.text, /Consumer price index/);
|
||||
assert.equal(fetched.title, "CPI");
|
||||
assert.equal(result.toolEvents[0]?.status, "completed");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("fetch_url extracts a mislabeled PDF and returns page handles", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
|
||||
const pdf = createPdf(["Revenue is 42.", "Costs are 18."]);
|
||||
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
fetchCalls.push({ input, init });
|
||||
return new Response(pdf, { status: 200, headers: { "content-type": "text/plain" } });
|
||||
}) as typeof fetch;
|
||||
|
||||
try {
|
||||
const requestBodies: any[] = [];
|
||||
const client = {
|
||||
messages: {
|
||||
create: async (body: any) => {
|
||||
requestBodies.push(body);
|
||||
if (requestBodies.length === 1) {
|
||||
return {
|
||||
content: [{
|
||||
type: "tool_use",
|
||||
id: "pdf_call",
|
||||
name: "fetch_url",
|
||||
input: { url: "https://example.com/report.pdf", pages: [2] },
|
||||
}],
|
||||
usage: { input_tokens: 1, output_tokens: 1 },
|
||||
};
|
||||
}
|
||||
return { content: [{ type: "text", text: "Read page two" }], usage: { input_tokens: 1, output_tokens: 1 } };
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = await completeWithMessagesApi({
|
||||
client: client as any,
|
||||
model: "claude-test",
|
||||
messages: [{ role: "user", content: "Read the report" }],
|
||||
});
|
||||
const toolResult = requestBodies[1]?.messages?.at(-1)?.content?.[0];
|
||||
const fetched = JSON.parse(toolResult?.content ?? "{}");
|
||||
assert.equal(result.text, "Read page two");
|
||||
assert.equal(fetchCalls.length, 1);
|
||||
assert.equal(fetched.ok, true);
|
||||
assert.equal(fetched.pdfStatus, "ok");
|
||||
assert.equal(fetched.pageCount, 2);
|
||||
assert.deepEqual(fetched.pages, [2]);
|
||||
assert.equal(fetched.pagesTruncated, true);
|
||||
assert.match(fetched.text, /--- Page 2 ---/);
|
||||
assert.match(fetched.text, /Costs are 18/);
|
||||
assert.doesNotMatch(fetched.text, /Revenue is 42/);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("fetch_url marks encrypted and text-free PDFs as unavailable", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let responseBytes = createPdf([""]);
|
||||
globalThis.fetch = (async () => new Response(responseBytes, {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/pdf" },
|
||||
})) as typeof fetch;
|
||||
|
||||
try {
|
||||
for (const [pdf, expectedStatus] of [[createPdf([""]), "no_text"], [createPdf(["Secret"], true), "encrypted"]] as const) {
|
||||
responseBytes = pdf;
|
||||
const requestBodies: any[] = [];
|
||||
const client = {
|
||||
messages: {
|
||||
create: async (body: any) => {
|
||||
requestBodies.push(body);
|
||||
if (requestBodies.length === 1) {
|
||||
return {
|
||||
content: [{ type: "tool_use", id: "pdf_status_call", name: "fetch_url", input: { url: "https://example.com/report.pdf" } }],
|
||||
usage: { input_tokens: 1, output_tokens: 1 },
|
||||
};
|
||||
}
|
||||
return { content: [{ type: "text", text: "Unavailable" }], usage: { input_tokens: 1, output_tokens: 1 } };
|
||||
},
|
||||
},
|
||||
};
|
||||
await completeWithMessagesApi({ client: client as any, model: "claude-test", messages: [{ role: "user", content: "Read PDF" }] });
|
||||
const toolResult = requestBodies[1]?.messages?.at(-1)?.content?.[0];
|
||||
const fetched = JSON.parse(toolResult?.content ?? "{}");
|
||||
assert.equal(fetched.ok, false);
|
||||
assert.equal(fetched.pdfStatus, expectedStatus);
|
||||
assert.equal(toolResult?.is_error, true);
|
||||
assert.ok(fetched.text.length > 0);
|
||||
}
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("Messages API executes tool_use blocks and sends tool_result follow-up", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
export function createPdf(pages: string[], encrypted = false): Uint8Array {
|
||||
const fontId = 3 + pages.length * 2;
|
||||
const objects = [
|
||||
"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
`<< /Type /Pages /Kids [${pages.map((_, index) => `${3 + index * 2} 0 R`).join(" ")}] /Count ${pages.length} >>`,
|
||||
];
|
||||
for (let index = 0; index < pages.length; index += 1) {
|
||||
const pageId = 3 + index * 2;
|
||||
const contentId = pageId + 1;
|
||||
const text = pages[index]!;
|
||||
const operators = Array.from({ length: Math.ceil(text.length / 100) }, (_, partIndex) => {
|
||||
const part = text.slice(partIndex * 100, (partIndex + 1) * 100);
|
||||
const escaped = part.replaceAll("\\", "\\\\").replaceAll("(", "\\(").replaceAll(")", "\\)");
|
||||
return `(${escaped}) Tj`;
|
||||
});
|
||||
const content = text ? `BT /F1 12 Tf 72 720 Td ${operators.join(" ")} ET` : "";
|
||||
objects.push(
|
||||
`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontId} 0 R >> >> /Contents ${contentId} 0 R >>`,
|
||||
`<< /Length ${Buffer.byteLength(content)} >>\nstream\n${content}\nendstream`,
|
||||
);
|
||||
}
|
||||
objects.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>");
|
||||
const encryptionId = objects.length + 1;
|
||||
if (encrypted) objects.push(`<< /Filter /Standard /V 1 /R 2 /O <${"00".repeat(32)}> /U <${"00".repeat(32)}> /P -4 >>`);
|
||||
|
||||
let output = "%PDF-1.4\n";
|
||||
const offsets = [0];
|
||||
objects.forEach((object, index) => {
|
||||
offsets.push(Buffer.byteLength(output));
|
||||
output += `${index + 1} 0 obj\n${object}\nendobj\n`;
|
||||
});
|
||||
const xrefOffset = Buffer.byteLength(output);
|
||||
output += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
|
||||
for (const offset of offsets.slice(1)) output += `${String(offset).padStart(10, "0")} 00000 n \n`;
|
||||
output += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R${encrypted ? ` /Encrypt ${encryptionId} 0 R` : ""} >>\nstartxref\n${xrefOffset}\n%%EOF`;
|
||||
return new TextEncoder().encode(output);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { extractPdfText, hasPdfMagic } from "../src/llm/pdf-text.js";
|
||||
import { createPdf } from "./pdf-fixture.js";
|
||||
|
||||
test("extracts selectable text with page markers and selected-page support", async () => {
|
||||
const bytes = createPdf(["Quarterly revenue was 42.", "Costs were 18."]);
|
||||
const result = await extractPdfText(bytes, [2]);
|
||||
assert.equal(result.status, "ok");
|
||||
assert.equal(result.pageCount, 2);
|
||||
assert.deepEqual(result.pages, [2]);
|
||||
assert.match(result.text, /--- Page 2 ---/);
|
||||
assert.match(result.text, /Costs were 18\./);
|
||||
assert.doesNotMatch(result.text, /Quarterly revenue/);
|
||||
});
|
||||
|
||||
test("reports PDFs with no selectable text clearly", async () => {
|
||||
const result = await extractPdfText(createPdf([""]));
|
||||
assert.equal(result.status, "no_text");
|
||||
assert.match(result.text, /No extractable text/);
|
||||
});
|
||||
|
||||
test("reports encrypted PDFs clearly", async () => {
|
||||
const result = await extractPdfText(createPdf(["Secret"], true));
|
||||
assert.equal(result.status, "encrypted");
|
||||
assert.match(result.text, /encrypted and requires a password/);
|
||||
});
|
||||
|
||||
test("reports page-free PDFs and rejects invalid requested pages", async () => {
|
||||
const empty = await extractPdfText(createPdf([]));
|
||||
assert.equal(empty.status, "no_pages");
|
||||
assert.match(empty.text, /no pages/);
|
||||
const outOfRange = await extractPdfText(createPdf(["Only page"]), [2]);
|
||||
assert.equal(outOfRange.status, "error");
|
||||
assert.match(outOfRange.text, /has 1 pages/);
|
||||
});
|
||||
|
||||
test("limits each extraction to 30 pages and preserves the full page count", async () => {
|
||||
const result = await extractPdfText(createPdf(Array.from({ length: 31 }, (_, index) => `Text on page ${index + 1}.`)));
|
||||
assert.equal(result.status, "ok");
|
||||
assert.equal(result.pageCount, 31);
|
||||
assert.equal(result.pages.length, 30);
|
||||
assert.equal(result.pages.at(-1), 30);
|
||||
});
|
||||
|
||||
test("caps accumulated extracted text at the caller limit", async () => {
|
||||
const result = await extractPdfText(createPdf(["Long sentence. ".repeat(200)]), undefined, 50);
|
||||
assert.equal(result.status, "ok");
|
||||
assert.equal(result.truncated, true);
|
||||
assert.ok(result.text.length <= 50);
|
||||
});
|
||||
|
||||
test("terminates an over-time worker and returns an explicit timeout result", async () => {
|
||||
const result = await extractPdfText(createPdf(["Small document"]), undefined, 12_000, 1);
|
||||
assert.equal(result.status, "error");
|
||||
assert.match(result.text, /time limit/);
|
||||
});
|
||||
|
||||
test("recognizes PDF magic in a mislabeled or prefixed response", () => {
|
||||
assert.equal(hasPdfMagic(new TextEncoder().encode("%PDF-1.4")), true);
|
||||
assert.equal(hasPdfMagic(new TextEncoder().encode(`${" ".repeat(500)}%PDF-1.7`)), true);
|
||||
assert.equal(hasPdfMagic(new TextEncoder().encode("<html>not a PDF</html>")), false);
|
||||
});
|
||||
Reference in New Issue
Block a user