feat: extract text from fetched PDFs

server/src/llm/pdf-text-worker.mjs

@@ -0,0 +1,100 @@
+import { getDocument } from "pdfjs-dist/legacy/build/pdf.mjs";
+import { parentPort, workerData } from "node:worker_threads";
+
+const { bytes, requestedPages, maxCharacters, maxPages } = workerData;
+
+function isPasswordError(error) {
+  return Boolean(error && typeof error === "object" && error.name === "PasswordException");
+}
+
+async function extract() {
+  let loadingTask;
+  try {
+    loadingTask = getDocument({
+      data: bytes,
+      standardFontDataUrl: new URL("../../node_modules/pdfjs-dist/standard_fonts/", import.meta.url).href,
+      isEvalSupported: false,
+      disableFontFace: true,
+      stopAtErrors: true,
+    });
+    const document = await loadingTask.promise;
+    if (document.numPages === 0) {
+      return { status: "no_pages", pageCount: 0, pages: [], text: "This PDF has no pages.", truncated: false };
+    }
+
+    const pages = requestedPages?.length
+      ? [...new Set(requestedPages)].slice(0, maxPages)
+      : Array.from({ length: Math.min(document.numPages, maxPages) }, (_, index) => index + 1);
+    const invalidPage = pages.find((page) => page > document.numPages);
+    if (invalidPage) {
+      return {
+        status: "error",
+        pageCount: document.numPages,
+        pages: [],
+        text: `Requested page ${invalidPage}, but this PDF has ${document.numPages} pages.`,
+        truncated: false,
+      };
+    }
+
+    const chunks = [];
+    let characterCount = 0;
+    let foundText = false;
+    let truncated = false;
+    const append = (text) => {
+      const remaining = maxCharacters - characterCount;
+      if (remaining <= 0) {
+        truncated = true;
+        return;
+      }
+      const part = text.slice(0, remaining);
+      chunks.push(part);
+      characterCount += part.length;
+      if (part.length < text.length) truncated = true;
+    };
+
+    for (const pageNumber of pages) {
+      append(`${chunks.length ? "\n\n" : ""}--- Page ${pageNumber} ---\n`);
+      if (truncated) break;
+      const page = await document.getPage(pageNumber);
+      const reader = page.streamTextContent().getReader();
+      try {
+        while (!truncated) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          for (const item of value.items) {
+            if (!("str" in item) || !item.str) continue;
+            foundText = true;
+            append(item.str + (item.hasEOL ? "\n" : " "));
+            if (truncated) break;
+          }
+        }
+      } finally {
+        if (truncated) await reader.cancel().catch(() => undefined);
+        reader.releaseLock();
+      }
+      if (truncated) break;
+    }
+
+    const text = chunks.join("").trimEnd();
+    if (!foundText) {
+      return {
+        status: "no_text",
+        pageCount: document.numPages,
+        pages,
+        text: "No extractable text was found in the selected pages.",
+        truncated: false,
+      };
+    }
+    return { status: "ok", pageCount: document.numPages, pages, text, truncated };
+  } catch (error) {
+    if (isPasswordError(error)) {
+      return { status: "encrypted", pageCount: null, pages: [], text: "This PDF is encrypted and requires a password.", truncated: false };
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    return { status: "error", pageCount: null, pages: [], text: `PDF reading failed: ${message}`, truncated: false };
+  } finally {
+    await loadingTask?.destroy().catch(() => undefined);
+  }
+}
+
+parentPort?.postMessage(await extract());