feat: extract text from fetched PDFs

server/src/llm/bounded-response.ts

@@ -0,0 +1,36 @@
+export const MAX_FETCH_RESPONSE_BYTES = 10 * 1024 * 1024;
+
+export async function readBoundedResponseBytes(response: Response, maxBytes: number) {
+  const contentLength = Number(response.headers.get("content-length"));
+  if (Number.isFinite(contentLength) && contentLength > maxBytes) {
+    await response.body?.cancel();
+    throw new Error(`URL response exceeds the ${Math.floor(maxBytes / (1024 * 1024))} MiB download limit.`);
+  }
+
+  const reader = response.body?.getReader();
+  if (!reader) return new Uint8Array();
+  const chunks: Uint8Array[] = [];
+  let totalBytes = 0;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      totalBytes += value.byteLength;
+      if (totalBytes > maxBytes) {
+        await reader.cancel();
+        throw new Error(`URL response exceeds the ${Math.floor(maxBytes / (1024 * 1024))} MiB download limit.`);
+      }
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const bytes = new Uint8Array(totalBytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return bytes;
+}