非结构化文档加载与转换

import { UnstructuredLoader } from "langchain/document_loaders/fs/unstructured";
 
const loader = new UnstructuredLoader("./docs/report.pdf", {
  strategy: "hi_res", // hi_res=高精度（含版式），fast=更快
  pdfInference: true,
});
 
const docs = await loader.load();
// 每个 doc.metadata 含 category（Title/ListItem/Table 等）
console.log(docs[0].metadata.category, docs[0].pageContent);

import { UnstructuredDirectoryLoader } from "langchain/document_loaders/fs/unstructured";
import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
 
const dirLoader = new UnstructuredDirectoryLoader("./kb/", {
  recursive: true,
  // 按元素类型过滤，只保留正文与标题
});
 
const raw = await dirLoader.load();
const splits = await new RecursiveCharacterTextSplitter({
  chunkSize: 800,
  chunkOverlap: 80,
}).splitDocuments(raw);

const clean = raw.filter(
  (d) => !["Header", "Footer", "PageNumber"].includes(d.metadata.category)
);