项目实战：企业知识库问答

pip install langchain langchain-community \
            sentence-transformers faiss-cpu \
            rank_bm25 openai

from langchain_community.document_loaders import DirectoryLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from sentence_transformers import SentenceTransformer
import faiss, numpy as np, pickle
 
embedder = SentenceTransformer("BAAI/bge-large-zh")
 
# 1) 加载
loader = DirectoryLoader("./kb_docs", glob="**/*.{pdf,md,txt,docx}")
docs = loader.load()
 
# 2) 切分
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=80)
chunks = splitter.split_documents(docs)
 
# 3) 向量化 + 建 FAISS 索引
texts = [c.page_content for c in chunks]
metas = [c.metadata for c in chunks]
vecs = embedder.encode(texts, normalize_embeddings=True)
index = faiss.IndexFlatIP(vecs.shape[1])   # 归一化后内积=余弦
index.add(np.array(vecs, dtype="float32"))
 
# 4) 持久化
faiss.write_index(index, "kb.index")
pickle.dump({"texts": texts, "metas": metas}, open("kb.pkl", "wb"))

import openai, json
client = openai.OpenAI()
 
def answer(question, k=5):
    # 检索
    qv = embedder.encode([question], normalize_embeddings=True)
    D, I = index.search(np.array(qv, dtype="float32"), k=20)
    cand = [{"text": texts[i], "meta": metas[i]} for i in I[0]]
 
    # 组装上下文（带编号）
    ctx = "\n".join(
        f"[{i+1}] {c['text']}（来源：{c['meta'].get('source')}）"
        for i, c in enumerate(cand[:k])
    )
    prompt = f"""你是企业知识库助手，仅依据资料作答，未知则说不知道，句末标 [n]。
【资料】
{ctx}
【问题】{question}
【回答】"""
 
    resp = client.chat.completions.create(
        model="gpt-4o-mini", messages=[{"role": "user", "content": prompt}],
        max_tokens=512,
    )
    return resp.choices[0].message.content

from sentence_transformers import CrossEncoder
reranker = CrossEncoder("BAAI/bge-reranker-large")
 
def rerank(question, cand, k=5):
    pairs = [(question, c["text"]) for c in cand]
    scores = reranker.predict(pairs)
    return [cand[i] for i in np.argsort(scores)[::-1][:k]]