scripts/chunk_and_embed.py

from langchain.text_splitter import RecursiveCharacterTextSplitter
import pickle
import os

DOCS_PATH = "E:/courses/LangChain Project/main root/output/all_docs.pkl"
CHUNKS_PATH = "E:/courses/LangChain Project/main root/output/chunks.pkl"

if not os.path.exists(DOCS_PATH):
    raise FileNotFoundError("Run load_documents.py first")

with open(DOCS_PATH, "rb") as f:
    all_docs = pickle.load(f)

splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
chunks = splitter.split_documents(all_docs)
print(f"✅ Split into {len(chunks)} chunks")

with open(CHUNKS_PATH, "wb") as f:
    pickle.dump(chunks, f)
print(f"📦 Chunks saved to {CHUNKS_PATH}")