Spaces:
Sleeping
Sleeping
File size: 691 Bytes
5f096cc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
import pickle
import os
DOCS_PATH = "E:/courses/LangChain Project/main root/output/all_docs.pkl"
CHUNKS_PATH = "E:/courses/LangChain Project/main root/output/chunks.pkl"
if not os.path.exists(DOCS_PATH):
raise FileNotFoundError("Run load_documents.py first")
with open(DOCS_PATH, "rb") as f:
all_docs = pickle.load(f)
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
chunks = splitter.split_documents(all_docs)
print(f"✅ Split into {len(chunks)} chunks")
with open(CHUNKS_PATH, "wb") as f:
pickle.dump(chunks, f)
print(f"📦 Chunks saved to {CHUNKS_PATH}") |