create_database.py

#load pdf
#split into chunks
#create the embeddings
#stor into chroma DB

from langchain_community.document_loaders import PyPDFLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
from langchain_mistralai import MistralAIEmbeddings
from dotenv import load_dotenv
import os

load_dotenv()

loader = PyPDFLoader("deep-learning-material-dept-ece-ase-blr-1.pdf")
docs = loader.load()

splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200
)

chunks = splitter.split_documents(docs)

embedding_model = MistralAIEmbeddings(model="mistral-embed")

vectorstore = Chroma.from_documents(
    documents = chunks,
    embedding= embedding_model,
    persist_directory= "chroma_db"
)