data/embed_documents.py

import pandas as pd
from buster.documents_manager import DeepLakeDocumentsManager

if __name__ == "__main__":
    vector_store_path = "/Users/louis/Downloads/wiki_tai_langchain_hf_llm"
    chunk_file = "./llm_course.csv"
    overwrite = False

    df = pd.read_csv(chunk_file)

    print(f"before drop: {len(df)}")
    df = df.dropna()
    print(f"after drop: {len(df)}")

    dm = DeepLakeDocumentsManager(
        vector_store_path,
        overwrite=overwrite,
        required_columns=["url", "source", "content", "title"],
    )
    dm.batch_add(df)
    zipped_file_path = dm.to_zip()
    print(f"Contents zipped to: {zipped_file_path}")