Db to local disk (#42)

* copy db from hub to local

* add version to deeplake

* update .gitignore

* add completion time log

* save DB with overwrite=True

* remove time log

.gitignore

@@ -162,7 +162,7 @@ cython_debug/
 *.zip
 deeplake_store/
 .DS_Store
-__pycache__/
-.env
-env/
+
 .vscode/
+evals/
+local_dataset/

app.py

@@ -1,6 +1,7 @@
 import logging
 import os
 from typing import Optional
+import time
 
 import gradio as gr
 import pandas as pd
@@ -146,7 +147,6 @@ def get_answer(history, sources: Optional[list[str]] = None):
         display_ui_to_source = {
             ui: src for ui, src in zip(AVAILABLE_SOURCES_UI, AVAILABLE_SOURCES)
         }
-
         sources_renamed = [display_ui_to_source[disp] for disp in sources]
         completion = buster.process_input(user_input, sources=sources_renamed)
 

cfg.py

@@ -8,6 +8,7 @@ from buster.formatters.prompts import PromptFormatter
 from buster.retriever import DeepLakeRetriever, Retriever
 from buster.tokenizers import GPTTokenizer
 from buster.validators import QuestionAnswerValidator, Validator
+import deeplake
 
 from utils import init_mongo_db
 
@@ -31,6 +32,13 @@ DEEPLAKE_DATASET_PATH = os.getenv(
     "DEEPLAKE_DATASET_PATH", f"hub://{DEEPLAKE_ORG}/{DEEPLAKE_DATASET}"
 )
 
+deeplake.deepcopy(
+    DEEPLAKE_DATASET_PATH,
+    "local_dataset",
+    overwrite=True,
+)
+DEEPLAKE_DATASET_PATH = "local_dataset"
+
 logger.info(f"{DEEPLAKE_DATASET_PATH=}")
 
 example_questions = [

requirements.txt

@@ -1,3 +1,3 @@
 git+https://github.com/jerpint/buster@better-fallback
 gradio==3.50.2
-deeplake
+deeplake==3.8.9