Spaces:
Running
Running
move over to activeloop online hosting (#14)
Browse files
cfg.py
CHANGED
@@ -10,32 +10,22 @@ from buster.tokenizers import GPTTokenizer
|
|
10 |
from buster.validators import QuestionAnswerValidator, Validator
|
11 |
from huggingface_hub import hf_hub_download
|
12 |
|
13 |
-
from utils import extract_zip
|
14 |
-
|
15 |
logger = logging.getLogger(__name__)
|
16 |
logging.basicConfig(level=logging.INFO)
|
17 |
|
18 |
-
#
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
HUB_TOKEN = os.getenv("HUB_TOKEN")
|
23 |
-
REPO_ID = os.getenv("HF_DATASET")
|
24 |
|
25 |
DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "wiki_tai_langchain")
|
26 |
-
|
27 |
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
repo_type="dataset",
|
32 |
-
filename=ZIP_FILE,
|
33 |
-
token=HUB_TOKEN,
|
34 |
-
local_dir=".",
|
35 |
)
|
36 |
|
37 |
-
extract_zip(zip_file_path=ZIP_FILE, output_path=DEEPLAKE_DATASET)
|
38 |
-
|
39 |
example_questions = [
|
40 |
"What is the LLama model?",
|
41 |
"What is a LLM?",
|
|
|
10 |
from buster.validators import QuestionAnswerValidator, Validator
|
11 |
from huggingface_hub import hf_hub_download
|
12 |
|
|
|
|
|
13 |
logger = logging.getLogger(__name__)
|
14 |
logging.basicConfig(level=logging.INFO)
|
15 |
|
16 |
+
# required
|
17 |
+
ACTIVELOOP_TOKEN = os.getenv("ACTIVELOOP_TOKEN")
|
18 |
+
if ACTIVELOOP_TOKEN is None:
|
19 |
+
logger.warning("No activeloop token found, you will not be able to fetch data.")
|
|
|
|
|
20 |
|
21 |
DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "wiki_tai_langchain")
|
22 |
+
DEEPLAKE_ORG = os.getenv("DEEPLAKE_ORG", "towards_ai")
|
23 |
|
24 |
+
# if you want to use a local dataset, set the env. variable, it overrides all others
|
25 |
+
DEEPLAKE_DATASET_PATH = os.getenv(
|
26 |
+
"DEEPLAKE_DATASET_PATH", f"hub://{DEEPLAKE_ORG}/{DEEPLAKE_DATASET}"
|
|
|
|
|
|
|
|
|
27 |
)
|
28 |
|
|
|
|
|
29 |
example_questions = [
|
30 |
"What is the LLama model?",
|
31 |
"What is a LLM?",
|