Upgraded rag code

.gitignore

@@ -1 +1,2 @@
-**/.DS_Store
+**/.DS_Store
+**/__pycache__

rag.ipynb

@@ -0,0 +1,225 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "SCq5lAKuZxYx",
+        "outputId": "6c44cd5b-efe4-4364-d19c-4650be91f9c6"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Mounted at /content/gdrive\n"
+          ]
+        }
+      ],
+      "source": [
+        "from google.colab import drive\n",
+        "import os\n",
+        "\n",
+        "drive.mount('/content/gdrive')\n",
+        "\n",
+        "!ls\n",
+        "%cd /content/gdrive/MyDrive/rajat.bans/RAG/\n",
+        "!pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "/Users/lazyghost/VirtualEnvironments/langchain-rag-venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+            "  from .autonotebook import tqdm as notebook_tqdm\n",
+            "/Users/lazyghost/VirtualEnvironments/langchain-rag-venv/lib/python3.12/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
+            "  warnings.warn(\n"
+          ]
+        },
+        {
+          "data": {
+            "text/plain": [
+              "({'relation_answer': {'reasoning': \"No relevant ads found for the user's input 'Hola'.\",\n",
+              "   'classification': 0},\n",
+              "  'tokens_used_relation': 460,\n",
+              "  'question_answer': {'reasoning': '', 'question': '', 'options': []},\n",
+              "  'tokens_used_question': 0},\n",
+              " [])"
+            ]
+          },
+          "execution_count": 1,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "from rag import VARIABLE_MANAGER\n",
+        "vm = VARIABLE_MANAGER()\n",
+        "rag = vm.getRag()\n",
+        "# data = vm.QnAAdsSampleGenerationPreProcessing()\n",
+        "tot_cost = 0\n",
+        "rag.getRagResponse(\"Hola\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 59,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "zzJNW1fCcP3v",
+        "outputId": "6bcb20f5-6596-4e42-ffb7-d3a70566d2e8"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "20,"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "/usr/local/lib/python3.10/dist-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
+            "  warnings.warn(\n"
+          ]
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            " Total cost is up to now is 0.014402400000000001\n"
+          ]
+        }
+      ],
+      "source": [
+        "import pandas as pd\n",
+        "responses_file_name = './data/147_results_webmd_healthline_12Jun-18Jun_1000each_145BIGQSPRCR_QuestionSystemPromptImprovedClusteringAdded_.tsv'\n",
+        "try:\n",
+        "    responses = pd.read_csv(responses_file_name, sep='\\t')\n",
+        "except FileNotFoundError:\n",
+        "    responses = pd.DataFrame()\n",
+        "\n",
+        "new_rows = []\n",
+        "for i in range(len(responses), len(data)):\n",
+        "    print(i, end = ',')\n",
+        "    row = data.iloc[i, :]\n",
+        "    try:\n",
+        "        answer = {\n",
+        "            'domain_name': row['domain_name'],\n",
+        "            'url': row['url'],\n",
+        "            #   'input': '. '.join(row['stripped_url'].split('/')[3:]),\n",
+        "            'kwd_imp': row['kwd_imp'],\n",
+        "            'kwd_click': row['kwd_click'],\n",
+        "            'ad_click': row['ad_click'],\n",
+        "            'revenue': row['revenue'],\n",
+        "            'rank': row['rank'],\n",
+        "            'url_title': row['url_title'],\n",
+        "            'url_content': row['url_content'],\n",
+        "            'input': row['core_content'],\n",
+        "        }\n",
+        "\n",
+        "        reply, clustered_docs = rag.getRagResponse(row['core_content'])\n",
+        "        answer[\"relation_reasoning\"] = reply['relation_answer']['reasoning']\n",
+        "        answer[\"relation_classification\"] = reply['relation_answer']['classification']\n",
+        "        answer[\"relation_tokens_used\"] = reply['tokens_used_relation']\n",
+        "\n",
+        "        answer[\"reasoning\"] = reply['question_answer']['reasoning']\n",
+        "        answer[\"question\"] = reply['question_answer']['question']\n",
+        "        options = reply['question_answer']['options']\n",
+        "        options_res = \"\"\n",
+        "        for option in options:\n",
+        "            options_res += option + \"\\n\"\n",
+        "            for ad in options[option]:\n",
+        "                options_res += ad + \"\\n\"\n",
+        "            options_res += \"\\n\"\n",
+        "        answer[\"options\"] = options_res\n",
+        "        answer[\"options_count\"] = str(len(options))\n",
+        "        answer[\"question_tokens_used\"] = reply['tokens_used_question']\n",
+        "\n",
+        "        ads_data = \"\"\n",
+        "        for ind, cluster in enumerate(clustered_docs):\n",
+        "            ads_data += f\"*************** Cluster-:{ind+1} **************\\n\"\n",
+        "            for doc in cluster:\n",
+        "                ad = doc[0]\n",
+        "                ads_data += ad.page_content + \"\\n\"\n",
+        "                ads_data += \"publisher_url: \" + ad.metadata['publisher_url'] + \" | \"\n",
+        "                ads_data += \"keyword_term: \" + ad.metadata['keyword_term'] + \" | \"\n",
+        "                ads_data += \"ad_display_url: \" + ad.metadata['ad_display_url'] + \" | \"\n",
+        "                ads_data += \"revenue: \" + str(ad.metadata['revenue']) + \" | \"\n",
+        "                ads_data += \"ad_click_count: \" + str(ad.metadata['ad_click_count']) + \" | \"\n",
+        "                ads_data += \"RPC: \" + str(ad.metadata['RPC']) + \" | \"\n",
+        "                ads_data += \"Type: \" + ad.metadata['Type'] + \"\\n\"\n",
+        "                ads_data += \"Value: \" + str(doc[1]) + \"\\n\"\n",
+        "                ads_data += \"\\n\"\n",
+        "            ads_data += \"\\n\"\n",
+        "        answer[\"ads_data\"] = ads_data\n",
+        "\n",
+        "        cost = (answer[\"relation_tokens_used\"] + answer[\"question_tokens_used\"]) * 0.6/1000000\n",
+        "        tot_cost += cost\n",
+        "        answer['cost'] = float(cost)\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "    new_rows.append(answer)\n",
+        "\n",
+        "    if i % 10 == 0:\n",
+        "        print(\" Total cost is up to now is\", tot_cost)\n",
+        "        responses = pd.concat([responses, pd.DataFrame(new_rows)], ignore_index=True)\n",
+        "        responses.to_csv(responses_file_name, sep='\\t', index=False)\n",
+        "        new_rows = []\n",
+        "\n",
+        "responses = pd.concat([responses, pd.DataFrame(new_rows)], ignore_index=True)\n",
+        "responses.to_csv(responses_file_name, sep='\\t', index=False)\n",
+        "responses\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "5gRHp_nCJHlf",
+        "ScYo9Q38IbGr",
+        "Yd1qWPjlxCTd",
+        "P2soYnTaxE5c",
+        "DOLcMgW6IWX8"
+      ],
+      "gpuType": "T4",
+      "machine_shape": "hm",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.12.4"
+    },
+    "widgets": {
+      "application/vnd.jupyter.widget-state+json": {}
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

rag.py

@@ -1,39 +1,27 @@
-from dotenv import load_dotenv
-from langchain_community.vectorstores import FAISS
-from langchain_community.embeddings import HuggingFaceEmbeddings
-import gradio as gr
-from openai import OpenAI
-import random
-import pandas as pd
-import os
-import json
 from sklearn.cluster import KMeans, SpectralClustering
 from scipy.spatial.distance import euclidean
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
 import re
 import numpy as np
+from openai import OpenAI
+import json
 from itertools import count
-
-load_dotenv(override=True)
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-data_file_path = "./data/142_adclick_20May_20Jun_webmd_healthline_Health_dupRemoved0.8_someAdsCampaign.tsv"
-embedding_model_hf = "BAAI/bge-m3"
-embeddings_hf = HuggingFaceEmbeddings(model_name=embedding_model_hf)
-
+import time
+import os
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from dotenv import load_dotenv
+import pandas as pd
 
 class CLUSTERING:
-    def __init__(self):
-        self.clustering_algo = (
-            "kmeans-cc"  # ['kmeans-cc', 'kmeans-sp', 'spectral_clustering']
-        )
-
-    def cluster_embeddings(self, embeddings, no_of_clusters, no_of_points):
-        if self.clustering_algo in {"kmeans-cc", "kmeans-sp"}:
-            kmeans = KMeans(n_clusters=no_of_clusters, random_state=42)
+    def cluster_embeddings(self, embeddings, clustering_algo, no_of_clusters, no_of_points):
+        if clustering_algo in {"kmeans-cc", "kmeans-sp"}:
+            kmeans = KMeans(n_clusters=min(no_of_clusters, len(embeddings)), random_state=42, n_init="auto")
             kmeans.fit(embeddings)
             cluster_centers = kmeans.cluster_centers_
             labels = kmeans.labels_
 
-            if self.clustering_algo == "kmeans-cc":
+            if clustering_algo == "kmeans-cc":
                 clusters_indices = [[] for _ in range(no_of_clusters)]
                 for i, embedding in enumerate(embeddings):
                     cluster_idx = labels[i]
@@ -51,7 +39,7 @@ class CLUSTERING:
                         len(cluster) == no_of_points for cluster in clusters_indices
                     ):
                         break
-        elif self.clustering_algo == "spectral":
+        elif clustering_algo == "spectral":
             spectral_clustering = SpectralClustering(
                 n_clusters=no_of_clusters, affinity="nearest_neighbors", random_state=42
             )
@@ -67,24 +55,28 @@ class CLUSTERING:
             [cluster_point[0] for cluster_point in clusters_indices[i][:no_of_points]]
             for i in range(no_of_clusters)
         ]
-
-
+    
 class VECTOR_DB:
-    def __init__(self):
-        self.DB_FAISS_PATH = "./vectorstore/db_faiss_ads_20May_20Jun_webmd_healthline_Health_dupRemoved0.8"
-        self.default_threshold = 0.75
-        self.number_of_ads_to_fetch_from_db = 50
-        self.no_of_clusters = 3
-        self.no_of_ads_in_each_cluster = 6
+    def __init__(self, default_threshold, number_of_ads_to_fetch_from_db, clustering_algo, no_of_clusters, no_of_ads_in_each_cluster, DB_FAISS_PATH, embeddings_hf):
+        self.default_threshold = default_threshold
+        self.number_of_ads_to_fetch_from_db = number_of_ads_to_fetch_from_db
+        self.clustering_algo = clustering_algo # ['kmeans-cc', 'kmeans-sp', 'spectral_clustering']
+        self.no_of_clusters = no_of_clusters
+        self.no_of_ads_in_each_cluster = no_of_ads_in_each_cluster
+        self.embeddings_hf = embeddings_hf
         self.db = FAISS.load_local(
-            self.DB_FAISS_PATH, embeddings_hf, allow_dangerous_deserialization=True
+            DB_FAISS_PATH, embeddings_hf, allow_dangerous_deserialization=True
         )
 
-    def queryVectorDB(self, page_information, threshold):
+
+    def queryVectorDB(self, page_information, threshold = None):
         def remove_html_tags(text):
             clean = re.compile("<.*?>")
             return re.sub(clean, "", text)
 
+        if threshold == None:
+            threshold = self.default_threshold
+
         retreived_documents = [
             doc
             for doc in self.db.similarity_search_with_score(
@@ -98,13 +90,13 @@ class VECTOR_DB:
             )
         if len(retreived_documents):
             embeddings = np.array(
-                embeddings_hf.embed_documents(
+                self.embeddings_hf.embed_documents(
                     [doc[0].page_content for doc in retreived_documents]
                 )
             )
 
             clustered_indices = CLUSTERING().cluster_embeddings(
-                embeddings, self.no_of_clusters, self.no_of_ads_in_each_cluster
+                embeddings, self.clustering_algo, self.no_of_clusters, self.no_of_ads_in_each_cluster
             )
             documents_clusters = [
                 [retreived_documents[ind] for ind in cluster_indices]
@@ -115,94 +107,93 @@ class VECTOR_DB:
             return documents_clusters, best_value
         return [], 1
 
-
-class ADS_RAG:
+class FAISS_DB:
     def __init__(self):
-        self.client = OpenAI()
-        self.db = VECTOR_DB()
-        self.qa_model_name = "gpt-3.5-turbo"
-        self.relation_check_best_value_thresh = 0.6
-        self.bestRelationSystemPrompt = """You are an advertising concierge for text ads on websites. Given an INPUT and the available ad inventory (ADS_DATA), your task is to determine whether there are some relevant ADS to INPUT are present in ADS_DATA. ADS WHICH DON'T MATCH USER'S INTENT SHOULD BE CONSIDERED IRRELEVANT
-
----------------------------------------
-
-**Sample INPUT***: What Causes Bright-Yellow Urine and Other Changes in Color?
-
-Expected json output :
-{ 
-"reasoning" : "Given the user's search for 'What Causes Bright-Yellow Urine and Other Changes in Color?', it is evident that they are looking for information related to the causes and implications of changes in urine color. Therefore, ads that are related to urine color. However, none of the provided ads are related to urine. Ads 1, 2, and 3 are focused on chronic lymphocytic leukemia (CLL) treatment, Ad 4 addresses high blood pressure, and Ad 5 is about migraine treatment. Since none of these ads are not at all relevant to the urine, they can all be considered irrelevant to the user's intent.",
-"classification": 0
-}
-------------------------------------------------
-
-**Sample INPUT**: The Effects of Aging on Skin
-
-Expected json output :
-{ 
-"reasoning" : "Given the user's search for 'The Effects of Aging on Skin,' it is clear that they are seeking information related to skin aging. Therefore, the ads that are relevant to skin effects should be considered. Ads 1 and 2 focus on wrinkle treatment and anti-aging solutions, making them pertinent to the user's intent. Ad 3 targets vitiligo and not general skin aging but it is related to skin effect. So it is also relevant. Ads 4 and 5 are about advanced lung cancer, which do not address the interest in skin. Ads 1 and 2, 3 are most relevant to the user's search. So ADS_DATA is relevant to INPUT. ",
-"classification": 1
-}
----------------------------------------
-
-The ADS_DATA provided to you is as follows:
-"""
-
-        self.bestQuestionSystemPrompt = """1. You are an advertising concierge for text ads on websites. Given an INPUT and the available ad inventory (ADS_DATA), your task is to form a relevant QUESTION to ask the user visiting the webpage. This question should help identify the user's intent behind visiting the webpage and should be highly attractive. 
-2. Now form a highly attractive/lucrative and diverse/mutually exclusive OPTION which should be both the answer for the QUESTION and related to ads in this cluster.  
-3. Try to generate intelligent creatives for advertising and keep QUESTION within 70 characters and either 2, 3 or 4 options with each OPTION within 4 to 6 words.
-4. Provide your REASONING behind choosing the QUESTION and the OPTIONS. Now provide the QUESTION and the OPTIONS. Along with each OPTION, provide the ads from ADS_DATA that you associated with it.
-
----------------------------------------
-
-<Sample INPUT> 
-The Effects of Aging on Skin
-
-<Sample ADS_DATA> 
-{"Cluster 1 Ads": {"Ad 1": "Forget Retinol, Use This Household Item To Fill In Wrinkles - Celebrities Are Ditching Pricey Facelifts For This."}, "Cluster 2 Ads": {"Ad 2": "Stop Covering Your Wrinkles with Make Up - Do This Instead.", "Ad 3": "Living With Migraines? - Discover A Treatment Option. Learn about a type of prescription migraine treatment called CGRP receptor antagonists. Discover a range of resources that may help people dealing with migraines"}, "Cluster 3 Ads": {"Ad 4": "What is Advanced Skin Cancer? - Find Disease Information Here.Find Facts About Advanced Skin Cancer and a Potential Treatment Option.", "Ad 5": "Learn About Advanced Melanoma - Find Disease Information Here.Find Facts About Advanced Melanoma and a Potential Treatment Option.", "Ad 6": "Treatment For CKD - Reduce Risk Of Progressing CKD. Ask About A Treatment That Can Help Reduce Your Risk Of Kidney Failure", "Ad 7": "Are You Living With Vitiligo? - For Patients & Caregivers.Discover An FDA-Approved Topical Cream That May Help With Nonsegmental Vitiligo Repigmentation. Learn About A Copay Savings Card For Eligible Patients With Vitiligo."}]
-
-<Expected json output>
-{ 
-"reasoning" : "Among the seven ads in **Sample ADS_DATA**, Ads 3 and 6 are irrelevant to the INPUT, so they should be discarded. Ad 1, 2 closely aligns with the user's intent. Ads 4, 5, and 7 are also relevant to INPUT. The question will be formed in a way to connect the PAGE content with the goals of these five relevant ads, making sure they appeal to both specific and general user interests, with the OPTIONS being the answer for QUESTION(it is ensured that no irrelevant options are formed)",
-"question": "Interested in methods to combat aging skin?",
-"options": {"1. Retinol Alternatives for Wrinkle Treatment." : ["Ad 1: Forget Retinol, Use This Household Item To Fill In Wrinkles - Celebrities Are Ditching Pricey Facelifts For This."], "2. Reduce Wrinkles without Makeup.": ["Ad 2: Stop Covering Your Wrinkles with Make Up - Do This Instead."], "3. Information on Skin Diseases": ["Ad 3: What is Advanced Skin Cancer? - Find Disease Information Here.Find Facts About Advanced Skin Cancer and a Potential Treatment Option.", "Ad 4: Learn About Advanced Melanoma - Find Disease Information Here.Find Facts About Advanced Melanoma and a Potential Treatment Option.", "Ad 5: Are You Living With Vitiligo? - For Patients & Caregivers.Discover An FDA-Approved Topical Cream That May Help With Nonsegmental Vitiligo Repigmentation. Learn About A Copay Savings Card For Eligible Patients With Vitiligo."]}
-}
------------------------------------------------
-
-<Sample INPUT> 
-Got A Rosemary Bush? Here’re 20 Brilliant & Unusual Ways To Use All That Rosemary
-
-<Sample ADS_DATA> 
-<empty>
-
-<Expected json output>
-{ 
-"reasoning" : "No ads available",
-"question": "",
-"options": []
-}
------------------------------------------------
-
-The ADS_DATA provided to you is as follows:
-"""
-
-        old_system_prompt_additional_example = """
------------------------------------------------
-<Sample INPUT>
-7 Signs and Symptoms of Magnesium Deficiency
+        pass
 
-<Sample ADS_DATA>
-Ad 1: 4 Warning Signs Of Dementia - Fight Dementia and Memory Loss. 100% Natural Program To Prevent Cognitive Decline. Developed By Dr. Will Mitchell. Read The Reviews-Get a Special Offer. Doctor Recommended. High Quality Standards. 60-Day Refund.
-Ad 2: About Hyperkalemia - Learn About The Symptoms. High Potassium Can Be A Serious Condition. Learn More About Hyperkalemia Today.
-Ad 3: Weak or Paralyzed Muscles? - A Common Symptom of Cataplexy. About 70% of People With Narcolepsy Are Believed to Have Cataplexy Symptoms. Learn More. Download the Doctor Discussion Guide to Have a Informed Conversation About Your Health.
+    def createDocs(self, content, metadata, CHUNK_SIZE = 2048, CHUNK_OVERLAP = 512):
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=CHUNK_SIZE,
+            chunk_overlap=CHUNK_OVERLAP,
+        )
+        split_docs = text_splitter.create_documents(content, metadata)
+        print(f"Documents are split into {len(split_docs)} passages")
+        return split_docs
+
+    def createDBFromDocs(self, split_docs, embeddings_model):
+        db = FAISS.from_documents(split_docs, embeddings_model)
+        return db
+
+    def createAndSaveDBInChunks(self, split_docs, embeddings_model, DB_FAISS_PATH, chunk_size = 1000):
+        one_db_docs_size = chunk_size
+        starting_i = 0
+        for i in range(starting_i, len(split_docs), one_db_docs_size):
+            ctime = time.time()
+            print(i, end = ', ')
+
+            db = FAISS.from_documents(split_docs[i:i+one_db_docs_size], embeddings_model)
+            self.saveDB(db, DB_FAISS_PATH, f"index_{int(i/one_db_docs_size)}")
+
+            ctime = time.time() - ctime
+            print("Time remaining", (ctime / one_db_docs_size * (len(split_docs) - i))/60, "minutes")
+
+    def mergeSecondDbIntoFirst(self, db1, db2):
+        db1.merge_from(db2)
+
+    def saveDB(self, db, DB_FAISS_PATH, index_name = "index"):
+        db.save_local(DB_FAISS_PATH, index_name)
+
+    def readingAndCombining(self, DB_FAISS_PATH, embeddings_hf, index_name_1, index_name_2):
+        db1 = FAISS.load_local(DB_FAISS_PATH, embeddings_hf, allow_dangerous_deserialization=True, index_name = index_name_1)
+        db2 = FAISS.load_local(DB_FAISS_PATH, embeddings_hf, allow_dangerous_deserialization=True, index_name = index_name_2)
+        db1.merge_from(db2)
+
+        # db1.index
+        # db1.docstore.search(target_id)
+        # len(db0.index_to_docstore_id)
+    
+        return db1
+    
+    def combineChunksDbs(self, DB_FAISS_PATH, embeddings_hf):
+        files = os.listdir(DB_FAISS_PATH)
+        ind = 0
+        for fl in files:
+            if fl[-6:] == '.faiss':
+                cv = fl[6:-6]
+                ind = max(ind, int(cv))
+
+        all_dbs = []
+        for i in range(0, ind+1, 2):
+            print(i)
+            db1 = FAISS.load_local(DB_FAISS_PATH, embeddings_hf, allow_dangerous_deserialization=True, index_name = f'index_{i}')
+            all_dbs.append(db1)
+            if os.path.exists(DB_FAISS_PATH + f"/index_{i+1}.faiss"):
+                db2 = FAISS.load_local(DB_FAISS_PATH, embeddings_hf, allow_dangerous_deserialization=True, index_name = f'index_{i+1}')
+                all_dbs.append(db2)
+
+        while(len(all_dbs) != 1):
+            processed_dbs = []
+            print("ITERATION ----------->")
+            for i in range(0, len(all_dbs), 2):
+                db1 = all_dbs[i]
+                print(f"For {i} before length is ", len(db1.index_to_docstore_id), end = ", ")
+                if i+1 != len(all_dbs):
+                    db2 = all_dbs[i+1]
+                    self.mergeSecondDbIntoFirst(db1, db2)
+                processed_dbs.append(db1)
+                print(f"After length is ", len(db1.index_to_docstore_id))
+            all_dbs = processed_dbs
+
+        return all_dbs[0]
 
-<Expected json output>
-{
-"reasoning" : "Given the input '7 Signs and Symptoms of Magnesium Deficiency,' it is evident that the user is looking for information specifically about magnesium deficiency. Ads 1, 2, and 3 discuss topics such as dementia, hyperkalemia, weak muscles, which are not related to magnesium deficiency in any way. Therefore, all the ads in the ADS_DATA are not suitable for the user's query and will be discarded.",
-"question": "No related ads available to form question and options.",
-"options": []
-}
-------------------------------------------------
-        """
+class ADS_RAG:
+    def __init__(self, db, qa_model_name, relation_check_best_value_thresh, bestRelationSystemPrompt, bestQuestionSystemPrompt):
+        self.client = OpenAI()
+        self.db = db
+        self.qa_model_name = qa_model_name
+        self.relation_check_best_value_thresh = relation_check_best_value_thresh
+        self.bestRelationSystemPrompt = bestRelationSystemPrompt
+        self.bestQuestionSystemPrompt = bestQuestionSystemPrompt
 
     def callOpenAiApi(self, messages):
         while True:
@@ -232,10 +223,7 @@ Ad 3: Weak or Paralyzed Muscles? - A Common Symptom of Cataplexy. About 70% of P
         bestRetreivedAdValue,
     ):
         if adsData == "":
-            return ({"reasoning": "No ads data present", "classification": 0}, 0), (
-                {"reasoning": "", "question": "", "options": []},
-                0,
-            )
+            return ({"reasoning": "No ads data present", "classification": 0}, 0), ({"reasoning": "", "question": "", "options": []}, 0)
 
         relation_answer = {"reasoning": "", "classification": 1}
         question_answer = {"reasoning": "", "question": "", "options": []}
@@ -278,10 +266,12 @@ Ad 3: Weak or Paralyzed Muscles? - A Common Symptom of Cataplexy. About 70% of P
                     }
                 ]
             )
-        return (relation_answer, tokens_used_relation), (
-            question_answer,
-            tokens_used_question,
-        )
+        return {
+            "relation_answer": relation_answer,
+            "tokens_used_relation": tokens_used_relation,
+            "question_answer": question_answer,
+            "tokens_used_question": tokens_used_question
+        }
 
     def convertDocumentsClustersToStringForApiCall(self, documents_clusters):
         key_counter = count(1)
@@ -296,6 +286,31 @@ Ad 3: Weak or Paralyzed Muscles? - A Common Symptom of Cataplexy. About 70% of P
         )
         return res
 
+    def getRagResponse(
+        self, page_information, threshold = None, RelationPrompt = None, QuestionPrompt = None
+    ):
+        curr_relation_prompt = self.bestRelationSystemPrompt
+        if RelationPrompt != None and len(RelationPrompt):
+            curr_relation_prompt = RelationPrompt
+
+        curr_question_prompt = self.bestQuestionSystemPrompt
+        if QuestionPrompt != None and len(QuestionPrompt):
+            curr_question_prompt = QuestionPrompt
+
+        documents_clusters, best_value = self.db.queryVectorDB(
+            page_information, threshold
+        )
+        answer = self.getBestQuestionOnTheBasisOfPageInformationAndAdsData(
+                page_information,
+                self.convertDocumentsClustersToStringForApiCall(documents_clusters),
+                curr_relation_prompt,
+                curr_question_prompt,
+                best_value,
+        )
+        
+
+        return answer, documents_clusters
+
     def changeDocumentsToPrintableString(self, documents_clusters):
         res = ""
         i = 0
@@ -323,113 +338,301 @@ Ad 3: Weak or Paralyzed Muscles? - A Common Symptom of Cataplexy. About 70% of P
         curr_relation_prompt,
         curr_question_prompt,
         page_information,
-        relation_answer,
-        question_answer,
+        answer
     ):
         print(
             "**************************************************************************************************\n",
             # curr_relation_prompt,
             # curr_question_prompt,
-            page_information,
-            json.dumps(relation_answer, indent=4),
-            json.dumps(question_answer, indent=4),
+            page_information + "\n",
+            json.dumps(answer,indent=4),
             "\n************************************************************************************************\n\n",
         )
 
-    def getRagResponse(
-        self, RelationPrompt, QuestionPrompt, threshold, page_information
-    ):
-        curr_relation_prompt = self.bestRelationSystemPrompt
-        if RelationPrompt != None or len(RelationPrompt):
-            curr_relation_prompt = RelationPrompt
-
-        curr_question_prompt = self.bestQuestionSystemPrompt
-        if QuestionPrompt != None or len(QuestionPrompt):
-            curr_question_prompt = QuestionPrompt
-
-        documents_clusters, best_value = self.db.queryVectorDB(
-            page_information, threshold
-        )
-        relation_answer, question_answer = (
-            self.getBestQuestionOnTheBasisOfPageInformationAndAdsData(
-                page_information,
-                self.convertDocumentsClustersToStringForApiCall(documents_clusters),
-                curr_relation_prompt,
-                curr_question_prompt,
-                best_value,
-            )
-        )
+    def getRagGradioResponse(self, page_information, RelationPrompt, QuestionPrompt, threshold):
+        answer, documents_clusters = self.getRagResponse(page_information, threshold, RelationPrompt, QuestionPrompt)
         self.logResult(
-            curr_relation_prompt,
-            curr_relation_prompt,
+            RelationPrompt,
+            QuestionPrompt,
             page_information,
-            relation_answer,
-            question_answer,
+            answer
         )
 
         docs_info = self.changeDocumentsToPrintableString(documents_clusters)
         relation_answer_string = self.changeResponseToPrintableString(
-            relation_answer[0], "relation"
+            answer["relation_answer"], "relation"
         )
         question_answer_string = self.changeResponseToPrintableString(
-            question_answer[0], "question"
+            answer["question_answer"], "question"
         )
-        full_response = f"**ANSWER**: \n Relation answer:\n {relation_answer_string}\n Question answer:\n {question_answer_string}\n\n**RETREIVED DOCUMENTS CLUSTERS**:\n{docs_info}\n\n**TOKENS USED**:\nQuestion api call: {question_answer[1]}\nRelation api call: {relation_answer[1]}"
+        question_tokens = answer["tokens_used_question"]
+        relation_tokens = answer["tokens_used_relation"]
+        full_response = f"**ANSWER**: \n Relation answer:\n {relation_answer_string}\n Question answer:\n {question_answer_string}\n\n**RETREIVED DOCUMENTS CLUSTERS**:\n{docs_info}\n\n**TOKENS USED**:\nQuestion api call: {question_tokens}\nRelation api call: {relation_tokens}"
         return full_response
 
+class VARIABLE_MANAGER:
+    def __init__(self):
+        load_dotenv(override=True)
+        os.environ["TOKENIZERS_PARALLELISM"] = "false"
+        
+        self.embedding_model_hf = "BAAI/bge-m3"
+        # embedding_model_hf = "sentence-transformers/all-mpnet-base-v2"
+        self.DB_FAISS_PATH = "./vectorstore/db_faiss_ads_20May_20Jun_webmd_healthline_Health_dupRemoved0.8"
 
-data = pd.read_csv(data_file_path, sep="\t")
-# data.dropna(axis=0, how="any", inplace=True)
-data.drop_duplicates(subset=["ad_title", "ad_desc"], inplace=True)
-ad_title_content = list(data["ad_title"].values)
-rag = ADS_RAG()
-with gr.Blocks() as demo:
-    gr.Markdown("# RAG on ads data")
-    with gr.Row():
-        RelationPrompt = gr.Textbox(
-            rag.bestRelationSystemPrompt,
-            lines=1,
-            placeholder="Enter the relation system prompt for relation check",
-            label="Relation System prompt",
-        )
-        QuestionPrompt = gr.Textbox(
-            rag.bestQuestionSystemPrompt,
-            lines=1,
-            placeholder="Enter the question system prompt for question formulation",
-            label="Question System prompt",
+    def getRag(self):
+        # embeddings_oa = OpenAIEmbeddings(model=embedding_model_oa)
+        # embeddings_hf = HuggingFaceEmbeddings(model_name = embedding_model_hf, show_progress = True)
+        embeddings_hf = HuggingFaceEmbeddings(model_name=self.embedding_model_hf)
+        vector_db = VECTOR_DB(0.75, 50, "kmeans-cc", 3, 6, self.DB_FAISS_PATH, embeddings_hf)
+        rag = ADS_RAG(vector_db, "gpt-3.5-turbo", 0.6, self.getRelationSystemPrompt(), self.getQuestionSystemPrompt())
+        return rag
+    
+    def QnAAdsSampleGenerationPreProcessing(self):
+        data_file_path = "./data/144_webmd_healthline_12Jun-18Jun_top1000each_urlsContent.tsv"
+        data = pd.read_csv(data_file_path, sep='\t')
+        data.dropna(axis=0, how='any', inplace=True)
+        # data.drop_duplicates(subset = ['ad_title', 'ad_desc'], inplace=True)
+        # ad_title_content = list(data["ad_title"].values)
+        def get_core_content(row):
+            url_content = row['url_content']
+            url_title = row['url_title']
+            return 'Page Title -: ' +  url_title + '\nPage Content -: ' + '. '.join(url_content.split('. ')[:7])
+
+        data['core_content'] = data.apply(get_core_content, axis = 1)
+        # for i in range(len(data)):
+        #     print(data.loc[i, 'url'])
+        #     print(data.loc[i, 'url_content'])
+        #     print(data.loc[i, 'core_content'])
+        #     print()
+        #     if(i > 10):
+        #         break
+        return data
+    
+    def GradioRagPreProcessing(self):
+        data_file_path = "./data/142_adclick_20May_20Jun_webmd_healthline_Health_dupRemoved0.8_someAdsCampaign.tsv"
+        data = pd.read_csv(data_file_path, sep="\t")
+        # data.dropna(axis=0, how="any", inplace=True)
+        data.drop_duplicates(subset=["ad_title", "ad_desc"], inplace=True)
+        ad_title_content = list(data["ad_title"].values)
+        return ad_title_content
+
+    def getQuestionSystemPrompt(self):
+        bestQuestionSystemPrompt = """1. You are an advertising concierge for text ads on websites. Given an INPUT and the available ad inventory (ADS_DATA), your task is to form a relevant QUESTION to ask the user visiting the webpage. This question should help identify the user's intent behind visiting the webpage and should be highly attractive.
+2. Now form a highly attractive/lucrative and diverse/mutually exclusive OPTION which should be both the answer for the QUESTION and related to ads in this cluster.
+3. Try to generate intelligent creatives for advertising and keep QUESTION within 70 characters and either 2, 3 or 4 options with each OPTION within 4 to 6 words.
+4. Provide your REASONING behind choosing the QUESTION and the OPTIONS. Now provide the QUESTION and the OPTIONS. Along with each OPTION, provide the ads from ADS_DATA that you associated with it.
+
+---------------------------------------
+
+<Sample INPUT>
+The Effects of Aging on Skin
+
+<Sample ADS_DATA>
+{"Cluster 1 Ads": {"Ad 1": "Forget Retinol, Use This Household Item To Fill In Wrinkles - Celebrities Are Ditching Pricey Facelifts For This."}, "Cluster 2 Ads": {"Ad 2": "Stop Covering Your Wrinkles with Make Up - Do This Instead.", "Ad 3": "Living With Migraines? - Discover A Treatment Option. Learn about a type of prescription migraine treatment called CGRP receptor antagonists. Discover a range of resources that may help people dealing with migraines"}, "Cluster 3 Ads": {"Ad 4": "What is Advanced Skin Cancer? - Find Disease Information Here.Find Facts About Advanced Skin Cancer and a Potential Treatment Option.", "Ad 5": "Learn About Advanced Melanoma - Find Disease Information Here.Find Facts About Advanced Melanoma and a Potential Treatment Option.", "Ad 6": "Treatment For CKD - Reduce Risk Of Progressing CKD. Ask About A Treatment That Can Help Reduce Your Risk Of Kidney Failure", "Ad 7": "Are You Living With Vitiligo? - For Patients & Caregivers.Discover An FDA-Approved Topical Cream That May Help With Nonsegmental Vitiligo Repigmentation. Learn About A Copay Savings Card For Eligible Patients With Vitiligo."}]
+
+<Expected json output>
+{
+"reasoning" : "Among the seven ads in **Sample ADS_DATA**, Ads 3 and 6 are irrelevant to the INPUT, so they should be discarded. Ad 1, 2 closely aligns with the user's intent. Ads 4, 5, and 7 are also relevant to INPUT. The question will be formed in a way to connect the PAGE content with the goals of these five relevant ads, making sure they appeal to both specific and general user interests, with the OPTIONS being the answer for QUESTION(it is ensured that no irrelevant options are formed)",
+"question": "Interested in methods to combat aging skin?",
+"options": {"1. Retinol Alternatives for Wrinkle Treatment." : ["Ad 1: Forget Retinol, Use This Household Item To Fill In Wrinkles - Celebrities Are Ditching Pricey Facelifts For This."], "2. Reduce Wrinkles without Makeup.": ["Ad 2: Stop Covering Your Wrinkles with Make Up - Do This Instead."], "3. Information on Skin Diseases": ["Ad 3: What is Advanced Skin Cancer? - Find Disease Information Here.Find Facts About Advanced Skin Cancer and a Potential Treatment Option.", "Ad 4: Learn About Advanced Melanoma - Find Disease Information Here.Find Facts About Advanced Melanoma and a Potential Treatment Option.", "Ad 5: Are You Living With Vitiligo? - For Patients & Caregivers.Discover An FDA-Approved Topical Cream That May Help With Nonsegmental Vitiligo Repigmentation. Learn About A Copay Savings Card For Eligible Patients With Vitiligo."]}
+}
+-----------------------------------------------
+
+<Sample INPUT>
+Got A Rosemary Bush? Here’re 20 Brilliant & Unusual Ways To Use All That Rosemary
+
+<Sample ADS_DATA>
+<empty>
+
+<Expected json output>
+{
+"reasoning" : "No ads available",
+"question": "",
+"options": []
+}
+-----------------------------------------------
+
+The ADS_DATA provided to you is as follows:
+"""
+#         old_system_prompt_additional_example = """
+# -----------------------------------------------
+# <Sample INPUT>
+# 7 Signs and Symptoms of Magnesium Deficiency
+
+# <Sample ADS_DATA>
+# Ad 1: 4 Warning Signs Of Dementia - Fight Dementia and Memory Loss. 100% Natural Program To Prevent Cognitive Decline. Developed By Dr. Will Mitchell. Read The Reviews-Get a Special Offer. Doctor Recommended. High Quality Standards. 60-Day Refund.
+# Ad 2: About Hyperkalemia - Learn About The Symptoms. High Potassium Can Be A Serious Condition. Learn More About Hyperkalemia Today.
+# Ad 3: Weak or Paralyzed Muscles? - A Common Symptom of Cataplexy. About 70% of People With Narcolepsy Are Believed to Have Cataplexy Symptoms. Learn More. Download the Doctor Discussion Guide to Have a Informed Conversation About Your Health.
+
+# <Expected json output>
+# {
+# "reasoning" : "Given the input '7 Signs and Symptoms of Magnesium Deficiency,' it is evident that the user is looking for information specifically about magnesium deficiency. Ads 1, 2, and 3 discuss topics such as dementia, hyperkalemia, weak muscles, which are not related to magnesium deficiency in any way. Therefore, all the ads in the ADS_DATA are not suitable for the user's query and will be discarded.",
+# "question": "No related ads available to form question and options.",
+# "options": []
+# }
+# ------------------------------------------------
+# """
+        return bestQuestionSystemPrompt
+    
+    def getRelationSystemPrompt(self):
+        bestRelationSystemPrompt = """You are an advertising concierge for text ads on websites. Given an INPUT and the available ad inventory (ADS_DATA), your task is to determine whether there are some relevant ADS to INPUT are present in ADS_DATA. ADS WHICH DON'T MATCH USER'S INTENT SHOULD BE CONSIDERED IRRELEVANT
+
+---------------------------------------
+
+**Sample INPUT***: What Causes Bright-Yellow Urine and Other Changes in Color?
+
+Expected json output :
+{
+"reasoning" : "Given the user's search for 'What Causes Bright-Yellow Urine and Other Changes in Color?', it is evident that they are looking for information related to the causes and implications of changes in urine color. Therefore, ads that are related to urine color. However, none of the provided ads are related to urine. Ads 1, 2, and 3 are focused on chronic lymphocytic leukemia (CLL) treatment, Ad 4 addresses high blood pressure, and Ad 5 is about migraine treatment. Since none of these ads are not at all relevant to the urine, they can all be considered irrelevant to the user's intent.",
+"classification": 0
+}
+------------------------------------------------
+
+**Sample INPUT**: The Effects of Aging on Skin
+
+Expected json output :
+{
+"reasoning" : "Given the user's search for 'The Effects of Aging on Skin,' it is clear that they are seeking information related to skin aging. Therefore, the ads that are relevant to skin effects should be considered. Ads 1 and 2 focus on wrinkle treatment and anti-aging solutions, making them pertinent to the user's intent. Ad 3 targets vitiligo and not general skin aging but it is related to skin effect. So it is also relevant. Ads 4 and 5 are about advanced lung cancer, which do not address the interest in skin. Ads 1 and 2, 3 are most relevant to the user's search. So ADS_DATA is relevant to INPUT. ",
+"classification": 1
+}
+---------------------------------------
+
+The ADS_DATA provided to you is as follows:
+"""
+        return bestRelationSystemPrompt
+
+
+# *********************** DB GENERATION ******************************
+# df = pd.read_csv(data_file_path, sep="\t")
+
+# --------------------------------
+# WEB DATA PROCESSING
+# from urllib.parse import urlparse
+# import re
+# def get_cleaned_url(url):
+#     path = urlparse(url).path.strip()
+#     cleaned_path = re.sub(r'[^a-zA-Z0-9\s-]', ' ', path).replace('/', '')
+#     cleaned_path = re.sub(r'[^a-zA-Z0-9\s]', ' ', path).replace('-', '')
+#     return cleaned_path.strip()
+
+# df['cleaned_url'] = df['url'].map(get_cleaned_url)
+# df.dropna(subset=['cleaned_url', 'url_content', 'url_title'], inplace=True)
+# df['combined'] = df['cleaned_url'] + ". " + df['url_title'] + ". " + df['url_content']
+# content = df["combined"].tolist()
+# metadata = [
+#         {"title": row["url_title"], "url": row["url"]}
+#         for _, row in df.iterrows()
+#     ]
+# ------------------------------
+# ADS DATA PROCESSING
+# # df.dropna(axis=0, how='any', inplace=True)
+# df.drop_duplicates(subset = ['ad_title', 'ad_desc'], inplace=True)
+# dfRPC = df[df['RPC'] > 0]
+# dfRPC.dropna(how = 'any', inplace=True)
+# dfCampaign = df[df['type'] == 'campaign']
+# dfCampaign.fillna('', inplace=True)
+# df = pd.concat([dfRPC, dfCampaign])
+# df
+
+# content = (df["ad_title"] + ". " + df["ad_desc"]).tolist()
+# metadata = [
+#         {"publisher_url": row["publisher_url"], "keyword_term": row["keyword_term"], "ad_display_url": row["ad_display_url"], "revenue": row["revenue"], "ad_click_count": row["ad_click_count"], "RPC": row["RPC"], "Type": row["type"]}
+#         # {"revenue": row["revenue"], "ad_click_count": row["ad_click_count"]}
+#         for _, row in df.iterrows()
+#     ]
+# --------------------------------
+
+# faiss_db = FAISS_DB()
+# db = faiss_db.createDBFromDocs(content, metadata)
+# faiss_db.saveDB(db, '.')
+
+# ************************************************************************
+# PARALLELY CREATING DB - BACKUP FOR FUTURE USE
+# import time
+# import threading
+# import os
+# one_db_docs_size = 1000
+# starting_i = 0
+# parallel_processes = 3
+# def split_list(lst, n):
+#     k, m = divmod(len(lst), n)
+#     return (lst[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n))
+# def createDBForIndexes(inds):
+#     for i in inds:
+#         ctime = time.time()
+#         print(f"Processing {i}")
+#         if not os.path.exists(DB_FAISS_PATH + "/index_{int(i/one_db_docs_size)}.faiss"):
+#             db = FAISS.from_documents(split_docs[i:i+one_db_docs_size], embeddings_hf)
+#             db.save_local(DB_FAISS_PATH, index_name = f"index_{int(i/one_db_docs_size)}")
+#         ctime = time.time() - ctime
+#         print(f"{i})Time taken", ctime)
+# indexes = split_list(range(starting_i, len(split_docs), one_db_docs_size), parallel_processes)
+# threads = []
+# for i, one_process_indexes in enumerate(indexes):
+#     thread = threading.Thread(target=createDBForIndexes, args=(one_process_indexes,))
+#     thread.start()
+#     threads.append(thread)
+# for thread in threads:
+#     thread.join()
+# print("All threads completed.")
+# ************************************************************************
+
+if __name__ == "__main__":
+    import pandas as pd
+    import gradio as gr
+    import random
+
+    vm = VARIABLE_MANAGER()
+    rag = vm.getRag()
+    ad_title_content = vm.GradioRagPreProcessing()
+
+    with gr.Blocks() as demo:
+        gr.Markdown("# RAG on ads data")
+        with gr.Row():
+            RelationPrompt = gr.Textbox(
+                vm.getRelationSystemPrompt(),
+                lines=1,
+                placeholder="Enter the relation system prompt for relation check",
+                label="Relation System prompt",
+            )
+            QuestionPrompt = gr.Textbox(
+                vm.getQuestionSystemPrompt(),
+                lines=1,
+                placeholder="Enter the question system prompt for question formulation",
+                label="Question System prompt",
+            )
+            page_information = gr.Textbox(
+                lines=1, placeholder="Enter the page information", label="Page Information"
+            )
+            threshold = gr.Number(
+                value=rag.db.default_threshold, label="Threshold", interactive=True
+            )
+        output = gr.Textbox(label="Output")
+        submit_btn = gr.Button("Submit")
+
+        submit_btn.click(
+            rag.getRagGradioResponse,
+            inputs=[page_information, RelationPrompt, QuestionPrompt, threshold],
+            outputs=[output],
         )
-        page_information = gr.Textbox(
-            lines=1, placeholder="Enter the page information", label="Page Information"
+        page_information.submit(
+            rag.getRagGradioResponse,
+            inputs=[page_information, RelationPrompt, QuestionPrompt, threshold],
+            outputs=[output],
         )
-        threshold = gr.Number(
-            value=rag.db.default_threshold, label="Threshold", interactive=True
+        with gr.Accordion("Ad Titles", open=False):
+            ad_titles = gr.Markdown()
+
+        demo.load(
+            lambda: "<br>".join(
+                random.sample(
+                    [str(ad_title) for ad_title in ad_title_content],
+                    min(100, len(ad_title_content)),
+                )
+            ),
+            None,
+            ad_titles,
         )
-    output = gr.Textbox(label="Output")
-    submit_btn = gr.Button("Submit")
-
-    submit_btn.click(
-        rag.getRagResponse,
-        inputs=[RelationPrompt, QuestionPrompt, threshold, page_information],
-        outputs=[output],
-    )
-    page_information.submit(
-        rag.getRagResponse,
-        inputs=[RelationPrompt, QuestionPrompt, threshold, page_information],
-        outputs=[output],
-    )
-    with gr.Accordion("Ad Titles", open=False):
-        ad_titles = gr.Markdown()
-
-    demo.load(
-        lambda: "<br>".join(
-            random.sample(
-                [str(ad_title) for ad_title in ad_title_content],
-                min(100, len(ad_title_content)),
-            )
-        ),
-        None,
-        ad_titles,
-    )
 
-gr.close_all()
-demo.launch()
+    gr.close_all()
+    demo.launch()