Added all changes - responses are better then before

rag.py

@@ -22,7 +22,9 @@ embeddings_hf = HuggingFaceEmbeddings(model_name=embedding_model_hf)
 
 class CLUSTERING:
     def __init__(self):
-        self.clustering_algo = 'kmeans-cc' # ['kmeans-cc', 'kmeans-sp', 'spectral_clustering']
+        self.clustering_algo = (
+            "kmeans-cc"  # ['kmeans-cc', 'kmeans-sp', 'spectral_clustering']
+        )
 
     def cluster_embeddings(self, embeddings, no_of_clusters, no_of_points):
         if self.clustering_algo in {"kmeans-cc", "kmeans-sp"}:
@@ -45,10 +47,14 @@ class CLUSTERING:
                 for i, label in enumerate(labels):
                     if len(clusters_indices[label]) < no_of_points:
                         clusters_indices[label].append(i)
-                    if all(len(cluster) == no_of_points for cluster in clusters_indices):
+                    if all(
+                        len(cluster) == no_of_points for cluster in clusters_indices
+                    ):
                         break
         elif self.clustering_algo == "spectral":
-            spectral_clustering = SpectralClustering(n_clusters=no_of_clusters, affinity='nearest_neighbors', random_state=42)
+            spectral_clustering = SpectralClustering(
+                n_clusters=no_of_clusters, affinity="nearest_neighbors", random_state=42
+            )
             labels = spectral_clustering.fit_predict(embeddings)
 
             clusters_indices = [[] for _ in range(no_of_clusters)]
@@ -62,6 +68,7 @@ class CLUSTERING:
             for i in range(no_of_clusters)
         ]
 
+
 class VECTOR_DB:
     def __init__(self):
         self.DB_FAISS_PATH = "./vectorstore/db_faiss_ads_20May_20Jun_webmd_healthline_Health_dupRemoved0.8"
@@ -89,20 +96,25 @@ class VECTOR_DB:
             retreived_documents[i][0].page_content = remove_html_tags(
                 retreived_documents[i][0].page_content
             )
-        embeddings = np.array(embeddings_hf.embed_documents([doc[0].page_content for doc in retreived_documents]))
+        if len(retreived_documents):
+            embeddings = np.array(
+                embeddings_hf.embed_documents(
+                    [doc[0].page_content for doc in retreived_documents]
+                )
+            )
 
-        clustered_indices = CLUSTERING().cluster_embeddings(
-            embeddings, self.no_of_clusters, self.no_of_ads_in_each_cluster
-        )
-        documents_clusters = [
-            [retreived_documents[ind] for ind in cluster_indices]
-            for cluster_indices in clustered_indices
-        ]
+            clustered_indices = CLUSTERING().cluster_embeddings(
+                embeddings, self.no_of_clusters, self.no_of_ads_in_each_cluster
+            )
+            documents_clusters = [
+                [retreived_documents[ind] for ind in cluster_indices]
+                for cluster_indices in clustered_indices
+            ]
 
-        best_value = 1
-        if len(retreived_documents):
             best_value = retreived_documents[0][1]
-        return documents_clusters, best_value
+            return documents_clusters, best_value
+        return [], 1
+
 
 class ADS_RAG:
     def __init__(self):
@@ -110,11 +122,11 @@ class ADS_RAG:
         self.db = VECTOR_DB()
         self.qa_model_name = "gpt-3.5-turbo"
         self.relation_check_best_value_thresh = 0.6
-        self.bestRelationSystemPrompt = """You are an advertising concierge for text ads on websites. Given an INPUT(PAGE_TITLE) and the available ad inventory (ADS_DATA), your task is to determine whether there are some relevant ADS to INPUT are present in ADS_DATA. ADS WHICH DON'T MATCH USER'S INTENT SHOULD BE CONSIDERED IRRELEVANT
+        self.bestRelationSystemPrompt = """You are an advertising concierge for text ads on websites. Given an INPUT and the available ad inventory (ADS_DATA), your task is to determine whether there are some relevant ADS to INPUT are present in ADS_DATA. ADS WHICH DON'T MATCH USER'S INTENT SHOULD BE CONSIDERED IRRELEVANT
 
 ---------------------------------------
 
-**Sample INPUT(PAGE_TITLE)***: What Causes Bright-Yellow Urine and Other Changes in Color?
+**Sample INPUT***: What Causes Bright-Yellow Urine and Other Changes in Color?
 
 Expected json output :
 { 
@@ -123,42 +135,40 @@ Expected json output :
 }
 ------------------------------------------------
 
-**Sample INPUT(PAGE_TITLE)**: The Effects of Aging on Skin
+**Sample INPUT**: The Effects of Aging on Skin
 
 Expected json output :
 { 
-"reasoning" : "Given the user's search for 'The Effects of Aging on Skin,' it is clear that they are seeking information related to skin aging. Therefore, the ads that are relevant to skin effects should be considered. Ads 1 and 2 focus on wrinkle treatment and anti-aging solutions, making them pertinent to the user's intent. Ad 3 targets vitiligo and not general skin aging but it is related to skin effect. So it is also relevant. Ads 4 and 5 are about advanced lung cancer, which do not address the interest in skin. Ads 1 and 2, 3 are most relevant to the user's search. So ADS_DATA is relevant to INPUT TITLE. ",
+"reasoning" : "Given the user's search for 'The Effects of Aging on Skin,' it is clear that they are seeking information related to skin aging. Therefore, the ads that are relevant to skin effects should be considered. Ads 1 and 2 focus on wrinkle treatment and anti-aging solutions, making them pertinent to the user's intent. Ad 3 targets vitiligo and not general skin aging but it is related to skin effect. So it is also relevant. Ads 4 and 5 are about advanced lung cancer, which do not address the interest in skin. Ads 1 and 2, 3 are most relevant to the user's search. So ADS_DATA is relevant to INPUT. ",
 "classification": 1
 }
 ---------------------------------------
 
 The ADS_DATA provided to you is as follows:
+"""
 
-        """
-
-        self.bestQuestionSystemPrompt = """1. You are an advertising concierge for text ads on websites. Given an INPUT(PAGE_TITLE)  and the available ad inventory (ADS_DATA), your task is to form a relevant QUESTION to ask the user visiting the webpage. This question should help identify the user's intent behind visiting the webpage. 
-2. From the ADS_DATA clusters, discard all ads that are not related to the INPUT or do not match the user's intent behind visiting the page. Also, remove any ads that are distantly related to the user's intent. 
-3. FROM REMAINING ADS  in each ads cluster form an OPTION which should be both the answer for the QUESTION and related to ads in this cluster. 
-4. Try to generate intelligent creatives for advertising and keep QUESTION within 70 characters and each OPTION with either 4, 5, or 6 words.
-5. Provide your REASONING behind choosing the QUESTION and the OPTIONS. Now provide the QUESTION and the OPTIONS. Along with each OPTION, provide the ads from ADS_DATA that you associated with it.
+        self.bestQuestionSystemPrompt = """1. You are an advertising concierge for text ads on websites. Given an INPUT and the available ad inventory (ADS_DATA), your task is to form a relevant QUESTION to ask the user visiting the webpage. This question should help identify the user's intent behind visiting the webpage and should be highly attractive. 
+2. Now form a highly attractive/lucrative and diverse/mutually exclusive OPTION which should be both the answer for the QUESTION and related to ads in this cluster.  
+3. Try to generate intelligent creatives for advertising and keep QUESTION within 70 characters and either 2, 3 or 4 options with each OPTION within 4 to 6 words.
+4. Provide your REASONING behind choosing the QUESTION and the OPTIONS. Now provide the QUESTION and the OPTIONS. Along with each OPTION, provide the ads from ADS_DATA that you associated with it.
 
 ---------------------------------------
 
-<Sample INPUT(PAGE_TITLE)> 
+<Sample INPUT> 
 The Effects of Aging on Skin
 
 <Sample ADS_DATA> 
-[{"Ad 1": "Forget Retinol, Use This Household Item To Fill In Wrinkles - Celebrities Are Ditching Pricey Facelifts For This.", "Ad 2": "Stop Covering Your Wrinkles with Make Up - Do This Instead."}, {"Ad 3": "Living With Migraines? - Discover A Treatment Option. Learn about a type of prescription migraine treatment called CGRP receptor antagonists. Discover a range of resources that may help people dealing with migraines."}, {"Ad 4": "What is Advanced Skin Cancer? - Find Disease Information Here.Find Facts About Advanced Skin Cancer and a Potential Treatment Option.", "Ad 5": "Learn About Advanced Melanoma - Find Disease Information Here.Find Facts About Advanced Melanoma and a Potential Treatment Option.", "Ad 6": "Treatment For CKD - Reduce Risk Of Progressing CKD. Ask About A Treatment That Can Help Reduce Your Risk Of Kidney Failure.", "Ad 7": "Are You Living With Vitiligo? - For Patients & Caregivers.Discover An FDA-Approved Topical Cream That May Help With Nonsegmental Vitiligo Repigmentation. Learn About A Copay Savings Card For Eligible Patients With Vitiligo."}]
+{"Cluster 1 Ads": {"Ad 1": "Forget Retinol, Use This Household Item To Fill In Wrinkles - Celebrities Are Ditching Pricey Facelifts For This."}, "Cluster 2 Ads": {"Ad 2": "Stop Covering Your Wrinkles with Make Up - Do This Instead.", "Ad 3": "Living With Migraines? - Discover A Treatment Option. Learn about a type of prescription migraine treatment called CGRP receptor antagonists. Discover a range of resources that may help people dealing with migraines"}, "Cluster 3 Ads": {"Ad 4": "What is Advanced Skin Cancer? - Find Disease Information Here.Find Facts About Advanced Skin Cancer and a Potential Treatment Option.", "Ad 5": "Learn About Advanced Melanoma - Find Disease Information Here.Find Facts About Advanced Melanoma and a Potential Treatment Option.", "Ad 6": "Treatment For CKD - Reduce Risk Of Progressing CKD. Ask About A Treatment That Can Help Reduce Your Risk Of Kidney Failure", "Ad 7": "Are You Living With Vitiligo? - For Patients & Caregivers.Discover An FDA-Approved Topical Cream That May Help With Nonsegmental Vitiligo Repigmentation. Learn About A Copay Savings Card For Eligible Patients With Vitiligo."}]
 
 <Expected json output>
 { 
-"reasoning" : "Among the seven ads in **Sample ADS_DATA**, Ads 3 and 6 are irrelevant to the INPUT, so they should be discarded. Ad 1, 2, 4, 5, and 7 are relevant to INPUT. The question will be formed in a way to connect the PAGE TITLE content with the goals of these five relevant ads, making sure they appeal to both specific and general user interests.",
-"question": "Which of the following methods to combat aging skin are you most interested in?",
-"options": {"1. Reduce Wrinkles without Makeup.": ["Ad 1: Stop Covering Your Wrinkles with Make Up - Do This Instead."], "2. Retinol Alternatives for Wrinkle Treatment." : ["Ad 2: Forget Retinol, Use This Household Item To Fill In Wrinkles - Celebrities Are Ditching Pricey Facelifts For This."], "3. Information on Skin Diseases": ["Ad 4: What is Advanced Skin Cancer? - Find Disease Information Here.Find Facts About Advanced Skin Cancer and a Potential Treatment Option.", "Ad 5: Learn About Advanced Melanoma - Find Disease Information Here.Find Facts About Advanced Melanoma and a Potential Treatment Option.", "Ad 7: Are You Living With Vitiligo? - For Patients & Caregivers.Discover An FDA-Approved Topical Cream That May Help With Nonsegmental Vitiligo Repigmentation. Learn About A Copay Savings Card For Eligible Patients With Vitiligo."]}
+"reasoning" : "Among the seven ads in **Sample ADS_DATA**, Ads 3 and 6 are irrelevant to the INPUT, so they should be discarded. Ad 1, 2 closely aligns with the user's intent. Ads 4, 5, and 7 are also relevant to INPUT. The question will be formed in a way to connect the PAGE content with the goals of these five relevant ads, making sure they appeal to both specific and general user interests, with the OPTIONS being the answer for QUESTION(it is ensured that no irrelevant options are formed)",
+"question": "Interested in methods to combat aging skin?",
+"options": {"1. Retinol Alternatives for Wrinkle Treatment." : ["Ad 1: Forget Retinol, Use This Household Item To Fill In Wrinkles - Celebrities Are Ditching Pricey Facelifts For This."], "2. Reduce Wrinkles without Makeup.": ["Ad 2: Stop Covering Your Wrinkles with Make Up - Do This Instead."], "3. Information on Skin Diseases": ["Ad 3: What is Advanced Skin Cancer? - Find Disease Information Here.Find Facts About Advanced Skin Cancer and a Potential Treatment Option.", "Ad 4: Learn About Advanced Melanoma - Find Disease Information Here.Find Facts About Advanced Melanoma and a Potential Treatment Option.", "Ad 5: Are You Living With Vitiligo? - For Patients & Caregivers.Discover An FDA-Approved Topical Cream That May Help With Nonsegmental Vitiligo Repigmentation. Learn About A Copay Savings Card For Eligible Patients With Vitiligo."]}
 }
 -----------------------------------------------
 
-<Sample INPUT(PAGE_TITLE)> 
+<Sample INPUT> 
 Got A Rosemary Bush? Here’re 20 Brilliant & Unusual Ways To Use All That Rosemary
 
 <Sample ADS_DATA> 
@@ -173,25 +183,25 @@ Got A Rosemary Bush? Here’re 20 Brilliant & Unusual Ways To Use All That Rosem
 -----------------------------------------------
 
 The ADS_DATA provided to you is as follows:
-        """
+"""
 
         old_system_prompt_additional_example = """
-        -----------------------------------------------
-        <Sample INPUT(PAGE_TITLE)>
-        7 Signs and Symptoms of Magnesium Deficiency
-
-        <Sample ADS_DATA>
-        Ad 1: 4 Warning Signs Of Dementia - Fight Dementia and Memory Loss. 100% Natural Program To Prevent Cognitive Decline. Developed By Dr. Will Mitchell. Read The Reviews-Get a Special Offer. Doctor Recommended. High Quality Standards. 60-Day Refund.
-        Ad 2: About Hyperkalemia - Learn About The Symptoms. High Potassium Can Be A Serious Condition. Learn More About Hyperkalemia Today.
-        Ad 3: Weak or Paralyzed Muscles? - A Common Symptom of Cataplexy. About 70% of People With Narcolepsy Are Believed to Have Cataplexy Symptoms. Learn More. Download the Doctor Discussion Guide to Have a Informed Conversation About Your Health.
-
-        <Expected json output>
-        {
-        "reasoning" : "Given the input '7 Signs and Symptoms of Magnesium Deficiency,' it is evident that the user is looking for information specifically about magnesium deficiency. Ads 1, 2, and 3 discuss topics such as dementia, hyperkalemia, weak muscles, which are not related to magnesium deficiency in any way. Therefore, all the ads in the ADS_DATA are not suitable for the user's query and will be discarded.",
-        "question": "No related ads available to form question and options.",
-        "options": []
-        }
-        ------------------------------------------------
+-----------------------------------------------
+<Sample INPUT>
+7 Signs and Symptoms of Magnesium Deficiency
+
+<Sample ADS_DATA>
+Ad 1: 4 Warning Signs Of Dementia - Fight Dementia and Memory Loss. 100% Natural Program To Prevent Cognitive Decline. Developed By Dr. Will Mitchell. Read The Reviews-Get a Special Offer. Doctor Recommended. High Quality Standards. 60-Day Refund.
+Ad 2: About Hyperkalemia - Learn About The Symptoms. High Potassium Can Be A Serious Condition. Learn More About Hyperkalemia Today.
+Ad 3: Weak or Paralyzed Muscles? - A Common Symptom of Cataplexy. About 70% of People With Narcolepsy Are Believed to Have Cataplexy Symptoms. Learn More. Download the Doctor Discussion Guide to Have a Informed Conversation About Your Health.
+
+<Expected json output>
+{
+"reasoning" : "Given the input '7 Signs and Symptoms of Magnesium Deficiency,' it is evident that the user is looking for information specifically about magnesium deficiency. Ads 1, 2, and 3 discuss topics such as dementia, hyperkalemia, weak muscles, which are not related to magnesium deficiency in any way. Therefore, all the ads in the ADS_DATA are not suitable for the user's query and will be discarded.",
+"question": "No related ads available to form question and options.",
+"options": []
+}
+------------------------------------------------
         """
 
     def callOpenAiApi(self, messages):
@@ -202,14 +212,15 @@ The ADS_DATA provided to you is as follows:
                     messages=messages,
                     temperature=0,
                     seed=42,
-                    max_tokens=1000,
+                    max_tokens=1200,
                     response_format={"type": "json_object"},
                 )
                 tokens_used = response.usage.total_tokens
                 answer = json.loads(response.choices[0].message.content)
                 return answer, tokens_used
             except Exception as e:
-                print("Error-: ", e.message)
+                print(response.choices[0].message.content)
+                print("Error-: ", e)
                 print("Trying Again")
 
     def getBestQuestionOnTheBasisOfPageInformationAndAdsData(
@@ -253,7 +264,8 @@ The ADS_DATA provided to you is as follows:
             )
 
         if relation_answer["classification"] != 0:
-            question_answer, tokens_used_question = self.callOpenAiApi([
+            question_answer, tokens_used_question = self.callOpenAiApi(
+                [
                     {
                         "role": "system",
                         "content": questionSystemPrompt + adsData,
@@ -264,7 +276,8 @@ The ADS_DATA provided to you is as follows:
                         "role": "user",
                         "content": page_information + "\nThe JSON response: ",
                     }
-                ])
+                ]
+            )
         return (relation_answer, tokens_used_relation), (
             question_answer,
             tokens_used_question,
@@ -272,17 +285,22 @@ The ADS_DATA provided to you is as follows:
 
     def convertDocumentsClustersToStringForApiCall(self, documents_clusters):
         key_counter = count(1)
-        res = json.dumps([
-            {f"Ad {next(key_counter)}": document[0].page_content for j, document in enumerate(documents_cluster)}
-            for i, documents_cluster in enumerate(documents_clusters)
-        ], indent=4)
+        res = json.dumps(
+            {
+                f"Option {i+1} Ads": {
+                    f"Ad {next(key_counter)}": document[0].page_content
+                    for j, document in enumerate(documents_cluster)
+                }
+                for i, documents_cluster in enumerate(documents_clusters)
+            }
+        )
         return res
 
     def changeDocumentsToPrintableString(self, documents_clusters):
         res = ""
         i = 0
         for ind, documents_cluster in enumerate(documents_clusters):
-            res += f"Cluster {ind+1}-:\n"
+            res += f"Option {ind+1} Ads-:\n"
             for document in documents_cluster:
                 i += 1
                 res += f"[Ad {i}] Content: {document[0].page_content}\nRevenue: {document[0].metadata['revenue']}\nAd Click Count: {document[0].metadata['ad_click_count']}\nValue: {document[1]}\n"
@@ -300,8 +318,23 @@ The ADS_DATA provided to you is as follows:
             res += "\n"
         return res
 
-    def logResult(self, curr_relation_prompt, curr_question_prompt, page_information, relation_answer, question_answer):
-        print("----------------------------------------------------------------------------------------------------------------------------------------------------------------\n", curr_relation_prompt, curr_question_prompt, page_information, json.dumps(relation_answer, indent=4), json.dumps(question_answer, indent=4), "\n----------------------------------------------------------------------------------------------------------------------------------------------------------------\n\n")
+    def logResult(
+        self,
+        curr_relation_prompt,
+        curr_question_prompt,
+        page_information,
+        relation_answer,
+        question_answer,
+    ):
+        print(
+            "**************************************************************************************************\n",
+            # curr_relation_prompt,
+            # curr_question_prompt,
+            page_information,
+            json.dumps(relation_answer, indent=4),
+            json.dumps(question_answer, indent=4),
+            "\n************************************************************************************************\n\n",
+        )
 
     def getRagResponse(
         self, RelationPrompt, QuestionPrompt, threshold, page_information
@@ -314,7 +347,9 @@ The ADS_DATA provided to you is as follows:
         if QuestionPrompt != None or len(QuestionPrompt):
             curr_question_prompt = QuestionPrompt
 
-        documents_clusters, best_value = self.db.queryVectorDB(page_information, threshold)
+        documents_clusters, best_value = self.db.queryVectorDB(
+            page_information, threshold
+        )
         relation_answer, question_answer = (
             self.getBestQuestionOnTheBasisOfPageInformationAndAdsData(
                 page_information,
@@ -324,7 +359,13 @@ The ADS_DATA provided to you is as follows:
                 best_value,
             )
         )
-        self.logResult(curr_relation_prompt, curr_relation_prompt, page_information, relation_answer, question_answer)
+        self.logResult(
+            curr_relation_prompt,
+            curr_relation_prompt,
+            page_information,
+            relation_answer,
+            question_answer,
+        )
 
         docs_info = self.changeDocumentsToPrintableString(documents_clusters)
         relation_answer_string = self.changeResponseToPrintableString(