initial commit

.gitignore

@@ -0,0 +1,3 @@
+.venv/
+.env
+__pycache__/

.python-version

@@ -0,0 +1 @@
+3.10

app.py

@@ -0,0 +1,26 @@
+import gradio as gr
+from arxiv_metadata_service import ArxivMetadataService
+import traceback
+
+arxiv_service = ArxivMetadataService()
+
+def extract_metadata(query: str, max_results: int):
+    try:
+        return arxiv_service.extract_and_update(query, max_results)
+    except Exception as e:
+        error_msg = f"An error occurred: {str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+        return error_msg
+
+demo = gr.Interface(
+    fn=extract_metadata,
+    inputs=[
+        gr.Textbox(label="ArXiv Query"),
+        gr.Slider(minimum=1, maximum=100, value=10, step=1, label="Max Results")
+    ],
+    outputs="text",
+    title="ArXiv Metadata Extractor",
+    description="Extract metadata from ArXiv papers and update the dataset."
+)
+
+if __name__ == "__main__":
+    demo.launch()

arxiv_fetcher.py

@@ -0,0 +1,38 @@
+# arxiv_fetcher.py
+
+import arxiv
+from typing import List, Dict, Any
+import logging
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def fetch_arxiv_metadata(query: str, max_results: int = 10) -> List[Dict[str, Any]]:
+    logging.info(f"Fetching arXiv metadata for query: {query}")
+    if not query.strip():
+        logging.warning("Empty or whitespace-only query provided")
+        return []
+    
+    client = arxiv.Client(page_size=max_results, delay_seconds=3, num_retries=3)
+    search = arxiv.Search(query=query, max_results=max_results, sort_by=arxiv.SortCriterion.SubmittedDate)
+    
+    results = []
+    try:
+        for result in client.results(search):
+            metadata = {
+                "title": result.title,
+                "authors": [author.name for author in result.authors],
+                "published": result.published.isoformat(),
+                "updated": result.updated.isoformat(),
+                "pdf_url": result.pdf_url,
+                "entry_id": result.entry_id,
+                "summary": result.summary,
+                "categories": result.categories,
+                "primary_category": result.primary_category,
+                "html_url": f"http://arxiv.org/abs/{result.entry_id.split('/')[-1]}"
+            }
+            results.append(metadata)
+        logging.info(f"Fetched metadata for {len(results)} papers")
+    except Exception as e:
+        logging.error(f"Error fetching metadata: {str(e)}")
+    
+    return results

arxiv_metadata_service.py

@@ -0,0 +1,36 @@
+from arxiv_fetcher import fetch_arxiv_metadata
+from datasets import load_dataset, Dataset
+from config import DATASET_NAME
+import logging
+from typing import List, Dict, Any
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+class ArxivMetadataService:
+    def extract_and_update(self, query: str, max_results: int = 10) -> str:
+        metadata_list = fetch_arxiv_metadata(query, max_results)
+        return self.update_dataset(metadata_list)
+
+    def update_dataset(self, metadata_list: List[Dict[str, Any]]) -> str:
+        try:
+            dataset = load_dataset(DATASET_NAME, split="train")
+            current_data = dataset.to_dict()
+            
+            for paper in metadata_list:
+                if paper['id'] not in current_data.get('id', []):
+                    for key, value in paper.items():
+                        if key not in current_data:
+                            current_data[key] = []
+                        current_data[key].append(value)
+                else:
+                    index = current_data['id'].index(paper['id'])
+                    for key, value in paper.items():
+                        current_data[key][index] = value
+            
+            updated_dataset = Dataset.from_dict(current_data)
+            updated_dataset.push_to_hub(DATASET_NAME, split="train")
+            
+            return f"Successfully updated dataset with {len(metadata_list)} papers"
+        except Exception as e:
+            logging.error(f"Failed to update dataset: {str(e)}")
+            return f"Failed to update dataset: {str(e)}"

config.py

@@ -0,0 +1,15 @@
+# File: config.py
+import os
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+QDRANT_API_KEY = os.getenv("QDRANT_API_KEY")
+QDRANT_API_URL = os.getenv("QDRANT_API_URL")
+GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+COLLECTION_NAME = "arxiv_papers"
+DATASET_NAME = "dwb2023/arxiv-papers-dataset"
+
+LANGCHAIN_PROJECT="arxiv_papers"
+LANGCHAIN_ENDPOINT="https://api.smith.langchain.com"
+LANGCHAIN_TRACING_V2="true"
+LANGCHAIN_HUB_PROMPT="rlm/rag-prompt-llama3"
+LANGCHAIN_API_KEY=os.getenv("LANGCHAIN_API_KEY")

requirements.txt

@@ -0,0 +1,3 @@
+arxiv
+datasets
+gradio