{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This part only runs periodically for updating infos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install schedule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import schedule\n",
    "import time\n",
    "\n",
    "def my_periodic_function():\n",
    "    print(\"This function runs periodically!\")\n",
    "\n",
    "# Schedule the function to run every 1 minute\n",
    "schedule.every(1).minutes.do(my_periodic_function)\n",
    "\n",
    "# Run the scheduler\n",
    "while True:\n",
    "    schedule.run_pending()\n",
    "    time.sleep(1)  # Optional: Sleep to avoid high CPU usage\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pedro/.local/lib/python3.10/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'vodafone.pt'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n",
      "/home/pedro/.local/lib/python3.10/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'www.vodafone.pt'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n",
      "Fetching pages:  38%|###8      | 2015/5279 [08:18<10:50,  5.01it/s] Error fetching https://www.vodafone.pt/ajuda/artigos/despiste-de-problemas-fixo/Internet/nao-sei-a-minha-password-ou-o-nome-da-rede-wi-fi.html with attempt 1/3: Server disconnected. Retrying...\n",
      "Fetching pages:  49%|####8     | 2579/5279 [09:58<13:25,  3.35it/s]Error fetching https://www.vodafone.pt/press-releases/2015/10/fox-play-chega-hoje-a-tv-box-da-vodafone.html with attempt 1/3: Server disconnected. Retrying...\n",
      "Fetching pages:  55%|#####5    | 2929/5279 [10:57<09:42,  4.03it/s]Error fetching https://www.vodafone.pt/press-releases/2009/5/geota-fundacao-vodafone-e-c-m-loures-apelam-a-criacao-de-corredores-verdes.html with attempt 1/3: Server disconnected. Retrying...\n",
      "Fetching pages: 100%|##########| 5279/5279 [18:02<00:00,  4.88it/s]\n"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "import nest_asyncio\n",
    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "from langchain.document_loaders import SitemapLoader\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import re\n",
    "\n",
    "# Wrap the function in a list\n",
    "filter_urls = ['^(?!https:\\/\\/www\\.vodafone\\.ptpesquisa$)(https?|ftp):\\/\\/[^\\s/$.?#].[^\\s]*$'] #remove https://www.vodafone.ptpesquisas\n",
    "\n",
    "sitemap_loader = SitemapLoader(\n",
    "    web_path=\"https://vodafone.pt/sitemap.xml\",\n",
    "    filter_urls=filter_urls,\n",
    ")\n",
    "\n",
    "sitemap_loader.requests_per_second = 10\n",
    "# Optional: avoid `[SSL: CERTIFICATE_VERIFY_FAILED]` issue\n",
    "sitemap_loader.requests_kwargs = {\"verify\": False}\n",
    "\n",
    "docs = sitemap_loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}