reran notebook

Author: haesleinhuepf

docs/63_chat_with_docs/llm-based-rag.ipynb

@@ -112,12 +112,14 @@
      "text": [
       "access/jupyterhub.md\n",
       "access/overview.md\n",
-      "access/desktop_cloud_visualization.md\n",
-      "access/graphical_applications_with_webvnc.md\n",
       "access/jupyterlab.md\n",
+      "quickstart/getting_started.md\n",
+      "software/big_data_frameworks.md\n",
       "software/data_analytics_with_python.md\n",
-      "software/jupyterhub_for_teaching.md\n",
-      "software/jupyterhub_teaching_example.md\n"
+      "software/data_analytics_with_r.md\n",
+      "software/data_analytics_with_rstudio.md\n",
+      "access/desktop_cloud_visualization.md\n",
+      "access/graphical_applications_with_webvnc.md\n"
      ]
     }
    ],
@@ -151,16 +153,14 @@
    "metadata": {},
    "outputs": [
     {
-     "ename": "FileNotFoundError",
-     "evalue": "[Errno 2] No such file or directory: 'hpc-compendium/doc.zih.tu-dresden.de/docs/software/jupyterhub_for_teaching.md'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
-      "Cell \u001b[1;32mIn[7], line 3\u001b[0m\n\u001b[0;32m      1\u001b[0m full_texts \u001b[38;5;241m=\u001b[39m {}\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m relevant_file_paths:\n\u001b[1;32m----> 3\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mdocs_root_folder\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mutf-8\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m      4\u001b[0m         full_texts[compendium_url \u001b[38;5;241m+\u001b[39m file[:\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m3\u001b[39m]] \u001b[38;5;241m=\u001b[39m f\u001b[38;5;241m.\u001b[39mread()\n\u001b[0;32m      7\u001b[0m documents \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin([\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m### \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m```\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mcontent\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m```\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m file, content \u001b[38;5;129;01min\u001b[39;00m full_texts\u001b[38;5;241m.\u001b[39mitems()])\n",
-      "File \u001b[1;32m~\\miniforge3\\envs\\genai-gpu\\Lib\\site-packages\\IPython\\core\\interactiveshell.py:324\u001b[0m, in \u001b[0;36m_modified_open\u001b[1;34m(file, *args, **kwargs)\u001b[0m\n\u001b[0;32m    317\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m {\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m}:\n\u001b[0;32m    318\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    319\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIPython won\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt let you open fd=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m by default \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    320\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mas it is likely to crash IPython. If you know what you are doing, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    321\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myou can use builtins\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m open.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    322\u001b[0m     )\n\u001b[1;32m--> 324\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mio_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
-      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'hpc-compendium/doc.zih.tu-dresden.de/docs/software/jupyterhub_for_teaching.md'"
-     ]
+     "data": {
+      "text/plain": [
+       "'### https://compendium.hpc.tu-dresden.de/access/jupyterhub \\n\\n```\\n# JupyterHub\\n\\nWith our JupyterHub service, we offer you a quick and easy way to work with\\nJupyter notebooks on ZIH systems. This page covers starting and stopping\\nJupyterHub sessions, error handling and customizing the environment.\\n\\nWe also provide a comprehensive documentation on how to use\\n[JupyterHub for Teaching (git-pull feature, quickstart links, direct links to notebook files)](jupyterhub_for_teaching.md).\\n\\n## Disclaimer\\n\\n!!'"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -177,10 +177,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "id": "4eab789f-aca4-4527-ae74-3ae1a62f9d17",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "To access the Jupyter Hub on the HPC system, you can visit the [JupyterHub](https://compendium.hpc.tu-dresden.de/access/jupyterhub) page and follow the instructions provided. According to the [https://compendium.hpc.tu-dresden.de/access/jupyterhub](https://compendium.hpc.tu-dresden.de/access/jupyterhub) document, you can access JupyterHub at [https://jupyterhub.hpc.tu-dresden.de](https://jupyterhub.hpc.tu-dresden.de) and log in with your ZIH credentials. \n",
+       "\n",
+       "Additionally, you can also find more information on how to access JupyterHub in other documents such as [https://compendium.hpc.tu-dresden.de/access/overview](https://compendium.hpc.tu-dresden.de/access/overview) and [https://compendium.hpc.tu-dresden.de/quickstart/getting_started](https://compendium.hpc.tu-dresden.de/quickstart/getting_started). \n",
+       "\n",
+       "Please note that you need to have a ZIH HPC login to access JupyterHub, and you can apply for it via the [HPC login application form](https://selfservice.zih.tu-dresden.de/index.php/hpclogin/noLogin) as mentioned in the [https://compendium.hpc.tu-dresden.de/quickstart/getting_started](https://compendium.hpc.tu-dresden.de/quickstart/getting_started) document. \n",
+       "\n",
+       "It is also worth mentioning that JupyterHub is available on other clusters such as `vis` as mentioned in the [https://compendium.hpc.tu-dresden.de/access/desktop_cloud_visualization](https://compendium.hpc.tu-dresden.de/access/desktop_cloud_visualization) document. \n",
+       "\n",
+       "Please let me know if you need further assistance!"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "response = prompt_scadsai_llm(f\"\"\"\n",
     "Given a question and a list of document summaries, identify documents that might be helpful for answering the question.\n",