Update notebook

1 files changed, 77 insertions, 102 deletions

diff --git a/notebooks/testing.ipynb b/notebooks/testing.ipynb
index 36f3c16..de5e76e 100644
--- a/notebooks/testing.ipynb
+++ b/notebooks/testing.ipynb
@@ -2,18 +2,30 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "c1f56ae3-a056-4b31-bcab-27c2c97c00f1",
    "metadata": {},
    "outputs": [],
    "source": [
     "from pathlib import Path\n",
-    "from rag.parser.pdf import parser"
+    "from dotenv import load_dotenv\n",
+    "import numpy as np\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "from importlib.util import find_spec\n",
+    "if find_spec(\"rag\") is None:\n",
+    "    import sys\n",
+    "    sys.path.append('..')\n",
+    "from rag.parser.pdf import parser\n",
+    "from rag.db.embeddings import Embeddings\n",
+    "from rag.llm.encoder import Encoder\n",
+    "from rag.db.documents import Documents"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "6b5cb12e-df7e-4532-b78b-216e11ed6161",
    "metadata": {},
    "outputs": [],
@@ -23,17 +35,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "8e378ed2-e346-4360-8a8e-265f0f1d173d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "path"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "f1a57be8-21a2-48d3-b99f-d1bbf7b8780a",
    "metadata": {},
    "outputs": [],
@@ -43,48 +45,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "b848627e-1e30-492a-ada6-a899112cd515",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "chunks[32]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "5f01d39c-3310-4d6d-8b7b-f253975b5e37",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "\n",
-    "type(chunks[32])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "49bf6768-1350-42ee-8526-2e6bee2dd7bb",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from rag.db.vector import VectorStore"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "98516091-7bc5-43c2-9aa7-967e63b795f1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from rag.llms.encoder import Encoder"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "14408e20-3dec-40b4-9dda-3397beb0c453",
    "metadata": {},
    "outputs": [],
@@ -94,47 +55,58 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "e94993c5-6b19-4bac-b7e8-c26a5200a626",
    "metadata": {},
    "outputs": [],
    "source": [
-    "embs = encoder.encode([chunks[33].page_content])"
+    "embs = encoder.encode_document(chunks)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "b845bb31-0909-42cb-9957-9a8b3bb0b5c4",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1024,)"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "embs[0].shape"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "id": "aa279b1a-465e-4820-ab56-b25fc513c0a1",
    "metadata": {},
    "outputs": [],
    "source": [
-    "vec_db = VectorStore()"
+    "emb_db = Embeddings()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "id": "1dc655de-2359-42ce-b705-76ec06c5f72f",
    "metadata": {},
    "outputs": [],
    "source": [
-    "vec_db.add(embs[0])"
+    "emb_db.add(embs)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "117d3416-e79f-436f-a33e-ffb45b972b72",
    "metadata": {},
    "outputs": [],
@@ -144,7 +116,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "id": "3a6ef474-678c-4525-8dcb-ece67aa9c7ea",
    "metadata": {},
    "outputs": [],
@@ -154,70 +126,73 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "id": "4c8a16ba-6025-4a6e-95c2-bbba7a9a5de5",
    "metadata": {},
-   "outputs": [],
-   "source": [
-    "s, i = vec_db.search(qe, 1)\n",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(array([[122.61588, 127.4572 , 128.07301, 128.97739, 131.64783]],\n",
+       "       dtype=float32),\n",
+       " array([[149,  47, 224, 255, 254]]))"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "s, i = emb_db.search(qe, 5)\n",
     "s,i"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "2d5f4601-82b0-47b7-8c79-938435591b59",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "embs[0]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e7a65b4d-7269-44bf-9b29-9ef0ee2b2052",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "qe"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "016bae35-e22b-4e13-9c1d-6fcd09a8ca9f",
+   "execution_count": 12,
+   "id": "1c90dd20-c640-48b5-88c0-4ba93b60c5e6",
    "metadata": {},
    "outputs": [],
    "source": [
-    "import hashlib"
+    "docs_db = Documents()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "66f7fd80-a970-4ad9-b5c9-02ab2e39855b",
+   "execution_count": 14,
+   "id": "ed69d8bf-93f1-4353-a4c2-c4aacbe25420",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
-    "document = [chunk.page_content for chunk in chunks]\n",
-    "text = str.encode(\"\".join(document))"
+    "docs_db.add_document(chunks)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "9d385588-cdd5-4f8c-beae-31685a9f4c63",
+   "execution_count": 16,
+   "id": "40ebc825-2e2c-4110-93ff-ae6ec3dc1322",
    "metadata": {},
    "outputs": [],
    "source": [
-    "type(hashlib.sha256(text).hexdigest())"
+    "docs_db.add_chunk(chunks[100].page_content)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "1c90dd20-c640-48b5-88c0-4ba93b60c5e6",
+   "id": "4352b32d-c2e7-4fbf-aa05-fc46baf7c9f8",
    "metadata": {},
    "outputs": [],
    "source": []