From aaf4705804af1e73617f0b460826776be8039f8e Mon Sep 17 00:00:00 2001
From: Gustaf Rydholm <gustaf.rydholm@gmail.com>
Date: Fri, 5 Apr 2024 00:25:42 +0200
Subject: Add pdf parser

---
 rag/parser/__init__.py |  0
 rag/parser/pdf.py      | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+)
 create mode 100644 rag/parser/__init__.py
 create mode 100644 rag/parser/pdf.py

(limited to 'rag/parser')

diff --git a/rag/parser/__init__.py b/rag/parser/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/rag/parser/pdf.py b/rag/parser/pdf.py
new file mode 100644
index 0000000..cb63fc4
--- /dev/null
+++ b/rag/parser/pdf.py
@@ -0,0 +1,19 @@
+from pathlib import Path
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+CHUNK_SIZE = 1024
+CHUNK_OVERLAP = 256
+
+
+def parser(filepath: Path):
+    content = PyPDFLoader(filepath).load()
+    print(content)
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP
+    )
+    chunks = splitter.split_documents(content)
+    return chunks
+
+
+# TODO: add parser for bytearray
-- 
cgit v1.2.3-70-g09d2