feat: add google doc support added (#155)

2023-07-06 14:04:27 +05:30
parent c6dbbf5dd3
commit 68e732a426
5 changed files with 70 additions and 27 deletions
--- a/embedchain/chunkers/doc_file.py
+++ b/embedchain/chunkers/doc_file.py
@@ -0,0 +1,16 @@
+from embedchain.chunkers.base_chunker import BaseChunker
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+
+TEXT_SPLITTER_CHUNK_PARAMS = {
+    "chunk_size": 500,
+    "chunk_overlap": 0,
+    "length_function": len,
+}
+
+
+class DocFileChunker(BaseChunker):
+    def __init__(self):
+        text_splitter = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)
+        super().__init__(text_splitter)