featL AddConfig should allow configuring Chunker (#200)

2023-07-11 04:23:56 +05:30
parent ae87dc4a6d
commit eda28cc491
10 changed files with 120 additions and 39 deletions
--- a/embedchain/chunkers/text.py
+++ b/embedchain/chunkers/text.py
@@ -1,4 +1,6 @@
+from typing import Optional
 from embedchain.chunkers.base_chunker import BaseChunker
+from embedchain.config.AddConfig import ChunkerConfig

 from langchain.text_splitter import RecursiveCharacterTextSplitter

@@ -11,6 +13,8 @@ TEXT_SPLITTER_CHUNK_PARAMS = {


 class TextChunker(BaseChunker):
-    def __init__(self):
-        text_splitter = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)
+    def __init__(self, config: Optional[ChunkerConfig] = None):
+        if config is None:
+            config = TEXT_SPLITTER_CHUNK_PARAMS
+        text_splitter = RecursiveCharacterTextSplitter(**config)
        super().__init__(text_splitter)