[Improvement] Add support for min chunk size (#1007)

2023-12-15 05:59:15 +05:30
parent 9303a1bf81
commit c0ee680546
11 changed files with 59 additions and 25 deletions
--- a/tests/embedchain/test_add.py
+++ b/tests/embedchain/test_add.py
@@ -33,7 +33,7 @@ def test_add_forced_type(app):


 def test_dry_run(app):
-    chunker_config = ChunkerConfig(chunk_size=1, chunk_overlap=0)
+    chunker_config = ChunkerConfig(chunk_size=1, chunk_overlap=0, min_chunk_size=0)
    text = """0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"""

    result = app.add(source=text, config=AddConfig(chunker=chunker_config), dry_run=True)