diff --git a/embedchain/chunkers/web_page.py b/embedchain/chunkers/web_page.py index 253b2b41..5ef7f40d 100644 --- a/embedchain/chunkers/web_page.py +++ b/embedchain/chunkers/web_page.py @@ -13,7 +13,7 @@ class WebPageChunker(BaseChunker): def __init__(self, config: Optional[ChunkerConfig] = None): if config is None: - config = ChunkerConfig(chunk_size=500, chunk_overlap=0, length_function=len) + config = ChunkerConfig(chunk_size=2000, chunk_overlap=0, length_function=len) text_splitter = RecursiveCharacterTextSplitter( chunk_size=config.chunk_size, chunk_overlap=config.chunk_overlap, diff --git a/tests/chunkers/test_chunkers.py b/tests/chunkers/test_chunkers.py index 1e725a8a..e0de9958 100644 --- a/tests/chunkers/test_chunkers.py +++ b/tests/chunkers/test_chunkers.py @@ -31,7 +31,7 @@ chunker_common_config = { QnaPairChunker: {"chunk_size": 300, "chunk_overlap": 0, "length_function": len}, TableChunker: {"chunk_size": 300, "chunk_overlap": 0, "length_function": len}, SitemapChunker: {"chunk_size": 500, "chunk_overlap": 0, "length_function": len}, - WebPageChunker: {"chunk_size": 500, "chunk_overlap": 0, "length_function": len}, + WebPageChunker: {"chunk_size": 2000, "chunk_overlap": 0, "length_function": len}, XmlChunker: {"chunk_size": 500, "chunk_overlap": 50, "length_function": len}, YoutubeVideoChunker: {"chunk_size": 2000, "chunk_overlap": 0, "length_function": len}, JSONChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},