klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: docs_site use chunker config implementation (#326)

Browse Source
This commit is contained in:
cachho
2023-07-20 08:29:59 +02:00
committed by GitHub
parent 4bb06147c1
commit a681d47bce
2 changed files with 7 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/advanced/query_configuration.mdx
View File

@@ -36,6 +36,7 @@ Default values of chunker config parameters for different `data_type`:
|web_page|500|0|len| |web_page|500|0|len|
|pdf_file|1000|0|len| |pdf_file|1000|0|len|
|youtube_video|2000|0|len| |youtube_video|2000|0|len|
|docs_site|500|50|len|
### LoaderConfig ### LoaderConfig

14
embedchain/chunkers/docs_site.py
View File

@@ -5,18 +5,16 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
from embedchain.chunkers.base_chunker import BaseChunker from embedchain.chunkers.base_chunker import BaseChunker
from embedchain.config.AddConfig import ChunkerConfig from embedchain.config.AddConfig import ChunkerConfig
TEXT_SPLITTER_CHUNK_PARAMS = {
"chunk_size": 500,
"chunk_overlap": 50,
"length_function": len,
}
class DocsSiteChunker(BaseChunker): class DocsSiteChunker(BaseChunker):
"""Chunker for code docs site.""" """Chunker for code docs site."""
def __init__(self, config: Optional[ChunkerConfig] = None): def __init__(self, config: Optional[ChunkerConfig] = None):
if config is None: if config is None:
config = TEXT_SPLITTER_CHUNK_PARAMS config = ChunkerConfig(chunk_size=500, chunk_overlap=50, length_function=len)
text_splitter = RecursiveCharacterTextSplitter(**config) text_splitter = RecursiveCharacterTextSplitter(
chunk_size=config.chunk_size,
chunk_overlap=config.chunk_overlap,
length_function=config.length_function,
)
super().__init__(text_splitter) super().__init__(text_splitter)