feat: Add new data type: code_docs_loader (#274)
This commit is contained in:
22
embedchain/chunkers/code_docs_page.py
Normal file
22
embedchain/chunkers/code_docs_page.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from typing import Optional
|
||||
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
|
||||
from embedchain.chunkers.base_chunker import BaseChunker
|
||||
from embedchain.config.AddConfig import ChunkerConfig
|
||||
|
||||
TEXT_SPLITTER_CHUNK_PARAMS = {
|
||||
"chunk_size": 500,
|
||||
"chunk_overlap": 50,
|
||||
"length_function": len,
|
||||
}
|
||||
|
||||
|
||||
class CodeDocsPageChunker(BaseChunker):
|
||||
"""Chunker for code docs page."""
|
||||
|
||||
def __init__(self, config: Optional[ChunkerConfig] = None):
|
||||
if config is None:
|
||||
config = TEXT_SPLITTER_CHUNK_PARAMS
|
||||
text_splitter = RecursiveCharacterTextSplitter(**config)
|
||||
super().__init__(text_splitter)
|
||||
Reference in New Issue
Block a user