feat: add local text (#44)
This commits extends the "add_local" function. It adds support to take text and index/embed it.
This commit is contained in:
16
embedchain/chunkers/text.py
Normal file
16
embedchain/chunkers/text.py
Normal file
@@ -0,0 +1,16 @@
|
||||
from embedchain.chunkers.base_chunker import BaseChunker
|
||||
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
|
||||
|
||||
TEXT_SPLITTER_CHUNK_PARAMS = {
|
||||
"chunk_size": 300,
|
||||
"chunk_overlap": 0,
|
||||
"length_function": len,
|
||||
}
|
||||
|
||||
|
||||
class TextChunker(BaseChunker):
|
||||
def __init__(self):
|
||||
text_splitter = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)
|
||||
super().__init__(text_splitter)
|
||||
Reference in New Issue
Block a user