feat: add local text (#44)

This commits extends the "add_local" function. It
adds support to take text and index/embed it.
This commit is contained in:
cachho
2023-06-25 19:43:41 +02:00
committed by GitHub
parent b9277c84c8
commit f5f5e7edd1
4 changed files with 40 additions and 1 deletions

View File

@@ -0,0 +1,16 @@
from embedchain.chunkers.base_chunker import BaseChunker
from langchain.text_splitter import RecursiveCharacterTextSplitter
TEXT_SPLITTER_CHUNK_PARAMS = {
"chunk_size": 300,
"chunk_overlap": 0,
"length_function": len,
}
class TextChunker(BaseChunker):
def __init__(self):
text_splitter = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)
super().__init__(text_splitter)