Update website to web page

This commit renames the website loader, chunker
to web page, as it is loading and chunking a single
url than the complete website.
This commit is contained in:
Taranjeet Singh
2023-06-20 16:50:54 +05:30
parent 0fc960e958
commit 08f155a551
3 changed files with 6 additions and 6 deletions

View File

@@ -0,0 +1,16 @@
from embedchain.chunkers.base_chunker import BaseChunker
from langchain.text_splitter import RecursiveCharacterTextSplitter
TEXT_SPLITTER_CHUNK_PARAMS = {
"chunk_size": 500,
"chunk_overlap": 0,
"length_function": len,
}
class WebPageChunker(BaseChunker):
def __init__(self):
text_splitter = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)
super().__init__(text_splitter)