feat: add SitemapLoader (#68)

This commit is contained in:
aaishikdutta
2023-07-12 12:13:30 +05:30
committed by GitHub
parent 6fbf45498a
commit d2e8f796ca
6 changed files with 37 additions and 1 deletions

View File

@@ -9,6 +9,7 @@ from embedchain.loaders.docx_file import DocxFileLoader
from embedchain.loaders.local_qna_pair import LocalQnaPairLoader
from embedchain.loaders.local_text import LocalTextLoader
from embedchain.loaders.pdf_file import PdfFileLoader
from embedchain.loaders.sitemap import SitemapLoader
from embedchain.loaders.web_page import WebPageLoader
from embedchain.loaders.youtube_video import YoutubeVideoLoader
@@ -39,6 +40,7 @@ class DataFormatter:
"qna_pair": LocalQnaPairLoader(),
"text": LocalTextLoader(),
"docx": DocxFileLoader(),
"sitemap": SitemapLoader(),
}
if data_type in loaders:
return loaders[data_type]
@@ -60,6 +62,7 @@ class DataFormatter:
"qna_pair": QnaPairChunker(config),
"text": TextChunker(config),
"docx": DocxFileChunker(config),
"sitemap": WebPageChunker(config),
}
if data_type in chunkers:
return chunkers[data_type]