klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: do not instantiate all loaders (#418)

Browse Source
This commit is contained in:
cachho
2023-08-12 02:02:40 +02:00
committed by GitHub
parent 3cab4415b7
commit a232d1b779
1 changed files with 11 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
embedchain/data_formatter/data_formatter.py
View File

@@ -37,18 +37,20 @@ class DataFormatter:
:raises ValueError: If an unsupported data type is provided.
"""
loaders = {
"youtube_video": YoutubeVideoLoader(),
"pdf_file": PdfFileLoader(),
"web_page": WebPageLoader(),
"qna_pair": LocalQnaPairLoader(),
"text": LocalTextLoader(),
"docx": DocxFileLoader(),
"sitemap": SitemapLoader(),
"docs_site": DocsSiteLoader(),
"youtube_video": YoutubeVideoLoader,
"pdf_file": PdfFileLoader,
"web_page": WebPageLoader,
"qna_pair": LocalQnaPairLoader,
"text": LocalTextLoader,
"docx": DocxFileLoader,
"sitemap": SitemapLoader,
"docs_site": DocsSiteLoader,
}
lazy_loaders = ("notion",)
if data_type in loaders:
return loaders[data_type]
loader_class = loaders[data_type]
loader = loader_class()
return loader
elif data_type in lazy_loaders:
if data_type == "notion":
from embedchain.loaders.notion import NotionLoader