klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: do not instantiate all loaders (#418)

Browse Source
This commit is contained in:
cachho
2023-08-12 02:02:40 +02:00
committed by GitHub
parent 3cab4415b7
commit a232d1b779
1 changed files with 11 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
embedchain/data_formatter/data_formatter.py
View File

@@ -37,18 +37,20 @@ class DataFormatter:
:raises ValueError: If an unsupported data type is provided. :raises ValueError: If an unsupported data type is provided.
""" """
loaders = { loaders = {
"youtube_video": YoutubeVideoLoader(), "youtube_video": YoutubeVideoLoader,
"pdf_file": PdfFileLoader(), "pdf_file": PdfFileLoader,
"web_page": WebPageLoader(), "web_page": WebPageLoader,
"qna_pair": LocalQnaPairLoader(), "qna_pair": LocalQnaPairLoader,
"text": LocalTextLoader(), "text": LocalTextLoader,
"docx": DocxFileLoader(), "docx": DocxFileLoader,
"sitemap": SitemapLoader(), "sitemap": SitemapLoader,
"docs_site": DocsSiteLoader(), "docs_site": DocsSiteLoader,
} }
lazy_loaders = ("notion",) lazy_loaders = ("notion",)
if data_type in loaders: if data_type in loaders:
return loaders[data_type] loader_class = loaders[data_type]
loader = loader_class()
return loader
elif data_type in lazy_loaders: elif data_type in lazy_loaders:
if data_type == "notion": if data_type == "notion":
from embedchain.loaders.notion import NotionLoader from embedchain.loaders.notion import NotionLoader