klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: don't instantiate every class in dict (#328)

Browse Source
This commit is contained in:
cachho
2023-07-20 08:22:44 +02:00
committed by GitHub
parent 91033c7221
commit 6b61b7e9c1
1 changed files with 12 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
embedchain/data_formatter/data_formatter.py
View File

@@ -58,18 +58,19 @@ class DataFormatter:
:return: The chunker for the given data type. :return: The chunker for the given data type.
:raises ValueError: If an unsupported data type is provided. :raises ValueError: If an unsupported data type is provided.
""" """
chunkers = { chunker_classes = {
"youtube_video": YoutubeVideoChunker(config), "youtube_video": YoutubeVideoChunker,
"pdf_file": PdfFileChunker(config), "pdf_file": PdfFileChunker,
"web_page": WebPageChunker(config), "web_page": WebPageChunker,
"qna_pair": QnaPairChunker(config), "qna_pair": QnaPairChunker,
"text": TextChunker(config), "text": TextChunker,
"docx": DocxFileChunker(config), "docx": DocxFileChunker,
"sitemap": WebPageChunker(config), "sitemap": WebPageChunker,
"docs_site": DocsSiteChunker(config), "docs_site": DocsSiteChunker,
} }
if data_type in chunkers: if data_type in chunker_classes:
chunker = chunkers[data_type] chunker_class = chunker_classes[data_type]
chunker = chunker_class(config)
chunker.set_data_type(data_type) chunker.set_data_type(data_type)
return chunker return chunker
else: else: