klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: don't instantiate every class in dict (#328)

Browse Source
This commit is contained in:
cachho
2023-07-20 08:22:44 +02:00
committed by GitHub
parent 91033c7221
commit 6b61b7e9c1
1 changed files with 12 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
embedchain/data_formatter/data_formatter.py
View File

@@ -58,18 +58,19 @@ class DataFormatter:
:return: The chunker for the given data type.
:raises ValueError: If an unsupported data type is provided.
"""
chunkers = {
"youtube_video": YoutubeVideoChunker(config),
"pdf_file": PdfFileChunker(config),
"web_page": WebPageChunker(config),
"qna_pair": QnaPairChunker(config),
"text": TextChunker(config),
"docx": DocxFileChunker(config),
"sitemap": WebPageChunker(config),
"docs_site": DocsSiteChunker(config),
chunker_classes = {
"youtube_video": YoutubeVideoChunker,
"pdf_file": PdfFileChunker,
"web_page": WebPageChunker,
"qna_pair": QnaPairChunker,
"text": TextChunker,
"docx": DocxFileChunker,
"sitemap": WebPageChunker,
"docs_site": DocsSiteChunker,
}
if data_type in chunkers:
chunker = chunkers[data_type]
if data_type in chunker_classes:
chunker_class = chunker_classes[data_type]
chunker = chunker_class(config)
chunker.set_data_type(data_type)
return chunker
else: