klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Feature] JSON data loader support (#816)

Browse Source
This commit is contained in:
Deven Patel
2023-10-18 13:53:15 -07:00
committed by GitHub
parent 4dc1785ef1
commit 7641cba01d
10 changed files with 99 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
embedchain/data_formatter/data_formatter.py
View File

@@ -2,6 +2,7 @@ from embedchain.chunkers.base_chunker import BaseChunker
from embedchain.chunkers.docs_site import DocsSiteChunker
from embedchain.chunkers.docx_file import DocxFileChunker
from embedchain.chunkers.images import ImagesChunker
from embedchain.chunkers.json import JSONChunker
from embedchain.chunkers.mdx import MdxChunker
from embedchain.chunkers.notion import NotionChunker
from embedchain.chunkers.pdf_file import PdfFileChunker
@@ -20,6 +21,7 @@ from embedchain.loaders.csv import CsvLoader
from embedchain.loaders.docs_site_loader import DocsSiteLoader
from embedchain.loaders.docx_file import DocxFileLoader
from embedchain.loaders.images import ImagesLoader
from embedchain.loaders.json import JSONLoader
from embedchain.loaders.local_qna_pair import LocalQnaPairLoader
from embedchain.loaders.local_text import LocalTextLoader
from embedchain.loaders.mdx import MdxLoader
@@ -75,6 +77,7 @@ class DataFormatter(JSONSerializable):
DataType.CSV: CsvLoader,
DataType.MDX: MdxLoader,
DataType.IMAGES: ImagesLoader,
DataType.JSON: JSONLoader,
}
lazy_loaders = {DataType.NOTION}
if data_type in loaders:
@@ -116,6 +119,7 @@ class DataFormatter(JSONSerializable):
DataType.MDX: MdxChunker,
DataType.IMAGES: ImagesChunker,
DataType.XML: XmlChunker,
DataType.JSON: JSONChunker,
}
if data_type in chunker_classes:
chunker_class: type = chunker_classes[data_type]