[Feature] JSON data loader support (#816)

tests/chunkers/test_chunkers.py

@@ -10,6 +10,7 @@ from embedchain.chunkers.text import TextChunker
 from embedchain.chunkers.web_page import WebPageChunker
 from embedchain.chunkers.xml import XmlChunker
 from embedchain.chunkers.youtube_video import YoutubeVideoChunker
+from embedchain.chunkers.json import JSONChunker
 from embedchain.config.add_config import ChunkerConfig
 
 chunker_config = ChunkerConfig(chunk_size=500, chunk_overlap=0, length_function=len)
@@ -27,6 +28,7 @@ chunker_common_config = {
     WebPageChunker: {"chunk_size": 500, "chunk_overlap": 0, "length_function": len},
     XmlChunker: {"chunk_size": 500, "chunk_overlap": 50, "length_function": len},
     YoutubeVideoChunker: {"chunk_size": 2000, "chunk_overlap": 0, "length_function": len},
+    JSONChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},
 }
 
 

tests/loaders/test_json.py

@@ -0,0 +1,31 @@
+import hashlib
+from unittest.mock import patch
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.json_loader import JSONLoader as LcJSONLoader
+
+from embedchain.loaders.json import JSONLoader
+
+
+def test_load_data():
+    mock_document = [
+        Document(page_content="content1", metadata={"seq_num": 1}),
+        Document(page_content="content2", metadata={"seq_num": 2}),
+    ]
+    with patch.object(LcJSONLoader, "load", return_value=mock_document):
+        content = "temp.json"
+
+        result = JsonLoader.load_data(content)
+
+        assert "doc_id" in result
+        assert "data" in result
+
+        expected_data = [
+            {"content": "content1", "meta_data": {"url": content, "row": 1}},
+            {"content": "content2", "meta_data": {"url": content, "row": 2}},
+        ]
+
+        assert result["data"] == expected_data
+
+        expected_doc_id = hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest()
+        assert result["doc_id"] == expected_doc_id