[Feature Improvement] Update JSON Loader to support loading data from more sources (#898)

Co-authored-by: Deven Patel <deven298@yahoo.com>

embedchain/embedchain.py

@@ -20,7 +20,7 @@ from embedchain.loaders.base_loader import BaseLoader
 from embedchain.models.data_type import (DataType, DirectDataType,
                                          IndirectDataType, SpecialDataType)
 from embedchain.telemetry.posthog import AnonymousTelemetry
-from embedchain.utils import detect_datatype
+from embedchain.utils import detect_datatype, is_valid_json_string
 from embedchain.vectordb.base import BaseVectorDB
 
 load_dotenv()
@@ -175,11 +175,27 @@ class EmbedChain(JSONSerializable):
         if data_type:
             try:
                 data_type = DataType(data_type)
+                if data_type == DataType.JSON:
+                    if isinstance(source, str):
+                        if not is_valid_json_string(source):
+                            raise ValueError(
+                                f"Invalid json input: {source}",
+                                "Provide the correct JSON formatted source, \
+                                    refer `https://docs.embedchain.ai/data-sources/json`",
+                            )
+                    elif not isinstance(source, str):
+                        raise ValueError(
+                            "Invaid content input. \
+                            If you want to upload (list, dict, etc.), do \
+                                `json.dump(data, indent=0)` and add the stringified JSON. \
+                                    Check - `https://docs.embedchain.ai/data-sources/json`"
+                        )
             except ValueError:
                 raise ValueError(
                     f"Invalid data_type: '{data_type}'.",
                     f"Please use one of the following: {[data_type.value for data_type in DataType]}",
                 ) from None
+
         if not data_type:
             data_type = detect_datatype(source)
 
@@ -287,6 +303,10 @@ class EmbedChain(JSONSerializable):
             # These types have a indirect source reference
             # As long as the reference is the same, they can be updated.
             where = {"url": src}
+            if chunker.data_type == DataType.JSON and is_valid_json_string(src):
+                url = hashlib.sha256((src).encode("utf-8")).hexdigest()
+                where = {"url": url}
+
             if self.config.id is not None:
                 where.update({"app_id": self.config.id})
 
@@ -368,6 +388,10 @@ class EmbedChain(JSONSerializable):
 
         # get existing ids, and discard doc if any common id exist.
         where = {"url": src}
+        if chunker.data_type == DataType.JSON and is_valid_json_string(src):
+            url = hashlib.sha256((src).encode("utf-8")).hexdigest()
+            where = {"url": url}
+
         # if data type is qna_pair, we check for question
         if chunker.data_type == DataType.QNA_PAIR:
             where = {"question": src[0]}