klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Feature Improvement] Update JSON Loader to support loading data from more sources (#898)

Browse Source 
Co-authored-by: Deven Patel <deven298@yahoo.com>
This commit is contained in:
Deven Patel
2023-11-03 10:00:27 -07:00
committed by GitHub
parent e2546a653d
commit 53037b5ed8
6 changed files with 166 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
embedchain/loaders/json.py
View File

@@ -6,33 +6,37 @@ import re
import requests
from embedchain.loaders.base_loader import BaseLoader
from embedchain.utils import clean_string
from embedchain.utils import clean_string, is_valid_json_string
VALID_URL_PATTERN = "^https:\/\/[0-9A-z.]+.[0-9A-z.]+.[a-z]+\/.*\.json$"
class JSONLoader(BaseLoader):
@staticmethod
def load_data(content):
"""Load a json file. Each data point is a key value pair."""
def _get_llama_hub_loader():
try:
from llama_hub.jsondata.base import \
JSONDataReader as LLHBUBJSONLoader
except ImportError:
JSONDataReader as LLHUBJSONLoader
except ImportError as e:
raise Exception(
f"Couldn't import the required packages to load {content}, \
Do `pip install --upgrade 'embedchain[json]`"
f"Failed to install required packages: {e}, \
install them using `pip install --upgrade 'embedchain[json]`"
)
loader = LLHBUBJSONLoader()
return LLHUBJSONLoader()
if not isinstance(content, str):
print(f"Invaid content input. Provide the correct path to the json file saved locally in {content}")
@staticmethod
def load_data(content):
"""Load a json file. Each data point is a key value pair."""
loader = JSONLoader._get_llama_hub_loader()
data = []
data_content = []
# Load json data from various sources. TODO: add support for dictionary
content_url_str = content
# Load json data from various sources.
if os.path.isfile(content):
with open(content, "r", encoding="utf-8") as json_file:
json_data = json.load(json_file)
@@ -45,13 +49,17 @@ class JSONLoader(BaseLoader):
f"Loading data from the given url: {content} failed. \
Make sure the url is working."
)
elif is_valid_json_string(content):
json_data = content
content_url_str = hashlib.sha256((content).encode("utf-8")).hexdigest()
else:
raise ValueError(f"Invalid content to load json data from: {content}")
docs = loader.load_data(json_data)
for doc in docs:
doc_content = clean_string(doc.text)
data.append({"content": doc_content, "meta_data": {"url": content}})
data.append({"content": doc_content, "meta_data": {"url": content_url_str}})
data_content.append(doc_content)
doc_id = hashlib.sha256((content + ", ".join(data_content)).encode()).hexdigest()
doc_id = hashlib.sha256((content_url_str + ", ".join(data_content)).encode()).hexdigest()
return {"doc_id": doc_id, "data": data}