24 lines
913 B
Python
24 lines
913 B
Python
import hashlib
|
|
|
|
from langchain.document_loaders.json_loader import JSONLoader as LcJSONLoader
|
|
|
|
from embedchain.loaders.base_loader import BaseLoader
|
|
|
|
langchain_json_jq_schema = 'to_entries | map("\(.key): \(.value|tostring)") | .[]'
|
|
|
|
|
|
class JSONLoader(BaseLoader):
|
|
@staticmethod
|
|
def load_data(content):
|
|
"""Load a json file. Each data point is a key value pair."""
|
|
data = []
|
|
data_content = []
|
|
loader = LcJSONLoader(content, text_content=False, jq_schema=langchain_json_jq_schema)
|
|
docs = loader.load()
|
|
for doc in docs:
|
|
meta_data = doc.metadata
|
|
data.append({"content": doc.page_content, "meta_data": {"url": content, "row": meta_data["seq_num"]}})
|
|
data_content.append(doc.page_content)
|
|
doc_id = hashlib.sha256((content + ", ".join(data_content)).encode()).hexdigest()
|
|
return {"doc_id": doc_id, "data": data}
|