[Improvements] Improve the default prompt and data loader util functions (#1272)

This commit is contained in:
Deshraj Yadav
2024-02-18 14:06:32 -08:00
committed by GitHub
parent 9a11683003
commit 6c12bc9044
19 changed files with 79 additions and 62 deletions

View File

@@ -19,10 +19,10 @@ class XmlLoader(BaseLoader):
data = loader.load()
content = data[0].page_content
content = clean_string(content)
meta_data = data[0].metadata
meta_data["url"] = meta_data["source"]
del meta_data["source"]
output = [{"content": content, "meta_data": meta_data}]
metadata = data[0].metadata
metadata["url"] = metadata["source"]
del metadata["source"]
output = [{"content": content, "meta_data": metadata}]
doc_id = hashlib.sha256((content + xml_url).encode()).hexdigest()
return {
"doc_id": doc_id,