[Improvements] Improve the default prompt and data loader util functions (#1272)

This commit is contained in:
Deshraj Yadav
2024-02-18 14:06:32 -08:00
committed by GitHub
parent 9a11683003
commit 6c12bc9044
19 changed files with 79 additions and 62 deletions

View File

@@ -26,12 +26,12 @@ class UnstructuredLoader(BaseLoader):
for page in pages:
content = page.page_content
content = clean_string(content)
meta_data = page.metadata
meta_data["url"] = url
metadata = page.metadata
metadata["url"] = url
data.append(
{
"content": content,
"meta_data": meta_data,
"meta_data": metadata,
}
)
all_content.append(content)