chore: linting (#597)
This commit is contained in:
@@ -46,7 +46,4 @@ class CsvLoader(BaseLoader):
|
||||
lines.append(line)
|
||||
result.append({"content": line, "meta_data": {"url": content, "row": i + 1}})
|
||||
doc_id = hashlib.sha256((content + " ".join(lines)).encode()).hexdigest()
|
||||
return {
|
||||
"doc_id": doc_id,
|
||||
"data": result
|
||||
}
|
||||
return {"doc_id": doc_id, "data": result}
|
||||
|
||||
@@ -22,5 +22,5 @@ class LocalQnaPairLoader(BaseLoader):
|
||||
"content": content,
|
||||
"meta_data": meta_data,
|
||||
}
|
||||
]
|
||||
],
|
||||
}
|
||||
|
||||
@@ -20,5 +20,5 @@ class LocalTextLoader(BaseLoader):
|
||||
"content": content,
|
||||
"meta_data": meta_data,
|
||||
}
|
||||
]
|
||||
],
|
||||
}
|
||||
|
||||
@@ -39,9 +39,9 @@ class NotionLoader(BaseLoader):
|
||||
return {
|
||||
"doc_id": doc_id,
|
||||
"data": [
|
||||
{
|
||||
"content": text,
|
||||
"meta_data": {"url": f"notion-{formatted_id}"},
|
||||
}
|
||||
],
|
||||
{
|
||||
"content": text,
|
||||
"meta_data": {"url": f"notion-{formatted_id}"},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
@@ -43,7 +43,4 @@ class SitemapLoader(BaseLoader):
|
||||
logging.warning(f"Page is not readable (too many invalid characters): {link}")
|
||||
except ParserRejectedMarkup as e:
|
||||
logging.error(f"Failed to parse {link}: {e}")
|
||||
return {
|
||||
"doc_id": doc_id,
|
||||
"data": [data[0] for data in output]
|
||||
}
|
||||
return {"doc_id": doc_id, "data": [data[0] for data in output]}
|
||||
|
||||
@@ -66,7 +66,7 @@ class WebPageLoader(BaseLoader):
|
||||
}
|
||||
content = content
|
||||
doc_id = hashlib.sha256((content + url).encode()).hexdigest()
|
||||
return {
|
||||
return {
|
||||
"doc_id": doc_id,
|
||||
"data": [
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user