klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Refactor] Improve logging package wide (#1315)

Browse Source
This commit is contained in:
Deshraj Yadav
2024-03-13 17:13:30 -07:00
committed by GitHub
parent ef69c91b60
commit 3616eaadb4
54 changed files with 263 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
embedchain/loaders/sitemap.py
View File

@@ -19,6 +19,8 @@ from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
from embedchain.loaders.web_page import WebPageLoader
logger = logging.getLogger(__name__)
@register_deserializable
class SitemapLoader(BaseLoader):
@@ -41,7 +43,7 @@ class SitemapLoader(BaseLoader):
response.raise_for_status()
soup = BeautifulSoup(response.text, "xml")
except requests.RequestException as e:
logging.error(f"Error fetching sitemap from URL: {e}")
logger.error(f"Error fetching sitemap from URL: {e}")
return
elif os.path.isfile(sitemap_source):
with open(sitemap_source, "r") as file:
@@ -60,7 +62,7 @@ class SitemapLoader(BaseLoader):
loader_data = web_page_loader.load_data(link)
return loader_data.get("data")
except ParserRejectedMarkup as e:
logging.error(f"Failed to parse {link}: {e}")
logger.error(f"Failed to parse {link}: {e}")
return None
with concurrent.futures.ThreadPoolExecutor() as executor:
@@ -72,6 +74,6 @@ class SitemapLoader(BaseLoader):
if data:
output.extend(data)
except Exception as e:
logging.error(f"Error loading page {link}: {e}")
logger.error(f"Error loading page {link}: {e}")
return {"doc_id": doc_id, "data": output}