From cd0c7bc971f513ccbd97833bfd315238ffb27c2f Mon Sep 17 00:00:00 2001 From: cachho Date: Sat, 15 Jul 2023 05:20:11 +0200 Subject: [PATCH] fix: escape bs4 parsing error (#271) --- embedchain/loaders/sitemap.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/embedchain/loaders/sitemap.py b/embedchain/loaders/sitemap.py index e38a2be8..516581e0 100644 --- a/embedchain/loaders/sitemap.py +++ b/embedchain/loaders/sitemap.py @@ -1,5 +1,8 @@ +import logging + import requests from bs4 import BeautifulSoup +from bs4.builder import ParserRejectedMarkup from embedchain.loaders.web_page import WebPageLoader @@ -19,6 +22,9 @@ class SitemapLoader: soup = BeautifulSoup(response.text, "xml") links = [link.text for link in soup.find_all("loc")] for link in links: - each_load_data = web_page_loader.load_data(link) - output.append(each_load_data) + try: + each_load_data = web_page_loader.load_data(link) + output.append(each_load_data) + except ParserRejectedMarkup as e: + logging.error(f"Failed to parse {link}: {e}") return [data[0] for data in output]