diff --git a/docs/advanced/data_types.mdx b/docs/advanced/data_types.mdx index 7e37efd1..b2ad8eb5 100644 --- a/docs/advanced/data_types.mdx +++ b/docs/advanced/data_types.mdx @@ -30,6 +30,14 @@ To add any web page, use the data_type as `web_page`. Eg: app.add('web_page', 'a_valid_web_page_url') ``` +### Sitemap + +Add all web pages from an xml-sitemap. Filters non-text files. Use the data_type as `sitemap`. Eg: + +```python +app.add('sitemap', 'https://example.com/sitemap.xml') +``` + ### Doc file To add any doc/docx file, use the data_type as `docx`. Eg: diff --git a/embedchain/loaders/sitemap.py b/embedchain/loaders/sitemap.py index 516581e0..09dd5f53 100644 --- a/embedchain/loaders/sitemap.py +++ b/embedchain/loaders/sitemap.py @@ -5,6 +5,7 @@ from bs4 import BeautifulSoup from bs4.builder import ParserRejectedMarkup from embedchain.loaders.web_page import WebPageLoader +from embedchain.utils import is_readable class SitemapLoader: @@ -20,11 +21,20 @@ class SitemapLoader: response.raise_for_status() soup = BeautifulSoup(response.text, "xml") - links = [link.text for link in soup.find_all("loc")] + + links = [link.text for link in soup.find_all("loc") if link.parent.name == "url"] + if len(links) == 0: + # Get all tags as a fallback. This might include images. + links = [link.text for link in soup.find_all("loc")] + for link in links: try: each_load_data = web_page_loader.load_data(link) - output.append(each_load_data) + + if is_readable(each_load_data[0].get("content")): + output.append(each_load_data) + else: + logging.warning(f"Page is not readable (too many invalid characters): {link}") except ParserRejectedMarkup as e: logging.error(f"Failed to parse {link}: {e}") return [data[0] for data in output] diff --git a/embedchain/utils.py b/embedchain/utils.py index f2b04490..02ee2deb 100644 --- a/embedchain/utils.py +++ b/embedchain/utils.py @@ -1,4 +1,5 @@ import re +import string def clean_string(text): @@ -33,3 +34,14 @@ def clean_string(text): cleaned_text = re.sub(r"([^\w\s])\1*", r"\1", cleaned_text) return cleaned_text + + +def is_readable(s): + """ + Heuristic to determine if a string is "readable" (mostly contains printable characters and forms meaningful words) + + :param s: string + :return: True if the string is more than 95% printable. + """ + printable_ratio = sum(c in string.printable for c in s) / len(s) + return printable_ratio > 0.95 # 95% of characters are printable