feat: filter sitemap (#304)

docs/advanced/data_types.mdx

@@ -30,6 +30,14 @@ To add any web page, use the data_type as `web_page`. Eg:
 app.add('web_page', 'a_valid_web_page_url')
 ```
 
+### Sitemap
+
+Add all web pages from an xml-sitemap. Filters non-text files. Use the data_type as `sitemap`. Eg:
+
+```python
+app.add('sitemap', 'https://example.com/sitemap.xml')
+```
+
 ### Doc file
 
 To add any doc/docx file, use the data_type as `docx`. Eg:

embedchain/loaders/sitemap.py

@@ -5,6 +5,7 @@ from bs4 import BeautifulSoup
 from bs4.builder import ParserRejectedMarkup
 
 from embedchain.loaders.web_page import WebPageLoader
+from embedchain.utils import is_readable
 
 
 class SitemapLoader:
@@ -20,11 +21,20 @@ class SitemapLoader:
         response.raise_for_status()
 
         soup = BeautifulSoup(response.text, "xml")
-        links = [link.text for link in soup.find_all("loc")]
+
+        links = [link.text for link in soup.find_all("loc") if link.parent.name == "url"]
+        if len(links) == 0:
+            # Get all <loc> tags as a fallback. This might include images.
+            links = [link.text for link in soup.find_all("loc")]
+
         for link in links:
             try:
                 each_load_data = web_page_loader.load_data(link)
-                output.append(each_load_data)
+
+                if is_readable(each_load_data[0].get("content")):
+                    output.append(each_load_data)
+                else:
+                    logging.warning(f"Page is not readable (too many invalid characters): {link}")
             except ParserRejectedMarkup as e:
                 logging.error(f"Failed to parse {link}: {e}")
         return [data[0] for data in output]

embedchain/utils.py

@@ -1,4 +1,5 @@
 import re
+import string
 
 
 def clean_string(text):
@@ -33,3 +34,14 @@ def clean_string(text):
     cleaned_text = re.sub(r"([^\w\s])\1*", r"\1", cleaned_text)
 
     return cleaned_text
+
+
+def is_readable(s):
+    """
+    Heuristic to determine if a string is "readable" (mostly contains printable characters and forms meaningful words)
+
+    :param s: string
+    :return: True if the string is more than 95% printable.
+    """
+    printable_ratio = sum(c in string.printable for c in s) / len(s)
+    return printable_ratio > 0.95  # 95% of characters are printable