Improve tests (#780)

embedchain/loaders/web_page.py

@@ -15,7 +15,25 @@ class WebPageLoader(BaseLoader):
         """Load data from a web page."""
         response = requests.get(url)
         data = response.content
-        soup = BeautifulSoup(data, "html.parser")
+        content = self._get_clean_content(data, url)
+
+        meta_data = {
+            "url": url,
+        }
+
+        doc_id = hashlib.sha256((content + url).encode()).hexdigest()
+        return {
+            "doc_id": doc_id,
+            "data": [
+                {
+                    "content": content,
+                    "meta_data": meta_data,
+                }
+            ],
+        }
+
+    def _get_clean_content(self, html, url) -> str:
+        soup = BeautifulSoup(html, "html.parser")
         original_size = len(str(soup.get_text()))
 
         tags_to_exclude = [
@@ -61,17 +79,4 @@ class WebPageLoader(BaseLoader):
                 f"[{url}] Cleaned page size: {cleaned_size} characters, down from {original_size} (shrunk: {original_size-cleaned_size} chars, {round((1-(cleaned_size/original_size)) * 100, 2)}%)"  # noqa:E501
             )
 
-        meta_data = {
-            "url": url,
-        }
-        content = content
-        doc_id = hashlib.sha256((content + url).encode()).hexdigest()
-        return {
-            "doc_id": doc_id,
-            "data": [
-                {
-                    "content": content,
-                    "meta_data": meta_data,
-                }
-            ],
-        }
+        return content