klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[New] Beehiiv loader (#963)

Browse Source
This commit is contained in:
Sidharth Mohanty
2023-12-08 03:41:56 +05:30
committed by GitHub
parent 4a5ed1dd8d
commit 51ebf3439b
8 changed files with 153 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
embedchain/loaders/substack.py
View File

@@ -12,9 +12,7 @@ from embedchain.utils import is_readable
@register_deserializable
class SubstackLoader(BaseLoader):
"""
This method takes a sitemap URL as input and retrieves
all the URLs to use the WebPageLoader to load content
of each page.
This loader is used to load data from Substack URLs.
"""
def load_data(self, url: str):
@@ -62,10 +60,10 @@ class SubstackLoader(BaseLoader):
def load_link(link: str):
try:
each_load_data = requests.get(link)
each_load_data.raise_for_status()
substack_data = requests.get(link)
substack_data.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser")
soup = BeautifulSoup(substack_data.text, "html.parser")
data = serialize_response(soup)
data = str(data)
if is_readable(data):