klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Speed up docs site loader (#1266)

Browse Source
This commit is contained in:
Saurabh Misra
2024-06-05 10:39:30 -07:00
committed by GitHub
parent 1354747ca8
commit 940f0128d5
1 changed files with 9 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
embedchain/loaders/docs_site_loader.py
View File

@@ -24,6 +24,9 @@ class DocsSiteLoader(BaseLoader):
self.visited_links = set() self.visited_links = set()
def _get_child_links_recursive(self, url): def _get_child_links_recursive(self, url):
if url in self.visited_links:
return
parsed_url = urlparse(url) parsed_url = urlparse(url)
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
current_path = parsed_url.path current_path = parsed_url.path
@@ -34,16 +37,15 @@ class DocsSiteLoader(BaseLoader):
return return
soup = BeautifulSoup(response.text, "html.parser") soup = BeautifulSoup(response.text, "html.parser")
all_links = [link.get("href") for link in soup.find_all("a")] all_links = (link.get("href") for link in soup.find_all("a", href=True))
child_links = [link for link in all_links if link and link.startswith(current_path) and link != current_path] child_links = (link for link in all_links if link.startswith(current_path) and link != current_path)
absolute_paths = [urljoin(base_url, link) for link in child_links] absolute_paths = set(urljoin(base_url, link) for link in child_links)
for link in absolute_paths: self.visited_links.update(absolute_paths)
if link not in self.visited_links:
self.visited_links.add(link) [self._get_child_links_recursive(link) for link in absolute_paths if link not in self.visited_links]
self._get_child_links_recursive(link)
def _get_all_urls(self, url): def _get_all_urls(self, url):
self.visited_links = set() self.visited_links = set()