Resolve conflicts (#208)

This commit is contained in:
Deshraj Yadav
2023-07-10 21:50:05 -07:00
committed by GitHub
parent 6936d6983d
commit 9ca836520f
32 changed files with 396 additions and 207 deletions

View File

@@ -1,22 +1,29 @@
import requests
from bs4 import BeautifulSoup
from embedchain.utils import clean_string
class WebPageLoader:
def load_data(self, url):
''' Load data from a web page. '''
"""Load data from a web page."""
response = requests.get(url)
data = response.content
soup = BeautifulSoup(data, 'html.parser')
for tag in soup([
"nav", "aside", "form", "header",
"noscript", "svg", "canvas",
"footer", "script", "style"
]):
soup = BeautifulSoup(data, "html.parser")
for tag in soup(
[
"nav",
"aside",
"form",
"header",
"noscript",
"svg",
"canvas",
"footer",
"script",
"style",
]
):
tag.string = " "
output = []
content = soup.get_text()
@@ -24,8 +31,10 @@ class WebPageLoader:
meta_data = {
"url": url,
}
output.append({
"content": content,
"meta_data": meta_data,
})
return output
output.append(
{
"content": content,
"meta_data": meta_data,
}
)
return output