Update website to web page

This commit renames the website loader, chunker
to web page, as it is loading and chunking a single
url than the complete website.
This commit is contained in:
Taranjeet Singh
2023-06-20 16:50:54 +05:30
parent 0fc960e958
commit 08f155a551
3 changed files with 6 additions and 6 deletions

View File

@@ -0,0 +1,30 @@
import requests
from bs4 import BeautifulSoup
from embedchain.utils import clean_string
class WebPageLoader:
def load_data(self, url):
response = requests.get(url)
data = response.content
soup = BeautifulSoup(data, 'html.parser')
for tag in soup([
"nav", "aside", "form", "header",
"noscript", "svg", "canvas",
"footer", "script", "style"
]):
tag.string = " "
output = []
content = soup.get_text()
content = clean_string(content)
meta_data = {
"url": url,
}
output.append({
"content": content,
"meta_data": meta_data,
})
return output