468db83337
This commit enables anyone to create a app and add 3 types of data sources: * pdf file * youtube video * website It exposes a function called query which first gets similar docs from vector db and then passes it to LLM to get the final answer.
30 lines
731 B
Python
30 lines
731 B
Python
import requests
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from embedchain.utils import clean_string
|
|
|
|
|
|
class WebsiteLoader:
|
|
|
|
def load_data(self, url):
|
|
response = requests.get(url)
|
|
data = response.content
|
|
soup = BeautifulSoup(data, 'html.parser')
|
|
for tag in soup([
|
|
"nav", "aside", "form", "header",
|
|
"noscript", "svg", "canvas",
|
|
"footer", "script", "style"
|
|
]):
|
|
tag.string = " "
|
|
output = []
|
|
content = soup.get_text()
|
|
content = clean_string(content)
|
|
meta_data = {
|
|
"url": url,
|
|
}
|
|
output.append({
|
|
"content": content,
|
|
"meta_data": meta_data,
|
|
})
|
|
return output |