klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Improvement] add default user-agent header in webpage loader (#1219)

Browse Source
This commit is contained in:
Deven Patel
2024-01-26 11:04:25 +05:30
committed by GitHub
parent 433f99dd78
commit b210723de1
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
embedchain/loaders/web_page.py
View File

@@ -22,7 +22,10 @@ class WebPageLoader(BaseLoader):
def load_data(self, url):
"""Load data from a web page using a shared requests' session."""
response = self._session.get(url, timeout=30)
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", # noqa:E501
}
response = self._session.get(url, headers=headers, timeout=30)
response.raise_for_status()
data = response.content
content = self._get_clean_content(data, url)