klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add UA header for pdf and sitemap (#1222)

Browse Source
This commit is contained in:
Taranjeet Singh
2024-01-26 23:59:09 -08:00
committed by GitHub
parent ec4fb11aa5
commit 8f28264aec
2 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
embedchain/loaders/sitemap.py
View File

@@ -31,10 +31,13 @@ class SitemapLoader(BaseLoader):
def load_data(self, sitemap_source):
output = []
web_page_loader = WebPageLoader()
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", # noqa:E501
}
if urlparse(sitemap_source).scheme in ("http", "https"):
try:
response = requests.get(sitemap_source)
response = requests.get(sitemap_source, headers=headers)
response.raise_for_status()
soup = BeautifulSoup(response.text, "xml")
except requests.RequestException as e: