From a98cbc880c6e2a862ee1a1716fe522410f91ccd9 Mon Sep 17 00:00:00 2001 From: Pinapelz Date: Wed, 14 May 2025 15:29:14 -0700 Subject: wmmt: 6rr and 6rr+ scraper --- site_scraper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'site_scraper.py') diff --git a/site_scraper.py b/site_scraper.py index 9efa4b6..e9301b5 100644 --- a/site_scraper.py +++ b/site_scraper.py @@ -68,7 +68,7 @@ class SiteScraper: print("WebDriver closed successfully") -def download_site_as_html(url: str, timeout: int = 10) -> str: +def download_site_as_html(url: str, timeout: int = 10, response_encoding=None) -> str: headers = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " @@ -84,6 +84,8 @@ def download_site_as_html(url: str, timeout: int = 10) -> str: try: response = requests.get(url, headers=headers, timeout=timeout) + if response_encoding: + response.encoding = response_encoding response.raise_for_status() return response.text except requests.RequestException as e: -- cgit v1.2.3