diff options
Diffstat (limited to 'site_scraper.py')
| -rw-r--r-- | site_scraper.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/site_scraper.py b/site_scraper.py index 9efa4b6..e9301b5 100644 --- a/site_scraper.py +++ b/site_scraper.py @@ -68,7 +68,7 @@ class SiteScraper: print("WebDriver closed successfully") -def download_site_as_html(url: str, timeout: int = 10) -> str: +def download_site_as_html(url: str, timeout: int = 10, response_encoding=None) -> str: headers = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " @@ -84,6 +84,8 @@ def download_site_as_html(url: str, timeout: int = 10) -> str: try: response = requests.get(url, headers=headers, timeout=timeout) + if response_encoding: + response.encoding = response_encoding response.raise_for_status() return response.text except requests.RequestException as e: |