diff options
| -rw-r--r-- | news_feed.py | 1 | ||||
| -rw-r--r-- | site_scraper.py | 5 |
2 files changed, 5 insertions, 1 deletions
diff --git a/news_feed.py b/news_feed.py index 2131676..7a57ac1 100644 --- a/news_feed.py +++ b/news_feed.py @@ -74,6 +74,7 @@ def get_news(news_url: str, version=None) -> list: elif news_url == constants.POLARIS_CHORD_NEWS_SITE: scraper = SiteScraper(headless=True) site_data = scraper.get_page_source(news_url) + scraper.close() news_posts = sorted(polaris_chord.parse_polaris_chord_news_site(site_data), key=lambda x: x['timestamp'], reverse=True) news_posts = translate.add_translate_text_to_en(news_posts, iidx.KEY_TERMS_TL) diff --git a/site_scraper.py b/site_scraper.py index e9301b5..2db7c31 100644 --- a/site_scraper.py +++ b/site_scraper.py @@ -88,6 +88,9 @@ def download_site_as_html(url: str, timeout: int = 10, response_encoding=None) - response.encoding = response_encoding response.raise_for_status() return response.text + except requests.exceptions.Timeout as e: + print(f"Timeout error downloading {url}: {e}") + return None except requests.RequestException as e: print(f"Error downloading {url}: {e}") - return "" + return None |