diff options
| -rw-r--r-- | constants.py | 4 | ||||
| -rw-r--r-- | generate.py | 6 | ||||
| -rw-r--r-- | news_feed.py | 4 | ||||
| -rw-r--r-- | sega/ongeki_jp.py | 48 |
4 files changed, 62 insertions, 0 deletions
diff --git a/constants.py b/constants.py index f1ff58c..4fb59f9 100644 --- a/constants.py +++ b/constants.py @@ -8,6 +8,7 @@ IIDX_PINKY_CRUSH_NEWS_SITE="https://p.eagate.573.jp/game/2dx/32/info/index.html" CHUNITHM_JP_NEWS_SITE="https://info-chunithm.sega.jp/" MAIMAIDX_JP_NEWS_SITE="https://info-maimai.sega.jp/" +ONGEKI_JP_NEWS_SITE="https://info-ongeki.sega.jp/" class CHUNITHM_VERSION(Enum): VERSE = 1 @@ -15,3 +16,6 @@ class CHUNITHM_VERSION(Enum): class MAIMAIDX_VERSION(Enum): PRISM = 1 PRISM_PLUS = 2 + +class ONGEKI_VERSION(Enum): + REFRESH = 1 diff --git a/generate.py b/generate.py index 08fd615..6395542 100644 --- a/generate.py +++ b/generate.py @@ -54,6 +54,12 @@ if __name__ == "__main__": with open(OUTPUT_DIR+'/maimaidx_jp_news.json', 'w') as json_file: json.dump(attach_news_meta_data(maimaidx_jp_news_data), json_file) + print("Fetching ONGEKI JPN Data") + ongeki_jp_news_data = feed.get_news(constants.ONGEKI_JP_NEWS_SITE, constants.ONGEKI_VERSION.REFRESH) + if len(ongeki_jp_news_data) != 0: + with open(OUTPUT_DIR+'/ongeki_jp_news.json', 'w') as json_file: + json.dump(attach_news_meta_data(ongeki_jp_news_data), json_file) + news = create_merged_feed(iidx_news_data, sdvx_news_data, chunithm_jp_news_data, maimaidx_jp_news_data) with open(OUTPUT_DIR+'/news.json', 'w') as json_file: json.dump(attach_news_meta_data(news), json_file) diff --git a/news_feed.py b/news_feed.py index 01ee3d1..d796d64 100644 --- a/news_feed.py +++ b/news_feed.py @@ -22,6 +22,7 @@ import bemani.sdvx as sound_voltex import bemani.iidx as iidx import sega.chuni_jp as chunithm_jp import sega.maimaidx_jp as maimaidx_jp +import sega.ongeki_jp as ongeki_jp import constants def get_news(news_url: str, version=None) -> list: @@ -37,6 +38,9 @@ def get_news(news_url: str, version=None) -> list: elif news_url == constants.MAIMAIDX_JP_NEWS_SITE: if version == constants.MAIMAIDX_VERSION.PRISM_PLUS: news_posts = sorted(maimaidx_jp.parse_maimaidx_jp_prism_plus_news_site(site_data), key=lambda x: x['timestamp'], reverse=True) + elif news_url == constants.ONGEKI_JP_NEWS_SITE: + if version == constants.ONGEKI_VERSION.REFRESH: + news_posts = sorted(ongeki_jp.parse_ongeki_refresh_news_site(site_data), key=lambda x: x['timestamp'], reverse=True) else: news_posts = [] scraper.close() diff --git a/sega/ongeki_jp.py b/sega/ongeki_jp.py new file mode 100644 index 0000000..587f358 --- /dev/null +++ b/sega/ongeki_jp.py @@ -0,0 +1,48 @@ +from bs4 import BeautifulSoup +from datetime import datetime +import time + +def parse_ongeki_refresh_news_site(html: str): + soup = BeautifulSoup(html, "html.parser") + items = [] + + for li in soup.select("li.p-news__listChild"): + a_tag = li.select_one("a.p-news__listLink") + url = a_tag["href"] if a_tag else None + + img_tag = li.select_one(".p-news__listThumb img") + image_url = img_tag["src"] if img_tag else None + image_alt = img_tag["alt"] if img_tag else "" + image_link = url if image_url else None + + date_type_text = li.select_one(".p-news__listTextUpper") + date_text = date_type_text.text.strip().split("/")[0].strip() if date_type_text else None + type_text = date_type_text.text.strip().split("/")[-1].strip() if "/" in date_type_text.text else None + + headline_tag = li.select_one(".p-news__listTextUnder") + headline = headline_tag.text.strip() if headline_tag else None + + timestamp = None + if date_text: + try: + dt = datetime.strptime(date_text, "%Y.%m.%d %a") + timestamp = int(time.mktime(dt.timetuple())) + except: + timestamp = None + entry = { + "date": date_text, + "identifier": "ONGEKI_JPN_REFRESH", + "type": type_text if type_text not in ["GAME", "CARDMAKER"] else None, + "timestamp": timestamp, + "headline": None, + "content": image_alt, + "url": url, + "images": [{ + "image": image_url, + "link": image_link + }] if image_url else [] + } + + items.append(entry) + + return items |