diff options
| -rw-r--r-- | constants.py | 4 | ||||
| -rw-r--r-- | generate.py | 6 | ||||
| -rw-r--r-- | news_feed.py | 21 | ||||
| -rw-r--r-- | sega/chuni_intl.py | 31 | ||||
| -rw-r--r-- | sega/maimaidx_intl.py | 3 |
5 files changed, 50 insertions, 15 deletions
diff --git a/constants.py b/constants.py index 7bdf630..f31a0eb 100644 --- a/constants.py +++ b/constants.py @@ -21,9 +21,11 @@ DANCE_AROUND_APP_ID="kmhqpindcodm0mkh" DANCE_RUSH_APP_ID="ns3maqirvf08ddhp" CHUNITHM_JP_NEWS_SITE="https://info-chunithm.sega.jp/" -CHUNITHM_INTL_NEWS_SITE="https://info-chunithm.sega.com/" +CHUNITHM_INTL_NEWS_SITE="https://info-chunithm.sega.com/wp-json/thistheme/v1/articlesRest" +CHUNITHM_INTL_RECENT_NEWS_LIMIT = 15 # gets the last 15 news posts MAIMAIDX_JP_NEWS_SITE="https://info-maimai.sega.jp/" MAIMAIDX_INTL_NEWS_SITE="https://maimai.sega.com/assets/data/index.json" +MAIMAIDX_INTL_RECENT_NEWS_LIMIT = 15 # gets the last 15 news posts ONGEKI_JP_NEWS_SITE="https://info-ongeki.sega.jp/" IDAC_NEWS_SITE="https://info-initialdac.sega.jp/" diff --git a/generate.py b/generate.py index 83fce32..4fd5735 100644 --- a/generate.py +++ b/generate.py @@ -147,7 +147,7 @@ def generate_news_file(filename, url, version=None, formatted_name: str = None): # by the e-amusement app. Information is different def generate_iidx_news_file(eamuse_feed: bool=False): if eamuse_feed: - news = generate_news_file("iidx_news", constants.EAMUSE_APP_FEED, constants.IIDX_EAMUSE_APP_ID) + news = generate_news_file("iidx_news", constants.EAMUSE_APP_API_ROUTE, constants.IIDX_EAMUSE_APP_ID) else: # legacy should not be used, use eamuse app feed above news = generate_news_file("iidx_news", constants.IIDX_PINKY_CRUSH_NEWS_SITE) attempt_broadcast_notifications(news, "New information for beatmania IIDX", "iidx") @@ -217,12 +217,12 @@ def generate_ongeki_jp_news_file(): return news def generate_maimaidx_intl_news_file(): - news = generate_news_file("maimaidx_intl_news", constants.MAIMAIDX_INTL_NEWS_SITE, constants.MAIMAIDX_VERSION.PRISM_PLUS) + news = generate_news_file("maimaidx_intl_news", constants.MAIMAIDX_INTL_NEWS_SITE) attempt_broadcast_notifications(news, "New information for maimai DX (International ver.)", "maimaidx_intl") return news def generate_chunithm_intl_news_file(): - news = generate_news_file("chunithm_intl_news", constants.CHUNITHM_INTL_NEWS_SITE, constants.CHUNITHM_VERSION.VERSE) + news = generate_news_file("chunithm_intl_news", constants.CHUNITHM_INTL_NEWS_SITE) attempt_broadcast_notifications(news, "New information for CHUNITHM (International ver.)", "chunithm_intl") return news diff --git a/news_feed.py b/news_feed.py index 2761f29..05378ca 100644 --- a/news_feed.py +++ b/news_feed.py @@ -131,15 +131,16 @@ def get_news(news_url: str, version=None) -> list: elif news_url == constants.CHUNITHM_INTL_NEWS_SITE: site_data = download_site_as_html(news_url) - if version in [constants.CHUNITHM_VERSION.LUMINOUS_PLUS, constants.CHUNITHM_VERSION.VERSE]: - news_posts = sorted(chuni_intl.parse_chuni_intl_news_site(site_data), key=lambda x: x['timestamp'], reverse=True) - if constants.CHUNI_RECURSIVE_IMAGE: - for i in range(len(news_posts)): - if not news_posts[i]["url"]: - continue - post_site_data = download_site_as_html(news_posts[i]["url"]) - post_images = chuni_intl.parse_chuni_intl_post_images(post_site_data) - news_posts[i]["images"].extend([image for image in post_images if not any(existing_image['image'] == image['image'] for existing_image in news_posts[i]["images"])]) + news_posts = sorted(chuni_intl.parse_chuni_intl_api_route(site_data, "CHUNITHM_INTL", constants.CHUNITHM_INTL_RECENT_NEWS_LIMIT), key=lambda x: x['timestamp'], reverse=True) + print(news_posts) + exit() + if constants.CHUNI_RECURSIVE_IMAGE: + for i in range(len(news_posts)): + if not news_posts[i]["url"]: + continue + post_site_data = download_site_as_html(news_posts[i]["url"]) + post_images = chuni_intl.parse_chuni_intl_post_images(post_site_data) + news_posts[i]["images"].extend([image for image in post_images if not any(existing_image['image'] == image['image'] for existing_image in news_posts[i]["images"])]) elif news_url == constants.MAIMAIDX_JP_NEWS_SITE: @@ -150,7 +151,7 @@ def get_news(news_url: str, version=None) -> list: elif news_url == constants.MAIMAIDX_INTL_NEWS_SITE: site_data = download_site_as_html(news_url) - news_posts = sorted(maimaidx_intl.parse_maimaidx_intl_api_route(site_data, "MAIMAIDX_INTL"), key=lambda x: x['timestamp'], reverse=True) + news_posts = sorted(maimaidx_intl.parse_maimaidx_intl_api_route(site_data, "MAIMAIDX_INTL", constants.MAIMAIDX_INTL_RECENT_NEWS_LIMIT), key=lambda x: x['timestamp'], reverse=True) _attach_llm_summaries(news_posts, "maimai DX International") elif news_url == constants.ONGEKI_JP_NEWS_SITE: diff --git a/sega/chuni_intl.py b/sega/chuni_intl.py index a6c4b5f..ea8cd7f 100644 --- a/sega/chuni_intl.py +++ b/sega/chuni_intl.py @@ -1,6 +1,7 @@ import re from datetime import datetime, timedelta, timezone from enum import Enum +import json from urllib.parse import urljoin from bs4 import BeautifulSoup @@ -94,6 +95,36 @@ def make_image_extractor(version: ParserVersion): else: raise ValueError("Unknown Parser Version") +def parse_chuni_intl_api_route(raw_api_data: str, identifier: str, limit: int): + route_data = json.loads(raw_api_data) + route_data = route_data[:limit] + entries = [] + for post_data in route_data: + date_str = post_data["date"] + dt = datetime.strptime(date_str, "%Y.%m.%d").replace(tzinfo=timezone(timedelta(hours=9))) + timestamp = int(dt.timestamp()) + full_image_url = post_data["thumbnail"] + content = post_data["desc"] + headline = post_data["title"] + url = post_data["permalink"] + images = [{ + "image": full_image_url, + "link": None + }] + entry = { + "date": date_str, + "identifier": identifier, + "type": None, + "timestamp": timestamp, + "headline": headline, + "content": content, + "url": url, + "images": images, + "is_ai_summary": False + } + entries.append(entry) + return entries + parse_chuni_intl_news_site = make_chuni_intl_parser( "CHUNITHM_INTL", ParserVersion.ALPHA diff --git a/sega/maimaidx_intl.py b/sega/maimaidx_intl.py index 073a211..34eaa43 100644 --- a/sega/maimaidx_intl.py +++ b/sega/maimaidx_intl.py @@ -49,8 +49,9 @@ def make_maimaidx_intl_parser(identifier: str, parser: ParserVersion): if parser == ParserVersion.ALPHA: return alpha_parser -def parse_maimaidx_intl_api_route(raw_api_data: str, identifier: str): +def parse_maimaidx_intl_api_route(raw_api_data: str, identifier: str, limit: int): route_data = json.loads(raw_api_data) + route_data = route_data[:limit] entries = [] for post_data in route_data: date_data = post_data["date"] |