diff options
Diffstat (limited to 'generate.py')
| -rw-r--r-- | generate.py | 154 |
1 files changed, 73 insertions, 81 deletions
diff --git a/generate.py b/generate.py index a136fa5..71093e5 100644 --- a/generate.py +++ b/generate.py @@ -1,118 +1,110 @@ """ Generates news JSON files +Generally you're expected to update the game versions manually +as for most games you only ever want the latest version (supported) of the game """ import news_feed as feed import requests import constants import json import os +import argparse from datetime import datetime, timedelta OUTPUT_DIR = "news" -def create_merged_feed(*news_lists): +def create_merged_feed(*news_lists, limit=constants.DAYS_LIMIT): + """ + Merge multiple news feeds into a singular one + limit = maximum number of days old to be included in the merged feed + """ merged_feed = [] for news_list in news_lists: merged_feed.extend(news_list) - cutoff_date = datetime.now() - timedelta(days=constants.DAYS_LIMIT) + cutoff_date = datetime.now() - timedelta(days=limit) filtered_feed = [news for news in merged_feed if datetime.fromtimestamp(news['timestamp']) >= cutoff_date] sorted_feed = sorted(filtered_feed, key=lambda x: x['timestamp'], reverse=True) return sorted_feed -def fetch_json_from_url(url): - response = requests.get(url) - response.raise_for_status() - return response.json() - def attach_news_meta_data(news_data: list): + """ + Attaches additional metadata to news data files + Currently this is only the time of completion + """ return { "fetch_time": int(datetime.now().timestamp()), "news_posts": news_data } -if __name__ == "__main__": - if not os.path.exists(OUTPUT_DIR): - os.makedirs(OUTPUT_DIR) +def log_output(message: str, type: str="DEBUG"): + """ + Prints a log line output with a timestamp + """ + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print(f"[{timestamp}] [{type}]: {message}") + +def generate_news_file(filename, url, version=None): + log_output(f"Fetching {filename.upper()} News Data", "NEWS") + news_data = feed.get_news(url, version) if version else feed.get_news(url) + if len(news_data) != 0: + log_output(f"Success. Got {filename.upper()} News Data. Saving to file...", "NEWS") + with open(f"{OUTPUT_DIR}/{filename}.json", 'w') as json_file: + json.dump(attach_news_meta_data(news_data), json_file) + elif os.path.exists(f"{OUTPUT_DIR}/{filename}.json"): + print(f"Failed. Couldn't fetch {filename.upper()} data. Using previously scraped data", "NEWS") + with open(f"{OUTPUT_DIR}/{filename}.json", 'r') as json_file: + news_data = json.load(json_file)['news_posts'] + else: + print(f"Failed. Couldn't fetch {filename.upper()} data. Skipping...", "NEWS") + return news_data - print("Fetching IIDX Data") - iidx_news_data = feed.get_news(constants.IIDX_PINKY_CRUSH_NEWS_SITE) - if len(iidx_news_data) != 0: - with open(OUTPUT_DIR+'/iidx_news.json', 'w') as json_file: - json.dump(attach_news_meta_data(iidx_news_data), json_file) - print("IIDX Data fetched and saved.") - elif len(iidx_news_data) == 0 and os.path.exists(OUTPUT_DIR+'/iidx_news.json'): - with open(OUTPUT_DIR+'/iidx_news.json', 'r') as json_file: - iidx_news_data = json.load(json_file)['news_posts'] - print("IIDX Data not fetched, using existing data.") +def generate_iidx_news_file(): + return generate_news_file("iidx_news", constants.IIDX_PINKY_CRUSH_NEWS_SITE) - print("Fetching SDVX Data") - sdvx_news_data = feed.get_news(constants.SOUND_VOLTEX_EXCEED_GEAR_NEWS_SITE) - if len(sdvx_news_data) != 0: - with open(OUTPUT_DIR+'/sdvx_news.json', 'w') as json_file: - json.dump(attach_news_meta_data(sdvx_news_data), json_file) - print("SDVX Data fetched and saved.") - elif len(sdvx_news_data) == 0 and os.path.exists(OUTPUT_DIR+'/sdvx_news.json'): - with open(OUTPUT_DIR+'/sdvx_news.json', 'r') as json_file: - sdvx_news_data = json.load(json_file)['news_posts'] - print("SDVX Data not fetched, using existing data.") +def generate_sdvx_news_file(): + return generate_news_file("sdvx_news", constants.SOUND_VOLTEX_EXCEED_GEAR_NEWS_SITE) - print("Fetching CHUNITHM JPN Data") - chunithm_jp_news_data = feed.get_news(constants.CHUNITHM_JP_NEWS_SITE, constants.CHUNITHM_VERSION.VERSE) - if len(chunithm_jp_news_data) != 0: - with open(OUTPUT_DIR+'/chunithm_jp_news.json', 'w') as json_file: - json.dump(attach_news_meta_data(chunithm_jp_news_data), json_file) - print("CHUNITHM JPN Data fetched and saved.") - elif len(chunithm_jp_news_data) == 0 and os.path.exists(OUTPUT_DIR+'/chunithm_jp_news.json'): - with open(OUTPUT_DIR+'/chunithm_jp_news.json', 'r') as json_file: - chunithm_jp_news_data = json.load(json_file)['news_posts'] - print("CHUNITHM JPN Data not fetched, using existing data.") +def generate_chunithm_jp_news_file(): + return generate_news_file("chunithm_jp_news", constants.CHUNITHM_JP_NEWS_SITE, constants.CHUNITHM_VERSION.VERSE) - print("Fetching MAIMAI DX JPN Data") - maimaidx_jp_news_data = feed.get_news(constants.MAIMAIDX_JP_NEWS_SITE, constants.MAIMAIDX_VERSION.PRISM_PLUS) - if len(maimaidx_jp_news_data) != 0: - with open(OUTPUT_DIR+'/maimaidx_jp_news.json', 'w') as json_file: - json.dump(attach_news_meta_data(maimaidx_jp_news_data), json_file) - print("MAIMAI DX JPN Data fetched and saved.") - elif len(maimaidx_jp_news_data) == 0 and os.path.exists(OUTPUT_DIR+'/maimaidx_jp_news.json'): - with open(OUTPUT_DIR+'/maimaidx_jp_news.json', 'r') as json_file: - maimaidx_jp_news_data = json.load(json_file)['news_posts'] - print("MAIMAI DX JPN Data not fetched, using existing data.") +def generate_maimaidx_jp_news_file(): + return generate_news_file("maimaidx_jp_news", constants.MAIMAIDX_JP_NEWS_SITE, constants.MAIMAIDX_VERSION.PRISM_PLUS) - print("Fetching ONGEKI JPN Data") - ongeki_jp_news_data = feed.get_news(constants.ONGEKI_JP_NEWS_SITE, constants.ONGEKI_VERSION.REFRESH) - if len(ongeki_jp_news_data) != 0: - with open(OUTPUT_DIR+'/ongeki_jp_news.json', 'w') as json_file: - json.dump(attach_news_meta_data(ongeki_jp_news_data), json_file) - print("ONGEKI JPN Data fetched and saved.") - elif len(ongeki_jp_news_data) == 0 and os.path.exists(OUTPUT_DIR+'/ongeki_jp_news.json'): - with open(OUTPUT_DIR+'/ongeki_jp_news.json', 'r') as json_file: - ongeki_jp_news_data = json.load(json_file)['news_posts'] - print("ONGEKI JPN Data not fetched, using existing data.") +def generate_ongeki_jp_news_file(): + return generate_news_file("ongeki_jp_news", constants.ONGEKI_JP_NEWS_SITE, constants.ONGEKI_VERSION.REFRESH) - print("Fetching MAIMAIDX INTL Data") - maimaidx_intl_news_data = feed.get_news(constants.MAIMAIDX_INTL_NEWS_SITE, constants.MAIMAIDX_VERSION.PRISM) - if len(maimaidx_intl_news_data) != 0: - with open(OUTPUT_DIR+'/maimaidx_intl_news.json', 'w') as json_file: - json.dump(attach_news_meta_data(maimaidx_intl_news_data), json_file) - print("MAIMAIDX INTLN Data fetched and saved.") - elif len(maimaidx_intl_news_data) == 0 and os.path.exists(OUTPUT_DIR+'/maimaidx_intl_news.json'): - with open(OUTPUT_DIR+'/maimaidx_intl_news.json', 'r') as json_file: - maimaidx_intl_news_data = json.load(json_file)['news_posts'] - print("MAIMAIDX INTL Data not fetched, using existing data.") +def generate_maimaidx_intl_news_file(): + return generate_news_file("maimaidx_intl_news", constants.MAIMAIDX_INTL_NEWS_SITE, constants.MAIMAIDX_VERSION.PRISM) + +def generate_chunithm_intl_news_file(): + return generate_news_file("chunithm_intl_news", constants.CHUNITHM_INTL_NEWS_SITE, constants.CHUNITHM_VERSION.LUMINOUS_PLUS) + +if __name__ == "__main__": + log_output("JOB START", "TASK") + if not os.path.exists(OUTPUT_DIR): + log_output(f"{OUTPUT_DIR} was not found. Creating this directory...") + os.makedirs(OUTPUT_DIR) - print("Fetching CHUNITHM INTL Data") - chunithm_intl_news_data = feed.get_news(constants.CHUNITHM_INTL_NEWS_SITE, constants.CHUNITHM_VERSION.LUMINOUS_PLUS) - if len(chunithm_intl_news_data) != 0: - with open(OUTPUT_DIR+'/chunithm_intl_news.json', 'w') as json_file: - json.dump(attach_news_meta_data(chunithm_intl_news_data), json_file) - print("CHUNITHM INTL Data fetched and saved.") - elif len(chunithm_intl_news_data) == 0 and os.path.exists(OUTPUT_DIR+'/chunithm_intl_news.json'): - with open(OUTPUT_DIR+'/chunithm_intl_news.json', 'r') as json_file: - chunithm_intl_news_data = json.load(json_file)['news_posts'] - print("CHUNITHM INTL Data not fetched, using existing data.") + iidx_news_data = generate_iidx_news_file() + sdvx_news_data = generate_sdvx_news_file() + chunithm_jp_news_data = generate_chunithm_jp_news_file() + maimaidx_jp_news_data = generate_maimaidx_jp_news_file() + ongeki_jp_news_data = generate_ongeki_jp_news_file() + maimaidx_intl_news_data = generate_maimaidx_intl_news_file() + chunithm_intl_news_data = generate_chunithm_intl_news_file() - news = create_merged_feed(iidx_news_data, sdvx_news_data, chunithm_jp_news_data, maimaidx_jp_news_data, ongeki_jp_news_data, maimaidx_intl_news_data, chunithm_intl_news_data) + news = create_merged_feed( + iidx_news_data, + sdvx_news_data, + chunithm_jp_news_data, + maimaidx_jp_news_data, + ongeki_jp_news_data, + maimaidx_intl_news_data, + chunithm_intl_news_data + ) + log_output("Creating merged news.json file for all news that are within " + str(constants.DAYS_LIMIT) + " days old") with open(OUTPUT_DIR+'/news.json', 'w') as json_file: json.dump(attach_news_meta_data(news), json_file) + log_output("JOB DONE", "TASK") |