diff options
| -rw-r--r-- | .gitignore | 174 | ||||
| -rw-r--r-- | bemani/iidx.py | 60 | ||||
| -rw-r--r-- | bemani/sdvx.py | 46 | ||||
| -rw-r--r-- | constants.py | 3 | ||||
| -rw-r--r-- | konami.py | 33 | ||||
| -rw-r--r-- | site_scraper.py | 58 |
6 files changed, 374 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a19790 --- /dev/null +++ b/.gitignore @@ -0,0 +1,174 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc diff --git a/bemani/iidx.py b/bemani/iidx.py new file mode 100644 index 0000000..e20dd7d --- /dev/null +++ b/bemani/iidx.py @@ -0,0 +1,60 @@ +from bs4 import BeautifulSoup +from datetime import datetime +from urllib.parse import urljoin +import re + + +def parse_pinky_crush_news_site(html: str, base_url): + type_map = { + "i_01": "NEWSONG", + "i_02": "RANKING", + "i_03": "EVENT", + "i_04": "SHOP", + "i_05": "OTHER" + } + soup = BeautifulSoup(html, "html.parser") + news_items = [] + + for li in soup.select("#info-news > li"): + date_elem = li.select_one(".news-main > li:nth-of-type(1)") + headline_elem = li.select_one(".news-main > li:nth-of-type(2)") + content_elem = li.select_one(".news-main > li:nth-of-type(3)") + type_class = li.get("class", [None])[0] + if not (date_elem and content_elem): + continue + date_str = date_elem.text.strip() + try: + dt = datetime.strptime(date_str, "%Y/%m/%d") + timestamp = int(dt.timestamp()) + except ValueError: + timestamp = None + + headline = headline_elem.a.text.strip() if headline_elem.a else headline_elem.text.strip() + + for a in content_elem.select("a[href]"): + href = urljoin(base_url, a["href"]) + text = a.get_text(strip=True) + a.replace_with(f"[{text}]({href})") + + for br in content_elem.find_all("br"): + br.replace_with("\n") + + content = content_elem.get_text().strip() + + content = content.replace( + " e-amusement ベーシックコース ", + " e-amusement ベーシックコース " + ) + content = content.replace("※", "\n※") + content = re.sub(r"\n[ \t]+", "\n", content) + content = re.sub(r'\s*/\s*', '/', content) + news_items.append({ + "date": date_str, + "type": type_map[type_class], + "timestamp": timestamp, + "headline": headline, + "content": content, + "images": [], + }) + + return news_items diff --git a/bemani/sdvx.py b/bemani/sdvx.py new file mode 100644 index 0000000..55d97ef --- /dev/null +++ b/bemani/sdvx.py @@ -0,0 +1,46 @@ +from bs4 import BeautifulSoup +from datetime import datetime +from urllib.parse import urljoin + +def parse_exceed_gear_news_site(html: str, base_url: str): + soup = BeautifulSoup(html, 'html.parser') + news_list = soup.select('.tab ul.news li') + + entries = [] + for li in news_list: + date = li.select_one('strong') + pre = li.select_one('pre') + + if not date or not pre: + continue + date_str = date.text.strip() + try: + dt = datetime.strptime(date_str, "%Y.%m.%d") + timestamp = int(dt.timestamp()) + except ValueError: + timestamp = None + headline = li.select_one('p.notice') + headline_text = headline.text.strip() if headline else None + for tag in pre.select('font, b, u, span'): + tag.unwrap() + content = pre.get_text(separator='\n', strip=True) + images = [] + for img in pre.select('img'): + src = img.get('data-original') or img.get('src') + if not src or src.startswith('data:'): + continue + src = urljoin(base_url, src) + parent = img.find_parent('a') + href = urljoin(base_url, parent['href']) if parent and parent.has_attr('href') else None + images.append({'image': src, 'link': href}) + + entries.append({ + 'date': date_str, + 'type': None, + 'timestamp': timestamp, + 'headline': headline_text, + 'content': content, + 'images': images + }) + + return entries diff --git a/constants.py b/constants.py new file mode 100644 index 0000000..f131a63 --- /dev/null +++ b/constants.py @@ -0,0 +1,3 @@ +EAMUSEMENT_BASE_URL = "https://p.eagate.573.jp" +SOUND_VOLTEX_EXCEED_GEAR_NEWS_SITE ="https://p.eagate.573.jp/game/sdvx/vi/news/index.html" +IIDX_PINKY_CRUSH_NEWS_SITE="https://p.eagate.573.jp/game/2dx/32/info/index.html" diff --git a/konami.py b/konami.py new file mode 100644 index 0000000..438b1ed --- /dev/null +++ b/konami.py @@ -0,0 +1,33 @@ +""" +Fetching data for Konami/Bemani games +{ + 'date': JST date of news post + 'type': Type of post if available, otherwise if not provided it will be None (aka Generic news) + 'timestamp': Unixtime of date above, + 'headline': Headline, + 'content': All text content of news, + 'images': { + 'image': URL to image, + 'link': If there's an associated href. Else None + + } +} +""" + +from email.utils import parsedate_to_datetime +from site_scraper import SiteScraper +import bemani.sdvx as sound_voltex +import bemani.iidx as iidx +import constants + +def get_news(news_url: str) -> list: + scraper = SiteScraper(headless=True) + site_data = scraper.get_page_source(news_url) + if news_url == constants.SOUND_VOLTEX_EXCEED_GEAR_NEWS_SITE: + news_posts = sorted(sound_voltex.parse_exceed_gear_news_site(site_data, constants.EAMUSEMENT_BASE_URL), key=lambda x: x['timestamp'], reverse=True) + elif news_url == constants.IIDX_PINKY_CRUSH_NEWS_SITE: + news_posts = sorted(iidx.parse_pinky_crush_news_site(site_data, constants.EAMUSEMENT_BASE_URL), key=lambda x: x['timestamp'], reverse=True) + else: + news_posts = [] + scraper.close() + return news_posts diff --git a/site_scraper.py b/site_scraper.py new file mode 100644 index 0000000..0a49c60 --- /dev/null +++ b/site_scraper.py @@ -0,0 +1,58 @@ +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.chrome.options import Options as ChromeOptions +from selenium import webdriver +from dotenv import load_dotenv +import time +import os + +load_dotenv() + +class SiteScraper: + def __init__(self, headless: bool = False, wait_time = 5): + """ + Initialize the SiteScraper with the path to ChromeDriver + :param chrome_driver_path: Path to the ChromeDriver executable + :param headless: Run the browser in headless mode if True + """ + self.wait_time = wait_time + try: + self.service = Service(os.environ.get("CHROME_DRIVER_PATH")) + self.chrome_options = ChromeOptions() + + if headless: + self.chrome_options.add_argument("--headless") + self.chrome_options.add_argument("--disable-gpu") + self.chrome_options.add_argument("--disable-dev-shm-usage") + self.chrome_options.add_argument("--window-size=1920,1080") + self.chrome_options.add_argument("--no-sandbox") + + self.driver = webdriver.Chrome(service=self.service, options=self.chrome_options) + except FileNotFoundError: + print("The ChromeDriver executable was not found. Is it installed and accessible in PATH?") + quit() + except Exception as e: + print(f"An unknown error occurred: {e}") + quit() + + def get_page_source(self, url) -> str: + """ + Get the page source of the given URL + :param url: The URL of the page to scrape + :param wait_time: The time to wait for the page to load (for JavaScript) + """ + try: + self.driver.get(url) + except Exception as e: + print(f"An error occurred while trying to get the page source: {e}") + return "" + if self.wait_time > 0: + time.sleep(self.wait_time) + return self.driver.page_source + + def close(self): + """ + Close the WebDriver + """ + self.driver.quit() + self.service.stop() + print("WebDriver closed successfully") |