initial wmmt NA 5dx+ scraper

4 files changed, 123 insertions, 0 deletions

diff --git a/README.md b/README.md
index 6af9982..cbc68b7 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,8 @@ Currently Supported:
 - jubeat (`jubeat_news`)
 - GITADORA (`gitadora_news`)
 - NOSTALGIA (`nostalgia_news`)
+- DanceRush (`dance_rush_news`)
+- DANCE aROUND (`dance_around_news`)
 - CHUNITHM (JPN) (`chunithm_jpn_news`)
 - CHUNITHM (INTL) (`chunithm_intl_news`)
 - maimai DX (JPN) (`maimaidx_jp_news`)
diff --git a/bandai_namco/wmmt.py b/bandai_namco/wmmt.py
new file mode 100644
index 0000000..86a2ce4
--- /dev/null
+++ b/bandai_namco/wmmt.py
@@ -0,0 +1,96 @@
+import re
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+from urllib.parse import urljoin
+import sys
+import os
+import pytz
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../")))
+import constants
+from bs4 import BeautifulSoup
+
+BASE_URL = "https://wanganmaxi-official.com"
+
+TYPE_MAP = {
+    "Online Events Information": "EVENTS",
+    "Update Information": "UPDATE",
+    "Future Lab News": "FUTURE LAB",
+    "Special Contents": "SPECIAL"
+}
+
+def make_wmmt_parser(version: constants.WANGAN_MAXI_VERSION):
+    def five_dx_plus_parser(html: str):
+        soup = BeautifulSoup(html, "html.parser")
+        results = []
+        for section in soup.select("div.parts_column_02 > div.parts_bg_01"):
+            type_heading = section.select_one("section h2.parts_txt_01")
+            type_name = type_heading.get_text(strip=True) if type_heading else None
+            count = 0
+            for a in section.select("ul.archiveNav a[href]"):
+                if count >= constants.WANGAN_MAXI_POSTS_PER_SECTION:
+                    break
+                href = a["href"]
+                title_tag = a.find("h4")
+                date_tag = a.find("p")
+                title_parts = []
+                for child in title_tag.children:
+                    if child.name == "span":
+                        title_parts.append(f"[{child.get_text(strip=True)}]")
+                    elif isinstance(child, str):
+                        title_parts.append(child.strip())
+                title = " ".join(title_parts).strip()
+                date = date_tag.get_text(strip=True) if date_tag else "No date"
+                url = urljoin(BASE_URL, href)
+                url = url.replace(".php", ".html")
+                results.append({
+                    "url": url,
+                    "title": title,
+                    "date": date,
+                    "type": TYPE_MAP[type_name]
+                })
+                count += 1
+        return results
+    if version == constants.WANGAN_MAXI_VERSION.FIVE_DX_PLUS:
+        return five_dx_plus_parser
+
+
+def make_wmmt_news_extractor(identifier: str, version: constants.WANGAN_MAXI_VERSION, internal_path: str):
+    def five_dx_plus_extractor(html: str, data: dict):
+        image_base = BASE_URL + "/" + internal_path
+        soup = BeautifulSoup(html, "html.parser")
+        container = soup.select_one(".parts_inner_01")
+        if not container:
+            return None
+        date_str = data["date"]
+        timestamp = int(datetime.strptime(date_str, "%Y/%m/%d").replace(tzinfo=timezone.utc).timestamp())
+        first_p = container.find("p")
+        content = first_p.get_text(" ", strip=True) if first_p else ""
+        images = []
+        for img in container.find_all("img"):
+            src = img.get("src").replace("./","")
+            if data["type"] == "EVENTS":
+                src = "event/online/" + src
+            elif data["type"] == "SPECIAL":
+                src =  "special/" + src
+            elif data["type"] == "FUTURE LAB":
+                src =  "miraiken/" + src
+            elif data["type"] == "UPDATE":
+                src = "update/" + src
+            img_url = image_base + "/" + src if src else None
+            parent = img.find_parent("a")
+            images.append({
+                "image": img_url,
+                "link": urljoin(BASE_URL, parent.get("href")) if parent and parent.get("href") else None
+            })
+        data["identifier"] = identifier
+        data["timestamp"] = timestamp
+        data["content"] = content
+        data["images"] = images
+        data["is_ai_summary"] = False
+        return data
+
+    if version == constants.WANGAN_MAXI_VERSION.FIVE_DX_PLUS:
+        return five_dx_plus_extractor
+
+get_wmmt_na_news_post_links = make_wmmt_parser(constants.WANGAN_MAXI_VERSION.FIVE_DX_PLUS)
+parse_wmmt_na_news = make_wmmt_news_extractor("WANGAN_MAXI_NA", constants.WANGAN_MAXI_VERSION.FIVE_DX_PLUS, "wanganmaxi5dxplus/na")
diff --git a/constants.py b/constants.py
index e990f87..e6b7e57 100644
--- a/constants.py
+++ b/constants.py
@@ -27,6 +27,12 @@ ONGEKI_JP_NEWS_SITE="https://info-ongeki.sega.jp/"
 MUSIC_DIVER_NEWS="https://mypage.musicdiver.jp/api/news?lang=en"
 
 TAIKO_BLOG_SITE="https://taiko-ch.net/blog/"
+WANGAN_MAXI_GENERIC="https://wanganmaxi-official.com/"
+WANGAN_MAXI_NA_NEWS_SITE="https://wanganmaxi-official.com/wanganmaxi5dxplus/na/archive"
+WANGAN_MAXI_ASIA_OCE_NEWS_SITE="https://wanganmaxi-official.com/wanganmaxi6rr/en/archive/"
+WANGAN_MAXI_JP_NEWS_SITE="https://wanganmaxi-official.com/wanganmaxi6rrplus/jp/archive/"
+WANGAN_MAXI_POSTS_PER_SECTION=3
+# due to how dead the NA version is, these will be merged into a singular feed
 
 ADD_EN_TRANSLATION=True # Only takes effect if an API key is provided in .env
 CHUNI_RECURSIVE_IMAGE=True # Scrape the individual post pages and get all images there
@@ -45,3 +51,10 @@ class MAIMAIDX_VERSION(Enum):
 
 class ONGEKI_VERSION(Enum):
     REFRESH = 1
+
+class WANGAN_MAXI_VERSION(Enum):
+    FIVE_DX_PLUS = 1,
+    SIX = 2,
+    SIX_R = 3,
+    SIX_RR = 4,
+    SIX_RR_PLUS = 5
diff --git a/news_feed.py b/news_feed.py
index 0a49707..d621984 100644
--- a/news_feed.py
+++ b/news_feed.py
@@ -32,6 +32,7 @@ import sega.maimaidx_intl as maimaidx_intl
 import sega.ongeki_jp as ongeki_jp
 import taito.music_diver as music_diver
 import bandai_namco.taiko as taiko
+import bandai_namco.wmmt as wmmt
 import community.disc as disc
 import community.wacca_plus.wacca_plus as wac_plus
 import community.museca_plus as mus_plus
@@ -166,6 +167,17 @@ def get_news(news_url: str, version=None) -> list:
         news_posts = sorted(taiko.parse_taiko_blog_site(site_data), key=lambda x: x['timestamp'], reverse=True)
         news_posts = translate.add_translate_text_to_en(news_posts)
 
+    elif news_url == constants.WANGAN_MAXI_GENERIC:
+        news_posts = []
+        na_site_data = download_site_as_html(constants.WANGAN_MAXI_NA_NEWS_SITE)
+        prelim_na_news_data = wmmt.get_wmmt_na_news_post_links(na_site_data)
+        for data in prelim_na_news_data:
+            post_site_data = download_site_as_html(data["url"])
+            news_posts.append(wmmt.parse_wmmt_na_news(post_site_data, data))
+        print(news_posts)
+        exit()
+
+
     elif news_url == constants.WACCA_PLUS_MAGIC_STRING:
         if not wac_plus.check_is_generation_possible():
             news_posts = []