diff options
| -rw-r--r-- | usc/seeds/nautica/README.md | 31 | ||||
| -rw-r--r-- | usc/seeds/nautica/add-usc-nautica.js | 143 | ||||
| -rw-r--r-- | usc/seeds/nautica/maps-nov-10-2025.db | bin | 0 -> 1961984 bytes | |||
| -rw-r--r-- | usc/seeds/nautica/nautica_to_maps.py | 111 |
4 files changed, 279 insertions, 6 deletions
diff --git a/usc/seeds/nautica/README.md b/usc/seeds/nautica/README.md index e69de29..721b5a9 100644 --- a/usc/seeds/nautica/README.md +++ b/usc/seeds/nautica/README.md @@ -0,0 +1,31 @@ +# Nautica (ksm.dev) to maps.db +This is a "seed" generation helper for Tachi. It generates the `maps.db` file which can be used to generate a new seed file. The instructions here expect that you are already somewhat understand how Tachi works (since you're persumably running your own instance) + +i.e https://github.com/zkrising/Tachi/blob/main/seeds/scripts/rerunners/usc/add-usc-converts.js + +This pulls **ALL** charts from Nautica and produces the minimal requirements for seed generation in Tachi. Please run responsibly since this requires downloading all charts and producing a `SHA-1` hash for them. + + +```bash +python nautica_to_maps.py --start-page <START_PAGE> --db <PATH_TO_DB> +``` +- START_PAGE default = 1 +- PATH_TO_DB default = maps.db + + +As an alternative, I provide the one I have generated on Nov.10 2025 (`maps-nov-10-2025.db`) + +# Merging into Tachi +Firstly, move the generated `maps.db` into Tachi's environment + +Tachi relies on fuzzy matching title and artist which doesn't work great for nautica since there are many people who re-chart songs. In these cases they will have the same name. + +To work around this, a custom script can be used to finally convert `maps.db` into seeds. This code is largely based off of the usc-converts re-runner. + +If there is a collision in `songs-usc.json` where the name and artist of a song already exists, the string `(<EFFECTOR_NAME> Nautica Edition)` is appended to the title. + +To run this script, place it in `seeds/scripts/single-use/usc` (you need to create the usc folder if it doesn't already exist in single-use). Then within Tachi's environment run `node add-usc-nautica.js -f /nautica -d maps.db`. + +Then just load the seeds and you're good to go! + +*It may still TECHNICALLY be possible to have collisions, but the chances of 2 effectors with the same name who create charts for the same song are unlikely. In those cases you should manually resolve the difference after running the script.* diff --git a/usc/seeds/nautica/add-usc-nautica.js b/usc/seeds/nautica/add-usc-nautica.js new file mode 100644 index 0000000..d1a359e --- /dev/null +++ b/usc/seeds/nautica/add-usc-nautica.js @@ -0,0 +1,143 @@ +const { Command } = require("commander"); +const sqlite3 = require("better-sqlite3"); +const fs = require("fs"); +const { + CreateChartID, + ReadCollection, + WriteCollection, + GetFreshSongIDGenerator, +} = require("../../util"); + +const program = new Command(); +program.requiredOption("-d, --db <maps.db>"); +program.requiredOption("-f, --filter <path_filter>"); +program.option("--debug", "Show debug logs", false); +program.parse(process.argv); +const options = program.opts(); + +const DEBUG = options.debug; +const db = sqlite3(options.db); +const dbRows = db.prepare(`SELECT * FROM Charts WHERE path LIKE '%${options.filter}%'`).all(); +console.log(`Found ${dbRows.length} charts.`); + +const songs = ReadCollection("songs-usc.json"); +const charts = ReadCollection("charts-usc.json"); +const folderIdToSongId = {}; + +let newSongs = 0; +let newCharts = 0; + +const getFreshSongID = GetFreshSongIDGenerator("usc"); +const log = DEBUG ? console.log : () => undefined; + +function makeUscEditionName(title, artist) { + return `${title} (${artist} Nautica Edition)`; +} + +for (const chart of dbRows) { + // 1️⃣ Skip if chart already exists (same hash) + const existingChart = charts.find((c) => c.data.hashSHA1 === chart.hash); + if (existingChart) { + log( + `Chart ${chart.title} ${existingChart.difficulty} already exists (hash match, internalId=${chart.internalId}).` + ); + continue; + } + + // 2️⃣ Determine song + if (!folderIdToSongId[chart.folderid]) { + const splitPath = chart.path.replaceAll("\\", "/").split("/"); + const folderName = splitPath[splitPath.length - 2]; + + const matchingSongs = songs.filter( + (s) => s.title === chart.title && s.artist === chart.artist + ); + + let songID; + if (matchingSongs.length === 0) { + // normal song + songID = getFreshSongID(); + songs.push({ + id: songID, + title: chart.title, + altTitles: [], + artist: chart.artist, + data: {}, + searchTerms: [folderName], + }); + newSongs++; + log(`Added song ${songID} for folder ${folderName}.`); + } else { + // duplicate name/artist → make USC Edition + const effectorName = chart.effector ? chart.effector : "Unknown Effector"; + const uscTitle = makeUscEditionName(chart.title, effectorName); + + const existingUsc = songs.find((s) => s.title === uscTitle && s.artist === chart.artist); + if (existingUsc) { + songID = existingUsc.id; + log(`Found existing USC edition song for ${chart.title}.`); + } else { + songID = getFreshSongID(); + songs.push({ + id: songID, + title: uscTitle, + altTitles: [chart.title], + artist: chart.artist, + data: {}, + searchTerms: [folderName], + }); + newSongs++; + log(`Duplicate name detected → created USC edition song ${uscTitle} (ID ${songID}).`); + } + } + + folderIdToSongId[chart.folderid] = songID; + } + + const songID = folderIdToSongId[chart.folderid]; + + // 3️⃣ Add charts, skip duplicates + for (const playtype of ["Controller", "Keyboard"]) { + const duplicate = charts.find( + (c) => + c.songID === songID && + c.playtype === playtype && + c.difficulty === chart.diff_shortname && + c.isPrimary === true + ); + + if (duplicate) { + log( + `Skipped duplicate chart: ${chart.title} ${chart.diff_shortname} (${playtype}, songID=${songID}, internalId=${chart.internalId}).` + ); + continue; + } + + charts.push({ + chartID: CreateChartID(), + data: { + effector: chart.effector, + hashSHA1: chart.hash, + isOfficial: false, + tableFolders: [], + }, + difficulty: chart.diff_shortname, + isPrimary: true, + level: chart.level.toString(), + levelNum: 0, + playtype, + songID, + versions: [], + }); + + newCharts++; + log( + `Added chart ${chart.title} ${chart.diff_shortname} (${playtype}, internalId=${chart.internalId}).` + ); + } +} + +console.log(`Added ${newSongs} new songs and ${newCharts} new charts.`); + +WriteCollection("songs-usc.json", songs); +WriteCollection("charts-usc.json", charts); diff --git a/usc/seeds/nautica/maps-nov-10-2025.db b/usc/seeds/nautica/maps-nov-10-2025.db Binary files differnew file mode 100644 index 0000000..99693b1 --- /dev/null +++ b/usc/seeds/nautica/maps-nov-10-2025.db diff --git a/usc/seeds/nautica/nautica_to_maps.py b/usc/seeds/nautica/nautica_to_maps.py index f62c144..0a727e5 100644 --- a/usc/seeds/nautica/nautica_to_maps.py +++ b/usc/seeds/nautica/nautica_to_maps.py @@ -13,6 +13,10 @@ import argparse import requests import json import sqlite3 +import zipfile +from pathlib import Path +import hashlib +import shutil import os headers = { @@ -38,6 +42,14 @@ DIFF_NAME_MAP = { 3: "EXH", 4: "INF" } + +USC_DIFFICULTY_MAP = { + "light": "NOV", + "challenge": "ADV", + "extended": "EXH", + "infinite": "INF" +} + def create_maps_db_if_not_exists(filepath: str): if os.path.exists(filepath): print("[DB] Maps DB already exists, skipping creation") @@ -73,10 +85,79 @@ def chart_already_processed(db_path: str, internal_id: str): conn.close() return result is not None -def download_and_generate_charts(): - pass +def find_all_ksh_files(path): + target_dir = Path(path) + ksh_files = [f for f in target_dir.rglob("*.ksh") if f.is_file() and "__MACOSX" not in f.parts] + return ksh_files + + +def get_ksh_difficulty(path): + try: + with open(path, "r") as f: + for line in f: + line = line.strip() + if line.startswith("difficulty="): + return line.split("=")[1] + return None + except: + return None + +def compute_sha1(path: Path) -> str: + sha1 = hashlib.sha1() + with path.open("rb") as f: + while chunk := f.read(0x80): + sha1.update(chunk) + digest = sha1.digest() + parts = [int.from_bytes(digest[i:i+4], "big") for i in range(0, 20, 4)] + return "".join(f"{x:08x}" for x in parts) + +def download_and_generate_chart_hash(download_url: str): + chart_file_name = download_url.split("/")[-1] + try: + print(f"[DOWNLOAD] Downloading Chart Data {chart_file_name}") + response = requests.get(download_url, stream=True) + response.raise_for_status() + with open("chart.zip", "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + print("[DOWNLOAD] Downloading ") + except Exception as e: + print(f"[DOWNLOAD] ERROR! Failed to download {chart_file_name}. EXCEPTION: {e}") + return None + os.makedirs("working", exist_ok=True) + try: + with zipfile.ZipFile("chart.zip", 'r') as zip_ref: + zip_ref.extractall("working") + except Exception: + print("[ERROR] Unable to extract, bad ZIP file? Skipping...") + return None + print("[EXTRACT] Successfully extracted chart data") + ksh_files = find_all_ksh_files("working") + processed_charts = {} + for chart_path in ksh_files: + print(f"[HASH] Now generating hash for {chart_path}") + ksh_difficulty = get_ksh_difficulty(chart_path) + if not ksh_difficulty: + print("[ERROR] No difficulty found in KSM chart. This may be an invalid chart") + continue + usc_diff_name = USC_DIFFICULTY_MAP[ksh_difficulty] + sha1_hash = compute_sha1(chart_path) + processed_charts[usc_diff_name] = sha1_hash + return processed_charts + +def create_row_db(data): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute("INSERT INTO Charts (internalId, hash, title, artist, effector, level, diff_shortname, path, folderid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", ( + data["internalId"], data["hash"], data["title"], data["artist"], data["effector"], data["level"], data["diff_shortname"], data["path"], data["folderid"] + )) + result = cursor.fetchone() + conn.commit() + print("[DB] Wrote row to DB") + conn.close() + return result is not None -def get_charts_from_page(page_num: int, db_path: str): +def process_chart_page(page_num: int, db_path: str): charts = [] response = requests.get(NAUTICA_URL + "?page="+str(page_num), headers=headers) resp_page_obj = json.loads(response.text) @@ -85,10 +166,19 @@ def get_charts_from_page(page_num: int, db_path: str): title = entry["title"] artist = entry["artist"] download_url = entry["cdn_download_url"] + if os.path.exists("working"): + shutil.rmtree("working") + print("[CLEANUP] Removed existing working directory") + if os.path.exists("chart.zip"): + os.remove("chart.zip") + print("[CLEANUP] Removed existing chart.zip") + hash_data = download_and_generate_chart_hash(download_url) + if hash_data is None: + continue for chart in entry["charts"]: difficulty = int(chart["difficulty"]) level = chart["level"] - if chart_already_processed(chart["id"]): + if chart_already_processed(db_path, chart["id"]): print(f"[SKIP] {title} - {difficulty} already exists. Skipping...") continue effector = chart["effector"] @@ -96,11 +186,12 @@ def get_charts_from_page(page_num: int, db_path: str): charts.append({ "internalId": chart["id"], "title": title, + "hash": hash_data[diff_shortname], "artist": artist, "effector": effector, "level": level, "diff_shortname": diff_shortname, - "path": "/charts/blahblah", + "path": f"/nautica/chart.ksh", "folderid": folder_id }) folder_id += 1 @@ -113,9 +204,17 @@ if __name__ == "__main__": description="Converts ALL charts on Nautica (ksm.dev) to a USC maps.db file", ) parser.add_argument("--db", help="Path to existing maps.db if none-specified this script will search in current working dir or create a new one") + parser.add_argument("--start-page", help="Start from this page on ksm.dev", default=1) args = parser.parse_args() db_path = args.db if not db_path: db_path = "maps.db" create_maps_db_if_not_exists(db_path) - print(get_charts_from_page(1, db_path)) + num_pages = get_nautica_num_pages() + start_page = int(args.start_page) + print(f"Found {num_pages} to process...") + for i in range(start_page, num_pages + 1): + print(f"[PROGRESS] {i}/{num_pages + 1} COMPLETED") + charts = process_chart_page(i, db_path) + for chart in charts: + create_row_db(chart) |