From 5d26d7a3f0571444f7a006b0c1be0c2bfe9912c5 Mon Sep 17 00:00:00 2001 From: Pinapelz Date: Fri, 28 Feb 2025 01:07:48 -0800 Subject: sdvx: add merge sdvx csv script --- sdvx/eamuse_csv/README.md | 18 ++++++ sdvx/eamuse_csv/eamuse_merge_csv.py | 34 +++++++++++ sdvx/eamuse_csv/sdvx_csv_to_tachi.py | 109 +++++++++++++++++++++++++++++++++++ sdvx/sdvx_csv_to_tachi.py | 109 ----------------------------------- 4 files changed, 161 insertions(+), 109 deletions(-) create mode 100644 sdvx/eamuse_csv/README.md create mode 100644 sdvx/eamuse_csv/eamuse_merge_csv.py create mode 100644 sdvx/eamuse_csv/sdvx_csv_to_tachi.py delete mode 100644 sdvx/sdvx_csv_to_tachi.py (limited to 'sdvx') diff --git a/sdvx/eamuse_csv/README.md b/sdvx/eamuse_csv/README.md new file mode 100644 index 0000000..7f9d8b3 --- /dev/null +++ b/sdvx/eamuse_csv/README.md @@ -0,0 +1,18 @@ +# e-amusement SDVX CSV to Tachi WARNING!!! +The CSV provided by Konami doesn't contain any information regarding the date which you played/obtained a particular score. It only keeps your BEST score. + +This causes for duplicate imports if you use this script more than once + +## Solution +The first time you import, run `sdvx_csv_to_tach.py` and import the output JSON file normally. Howver **KEEP THE OLD CSV FILE** + +The next time you play, export the CSV again but first run `eamuse_merge_csv.py --old --new `. Passing the old and new CSV files accordingly. + +This will check for differences between the 2 files and remove lines of the same value in the new file so that only newly obtained best scores are uploaded. + +# What if I want all plays? +I don't think there's a method right now to automatically import arcade data into Tachi for each play. Play history is locked behind the eAmusement basic subscription plan. + +The best you can do is download the eAmusement app, subscribe the the basic course and manually make your own score CSV in the same format. Then run `sdvx_csv_to_tach.py` + +Alternatively, you can take pictures of each play result screen and build your own JSON so that there is even more info available! diff --git a/sdvx/eamuse_csv/eamuse_merge_csv.py b/sdvx/eamuse_csv/eamuse_merge_csv.py new file mode 100644 index 0000000..490114e --- /dev/null +++ b/sdvx/eamuse_csv/eamuse_merge_csv.py @@ -0,0 +1,34 @@ +import argparse +import csv + +def merge_csv(old_file: str, new_file: str): + old_csv_set = set() + new_csv_set = set() + encoding = "utf-8" + with open(old_file, encoding=encoding) as old_csv: + reader = csv.reader(old_csv, delimiter=",") + [old_csv_set.add(tuple(row)) for row in reader] + with open(new_file, "r", encoding=encoding) as new_csv: + reader = csv.reader(new_csv, delimiter=",") + header = next(reader) + [new_csv_set.add(tuple(row)) for row in reader] + new_csv_set = new_csv_set - old_csv_set + + with open(new_file, "w", encoding=encoding, newline="") as new_csv: + writer = csv.writer(new_csv, delimiter=",") + writer.writerow(header) + for row in new_csv_set: + if row == (): + continue + writer.writerow(list(row)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="eamuse_merge_to_csv", + description="Merges an old SDVX e-amuse CSV with a new one. Keeping only new scores in the new file", + ) + parser.add_argument("--old", help="Old CSV file", required=True) + parser.add_argument("--new", help="New CSV file", required=True) + args = parser.parse_args() + merge_csv(args.old, args.new) diff --git a/sdvx/eamuse_csv/sdvx_csv_to_tachi.py b/sdvx/eamuse_csv/sdvx_csv_to_tachi.py new file mode 100644 index 0000000..5c18a73 --- /dev/null +++ b/sdvx/eamuse_csv/sdvx_csv_to_tachi.py @@ -0,0 +1,109 @@ +import csv +import json +import argparse +import time + +DIFFICULTY_MAPPING = { + "NOVICE": "NOV", + "ADVANCED": "ADV", + "EXHAUST": "EXH", + "INFINITE": "INF", + "GRAVITY": "GRV", + "HEAVENLY": "HVN", + "VIVD": "VVD", + "EXCEED": "XCD", + "MAXIMUM": "MXM" +} + +LAMP_MAPPING = { + "PLAYED": "FAILED", + "COMPLETE": "CLEAR", +} + +def convert_sdvx_csv_to_tachi_json(csv_file, game, playtype, service, unixtime): + encodings = ['utf-8-sig', 'utf-8', 'shift-jis', 'cp932'] + + for encoding in encodings: + try: + batch_manual = { + "meta": { + "game": game, + "playtype": playtype, + "service": service + }, + "scores": [] + } + + with open(csv_file, newline='', encoding=encoding) as f: + reader = csv.DictReader(f) + fieldnames = reader.fieldnames + required_fields = ["楽曲名", "難易度", "クリアランク", "ハイスコア"] + if not all(field in fieldnames for field in required_fields): + continue + + + for row in reader: + lamp = LAMP_MAPPING[row["クリアランク"].upper()] + if row.get("ULTIMATE CHAIN"): + lamp = "ULTIMATE CHAIN" + if row.get("PERFECT"): + lamp = "PERFECT ULTIMATE CHAIN" + + score_entry = { + "score": int(row["ハイスコア"]), + "lamp": lamp, + "matchType": "songTitle", + "identifier": row["楽曲名"], + "difficulty": DIFFICULTY_MAPPING[row["難易度"].upper()], + } + optional_fields = {} + if row.get("EXスコア"): + optional_fields["exScore"] = int(row["EXスコア"]) + if row.get("fast"): + optional_fields["fast"] = int(row["fast"]) + if row.get("slow"): + optional_fields["slow"] = int(row["slow"]) + if row.get("maxCombo"): + optional_fields["maxCombo"] = int(row["maxCombo"]) + if row.get("gauge"): + optional_fields["gauge"] = float(row["gauge"]) + if unixtime: + if unixtime == "now": + unixtime = int(time.time())*1000 + optional_fields["timeAchieved"] = unixtime + if optional_fields: + score_entry["optional"] = optional_fields + batch_manual["scores"].append(score_entry) + return batch_manual + except UnicodeDecodeError: + continue + except Exception as e: + print(f"Error with encoding {encoding}: {str(e)}") + continue + + raise ValueError("Failed to read CSV file with any supported encoding") + + +if __name__ == "__main__": + print("!!!!! WARNING !!!!!") + print("Please make sure you have read the README about SDVX's official CSV files before importing") + parser = argparse.ArgumentParser( + prog="sdvx_csv_to_tachi", + description="Converts CSV data exported from SDVX eAmuse site to Tachi compatibile JSON", + epilog="Note that not all information can be derived from the CSV so some fields will be missing from Tachi" + ) + parser.add_argument("csv_filename", help="Path to the CSV file") + parser.add_argument("-s", "--service", help="Service description to be shown on Tachi (Note for where this score came from)", default="SDVX Arcade Import") + parser.add_argument("-o", "--output", help="Output filename", default="sdvx_tachi.json") + parser.add_argument("-t", "--time", help="UNIX time (in milliseconds) that should be added to the scores. Input 'now' if you want to use current time. If no value is provided timeAchieved will not be added to the final JSON", default=None) + args = parser.parse_args() + +try: + output_json = convert_sdvx_csv_to_tachi_json(args.csv_filename, "sdvx", "Single", args.service) + + with open(args.output, "w", encoding="utf-8") as json_file: + json.dump(output_json, json_file, ensure_ascii=False, indent=4) + + print("Conversion completed. JSON saved as " + args.output) +except Exception as e: + print(f"Error: {str(e)}") diff --git a/sdvx/sdvx_csv_to_tachi.py b/sdvx/sdvx_csv_to_tachi.py deleted file mode 100644 index 5c18a73..0000000 --- a/sdvx/sdvx_csv_to_tachi.py +++ /dev/null @@ -1,109 +0,0 @@ -import csv -import json -import argparse -import time - -DIFFICULTY_MAPPING = { - "NOVICE": "NOV", - "ADVANCED": "ADV", - "EXHAUST": "EXH", - "INFINITE": "INF", - "GRAVITY": "GRV", - "HEAVENLY": "HVN", - "VIVD": "VVD", - "EXCEED": "XCD", - "MAXIMUM": "MXM" -} - -LAMP_MAPPING = { - "PLAYED": "FAILED", - "COMPLETE": "CLEAR", -} - -def convert_sdvx_csv_to_tachi_json(csv_file, game, playtype, service, unixtime): - encodings = ['utf-8-sig', 'utf-8', 'shift-jis', 'cp932'] - - for encoding in encodings: - try: - batch_manual = { - "meta": { - "game": game, - "playtype": playtype, - "service": service - }, - "scores": [] - } - - with open(csv_file, newline='', encoding=encoding) as f: - reader = csv.DictReader(f) - fieldnames = reader.fieldnames - required_fields = ["楽曲名", "難易度", "クリアランク", "ハイスコア"] - if not all(field in fieldnames for field in required_fields): - continue - - - for row in reader: - lamp = LAMP_MAPPING[row["クリアランク"].upper()] - if row.get("ULTIMATE CHAIN"): - lamp = "ULTIMATE CHAIN" - if row.get("PERFECT"): - lamp = "PERFECT ULTIMATE CHAIN" - - score_entry = { - "score": int(row["ハイスコア"]), - "lamp": lamp, - "matchType": "songTitle", - "identifier": row["楽曲名"], - "difficulty": DIFFICULTY_MAPPING[row["難易度"].upper()], - } - optional_fields = {} - if row.get("EXスコア"): - optional_fields["exScore"] = int(row["EXスコア"]) - if row.get("fast"): - optional_fields["fast"] = int(row["fast"]) - if row.get("slow"): - optional_fields["slow"] = int(row["slow"]) - if row.get("maxCombo"): - optional_fields["maxCombo"] = int(row["maxCombo"]) - if row.get("gauge"): - optional_fields["gauge"] = float(row["gauge"]) - if unixtime: - if unixtime == "now": - unixtime = int(time.time())*1000 - optional_fields["timeAchieved"] = unixtime - if optional_fields: - score_entry["optional"] = optional_fields - batch_manual["scores"].append(score_entry) - return batch_manual - except UnicodeDecodeError: - continue - except Exception as e: - print(f"Error with encoding {encoding}: {str(e)}") - continue - - raise ValueError("Failed to read CSV file with any supported encoding") - - -if __name__ == "__main__": - print("!!!!! WARNING !!!!!") - print("Please make sure you have read the README about SDVX's official CSV files before importing") - parser = argparse.ArgumentParser( - prog="sdvx_csv_to_tachi", - description="Converts CSV data exported from SDVX eAmuse site to Tachi compatibile JSON", - epilog="Note that not all information can be derived from the CSV so some fields will be missing from Tachi" - ) - parser.add_argument("csv_filename", help="Path to the CSV file") - parser.add_argument("-s", "--service", help="Service description to be shown on Tachi (Note for where this score came from)", default="SDVX Arcade Import") - parser.add_argument("-o", "--output", help="Output filename", default="sdvx_tachi.json") - parser.add_argument("-t", "--time", help="UNIX time (in milliseconds) that should be added to the scores. Input 'now' if you want to use current time. If no value is provided timeAchieved will not be added to the final JSON", default=None) - args = parser.parse_args() - -try: - output_json = convert_sdvx_csv_to_tachi_json(args.csv_filename, "sdvx", "Single", args.service) - - with open(args.output, "w", encoding="utf-8") as json_file: - json.dump(output_json, json_file, ensure_ascii=False, indent=4) - - print("Conversion completed. JSON saved as " + args.output) -except Exception as e: - print(f"Error: {str(e)}") -- cgit v1.2.3