sdvx_eamuse_csv: add option to scrape recent 20 date data from eamusement html

3 files changed, 69 insertions, 11 deletions

diff --git a/sdvx/eamuse_csv/README.md b/sdvx/eamuse_csv/README.md
index 7f9d8b3..4c33f96 100644
--- a/sdvx/eamuse_csv/README.md
+++ b/sdvx/eamuse_csv/README.md
@@ -1,18 +1,36 @@
+tl;dr:
+- e-amusement doesn't let you see entire play history even with premium
+- Web-UI only shows your best scores. If you get a score lower than your best, the score will be lost
+- Please keep an old version of your CSV before your play-session (script helps with this) so the script can tell what plays are new
+- Limit each session to 20 plays since play timestamp is only available for recent 20
+
 # e-amusement SDVX CSV to Tachi WARNING!!!
 The CSV provided by Konami doesn't contain any information regarding the date which you played/obtained a particular score. It only keeps your BEST score.
 
 This causes for duplicate imports if you use this script more than once
 
 ## Solution
-The first time you import, run `sdvx_csv_to_tach.py` and import the output JSON file normally. Howver **KEEP THE OLD CSV FILE**
+The first time you import, run `sdvx_csv_to_tach.py` and import the output JSON file normally. However **KEEP THE OLD CSV FILE**
 
 The next time you play, export the CSV again but first run `eamuse_merge_csv.py --old <file> --new <file>`. Passing the old and new CSV files accordingly.
 
 This will check for differences between the 2 files and remove lines of the same value in the new file so that only newly obtained best scores are uploaded.
 
+**Run the `sdvx_csv_to_tachi.csv` on the new file created by `eamuse_merge_csv`**
+
+The merge script can automatically handle this process. Choose (y) when prompted
+
 # What if I want all plays?
 I don't think there's a method right now to automatically import arcade data into Tachi for each play. Play history is locked behind the eAmusement basic subscription plan.
 
 The best you can do is download the eAmusement app, subscribe the the basic course and manually make your own score CSV in the same format. Then run `sdvx_csv_to_tach.py`
 
 Alternatively, you can take pictures of each play result screen and build your own JSON so that there is even more info available!
+
+# Adding Correct Date Data
+The e-amusement WEBUI only provides the time-played for your 20 most recent plays. If you want accurate tracking for session-length please limit each import to 20 plays, otherwise there is no way to know when that score was achieved.
+
+To get this data. Navigate to the e-amusement URL below and press CTRL+S to save the page as HTML. Then pass the filepath to this file as `-d`
+```
+https://p.eagate.573.jp/game/sdvx/vi/playdata/profile/index.html
+```
diff --git a/sdvx/eamuse_csv/eamuse_merge_csv.py b/sdvx/eamuse_csv/eamuse_merge_csv.py
index 490114e..7ff4ad9 100644
--- a/sdvx/eamuse_csv/eamuse_merge_csv.py
+++ b/sdvx/eamuse_csv/eamuse_merge_csv.py
@@ -1,7 +1,8 @@
 import argparse
 import csv
+import os
 
-def merge_csv(old_file: str, new_file: str):
+def merge_csv(old_file: str, new_file: str, output_file: str):
     old_csv_set = set()
     new_csv_set = set()
     encoding = "utf-8"
@@ -14,7 +15,7 @@ def merge_csv(old_file: str, new_file: str):
             [new_csv_set.add(tuple(row)) for row in reader]
             new_csv_set = new_csv_set - old_csv_set
 
-        with open(new_file, "w", encoding=encoding, newline="") as new_csv:
+        with open(output_file, "w", encoding=encoding, newline="") as new_csv:
             writer = csv.writer(new_csv, delimiter=",")
             writer.writerow(header)
             for row in new_csv_set:
@@ -22,6 +23,16 @@ def merge_csv(old_file: str, new_file: str):
                     continue
                 writer.writerow(list(row))
 
+        print("Done! Your newly merged CSV is at " + output_file)
+        print("Do you want to delete the old file, and create a new old.csv? (y/n)")
+        action = input()
+        if not action == "y":
+            print("OK. Exiting")
+            exit(0)
+        print("Renaming " + new_file + " to old.csv and deleting " + old_file + "...")
+        os.remove(old_file)
+        os.rename(new_file, "old.csv")
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
@@ -30,5 +41,6 @@ if __name__ == "__main__":
     )
     parser.add_argument("--old", help="Old CSV file", required=True)
     parser.add_argument("--new", help="New CSV file", required=True)
+    parser.add_argument("--output", help="Output File", default="sdvx_merged.csv")
     args = parser.parse_args()
-    merge_csv(args.old, args.new)
+    merge_csv(args.old, args.new, args.output)
diff --git a/sdvx/eamuse_csv/sdvx_csv_to_tachi.py b/sdvx/eamuse_csv/sdvx_csv_to_tachi.py
index e860f0b..cecb0a8 100644
--- a/sdvx/eamuse_csv/sdvx_csv_to_tachi.py
+++ b/sdvx/eamuse_csv/sdvx_csv_to_tachi.py
@@ -20,7 +20,25 @@ LAMP_MAPPING = {
     "COMPLETE": "CLEAR",
 }
 
-def convert_sdvx_csv_to_tachi_json(csv_file, game, playtype, service, unixtime):
+def create_date_dict(raw_site_data: str) -> dict:
+    from bs4 import BeautifulSoup
+    from datetime import datetime
+    import pytz
+    soup = BeautifulSoup(raw_site_data, 'html.parser')
+    cat_divs = soup.find_all('div', class_='cat')[5:]
+    jst = pytz.timezone('Asia/Tokyo')
+    date_data = {}
+    for div in cat_divs:
+        inners = div.find_all('div', class_='inner')
+        date = inners[0].get_text(strip=True)
+        music_name = inners[1].find('p', class_='music_name').get_text(strip=True)
+        date_obj = datetime.strptime(date, "%Y/%m/%d %H:%M:%S")
+        date_obj_jst = jst.localize(date_obj)
+        date_obj_unixtime = int(date_obj_jst.timestamp() * 1000)
+        date_data[music_name] = date_obj_unixtime
+    return date_data
+
+def convert_sdvx_csv_to_tachi_json(csv_file, game, playtype, service, unixtime, date_dict):
     encodings = ['utf-8-sig', 'utf-8', 'shift-jis', 'cp932']
 
     for encoding in encodings:
@@ -48,6 +66,7 @@ def convert_sdvx_csv_to_tachi_json(csv_file, game, playtype, service, unixtime):
                         lamp = "ULTIMATE CHAIN"
                     if row.get("PERFECT") == "1":
                         lamp = "PERFECT ULTIMATE CHAIN"
+                    time_stamp = date_dict.get(row["楽曲名"], unixtime if unixtime != "now" else int(time.time())*1000)
 
                     score_entry = {
                         "score": int(row["ハイスコア"]),
@@ -55,7 +74,7 @@ def convert_sdvx_csv_to_tachi_json(csv_file, game, playtype, service, unixtime):
                         "matchType": "songTitle",
                         "identifier": row["楽曲名"],
                         "difficulty": DIFFICULTY_MAPPING[row["難易度"].upper()],
-                        "timeAchieved": unixtime
+                        "timeAchieved": time_stamp
                     }
                     optional_fields = {}
                     if row.get("EXスコア"):
@@ -68,9 +87,6 @@ def convert_sdvx_csv_to_tachi_json(csv_file, game, playtype, service, unixtime):
                         optional_fields["maxCombo"] = int(row["maxCombo"])
                     if row.get("gauge"):
                         optional_fields["gauge"] = float(row["gauge"])
-                    if unixtime:
-                        if unixtime == "now":
-                            unixtime = int(time.time())*1000
                     if optional_fields:
                         score_entry["optional"] = optional_fields
                     batch_manual["scores"].append(score_entry)
@@ -87,6 +103,7 @@ def convert_sdvx_csv_to_tachi_json(csv_file, game, playtype, service, unixtime):
 if __name__ == "__main__":
     print("!!!!! WARNING !!!!!")
     print("Please make sure you have read the README about SDVX's official CSV files before importing")
+    print("If you pass in a date file make sure the HTML is unformatted otherwise the spacing for song names will be wrong!")
     parser = argparse.ArgumentParser(
         prog="sdvx_csv_to_tachi",
         description="Converts CSV data exported from SDVX eAmuse site to Tachi compatibile JSON",
@@ -95,12 +112,23 @@ if __name__ == "__main__":
     parser.add_argument("csv_filename", help="Path to the CSV file")
     parser.add_argument("-s", "--service", help="Service description to be shown on Tachi (Note for where this score came from)", default="SDVX Arcade Import")
     parser.add_argument("-o", "--output", help="Output filename", default="sdvx_tachi.json")
-    parser.add_argument("-t", "--time", help="UNIX time (in milliseconds) that should be added to the scores. Input 'now' if you want to use current time. If no value is provided timeAchieved will not be added to the final JSON", default=None)
+    parser.add_argument("-t", "--time", help="UNIX time (in milliseconds) that should be added to the scores. Defaults to current UNIX time", default=None)
+    parser.add_argument("-d", "--date_file", help="SDVX e-amusement profile site saved HTML. See README in sdvx/eamuse_csv for instructions. Overrides with --time input if missing", required=False)
+    parser.add_argument("-tz", "--timezone", help="Only needed if -d is used, specifies what timezone to convert to", required=False)
     args = parser.parse_args()
     curr_time = int(time.time() * 1000) if (args.time == "now" or args.time is None) else args.time
 
+    if args.date_file and not args.timezone:
+        print("ERROR: A date file is provided but no target timezone (-tz) has been specified. Please pass the timezone in which you played the scores")
+        exit(1)
+
+    date_data = {}
+    if args.date_file:
+        with open(args.date_file, "r") as file:
+            site_data = file.read()
+            date_data = create_date_dict(site_data)
 try:
-    output_json = convert_sdvx_csv_to_tachi_json(args.csv_filename, "sdvx", "Single", args.service, curr_time)
+    output_json = convert_sdvx_csv_to_tachi_json(args.csv_filename, "sdvx", "Single", args.service, curr_time, date_data)
 
     with open(args.output, "w", encoding="utf-8") as json_file:
         json.dump(output_json, json_file, ensure_ascii=False, indent=4)