Refactored entire codebase

- Replaced most code with HTML templates
- Fixed some janky SQL queries
- Replaced config.py with ini and json

1 files changed, 0 insertions, 168 deletions

diff --git a/main.py b/main.py
deleted file mode 100644
index 702eb9f..0000000
--- a/main.py
+++ /dev/null
@@ -1,168 +0,0 @@
-import os
-import sys
-import time
-
-import sql.graph_builder as gb
-import sql.table_builder as tb
-import fileutil as fs
-import html_gen.html_gen as ht
-from sql.sql_handler import SQLHandler
-from webapi.holodex import HolodexAPI
-from webapi.youtube import YouTubeAPI
-from config import *
-
-def update_database(server: SQLHandler, data):
-    server.clear_table(TABLE_NAME)
-    server.reset_auto_increment(TABLE_NAME)
-    excluded_channels = fs.get_excluded_channels()
-    refresh_diff_table = fs.check_diff_refresh()
-    for channel in data:
-        channel_id = channel['id']
-        if channel_id in excluded_channels:
-            continue
-        pfp = channel['photo']
-        name = channel['english_name']
-        sub_count = channel['subscriber_count']
-
-        if name is None: # if the channel doesn't have an english name, use the japanese name
-            name = channel['name']
-        name = name.encode("ascii", "ignore").decode()
-
-        data_row = f"'{channel_id}', '{pfp}','{name}', {sub_count}, '{time.strftime('%Y-%m-%d %H:%M:%S')}'"
-        server.create_table(DAY_DIFF_TABLE_NAME, DIFF_COLUMNS)
-
-        # Difference should only be calculated every 24 hours
-        # If the channel is new then calculate now, else then make sure 24 hours has passed since last reading
-        if refresh_diff_table or not server.check_row_exists(DAY_DIFF_TABLE_NAME, "channel_id", channel_id):
-            if REFRESH_DAILY:
-                generate_channel_files()
-            if not server.check_row_exists(DAY_DIFF_TABLE_NAME, "channel_id", channel_id):
-                server.insert_data(DAY_DIFF_TABLE_NAME,
-                                   diff_columns, f"'{channel_id}', {sub_count}")
-            else:
-                server.update_row(DAY_DIFF_TABLE_NAME, "channel_id", channel_id, "sub_diff", sub_count)
-            server.insert_data(HISTORICAL_TABLE_NAME, table_columns, data_row)
-
-        # make updates to the main table
-        server.insert_data(TABLE_NAME, table_columns, data_row)
-        name_table = "channel_"+channel_id.lower()+"_subscriber_data"
-        name_table = name_table.replace("-", "$")
-        server.create_table(name_table, TABLE_COLUMNS)
-        server.insert_data(name_table, table_columns, data_row)
-        # server.drop_table(name_table)
-
-def generate_individual_stats(server: SQLHandler, data):
-    for channel in data:
-        channel_id = channel['id']
-        if channel_id in fs.get_excluded_channels():
-            continue
-        name = channel['english_name']
-        desc = channel['description']
-        pfp = channel['photo']
-        sub_count = channel['subscriber_count']
-        sub_count_str = "{:,.0f}".format(int(sub_count))
-        if name is None:
-            name = channel['name']
-        name = name.encode("ascii", "ignore").decode()
-        print("GENERATING PAGE FOR "+name+"...")
-
-        # Calculate table key name stored in the database
-        table_key = "channel_"+channel_id.lower()+"_subscriber_data"
-        table_key = table_key.replace("-", "$")
-
-        # Generate missing directories
-        if not os.path.exists("stats"):
-            os.mkdir("stats")
-        if not os.path.exists("tables"):
-            os.mkdir("tables")
-
-        with open(ROOT_STORAGE_PATH+"stats/"+name+".html", "w", encoding="utf-8") as file:
-            range_query = f"SELECT name, subscriber_count, timestamp FROM {table_key} WHERE timestamp >= DATE_SUB(CURRENT_DATE(), INTERVAL 7 DAY) ORDER BY timestamp DESC"
-            group_query = f"SELECT name, subscriber_count, timestamp FROM {table_key} GROUP BY DATE(timestamp) ORDER BY timestamp DESC"
-            file.write(ht.generate_meta_data(name + " - "+ META_DATA_TITLE, META_DATA_DESCRIPTION, META_PROFILE_PIC)+"\n"+
-                       ht.side_swipe_header(sub_count_str + " Subscribers", COLOR_THEME, ROOT_URL)+"\n"+
-                       ht.generate_info_card(name, channel_id, pfp, desc)+
-                       gb.generate_projection(server.host_name, server.username, server.password, DB_NAME, table_key, int(sub_count), TIMEZONE_STR)+
-                       gb.plot_subscriber_count_over_time(server.host_name, server.username, server.password, DB_NAME, table_name=table_key, gtitle=name+" Subscriber Trend", overrideQuery=group_query) +
-                       ht.generate_html_divider("Recent Subscriber Data:") +
-                       gb.plot_subscriber_count_over_time(server.host_name, server.username, server.password, DB_NAME, table_name=table_key, gtitle=name+" Last 7 Days Trend", overrideQuery=range_query, markers="lines+markers") +
-                       tb.generate_individual_table(server.host_name, server.username, server.password, DB_NAME, table_key)+
-                       ht.generate_full_table_button(ROOT_URL+"/tables/"+name) +
-                       ht.generate_doctype_footer()
-                       )
-        with open(ROOT_STORAGE_PATH+"tables/"+name+".html", "w", encoding="utf-8") as file:
-            file.write(ht.generate_meta_data(name + " - "+ META_DATA_TITLE, META_DATA_DESCRIPTION, META_PROFILE_PIC)+"\n"+
-                       ht.side_swipe_header(name, COLOR_THEME, ROOT_URL)+"\n"+
-                       ht.generate_info_card(name, channel_id, pfp, desc)+
-                       tb.generate_individual_table(server.host_name, server.username, server.password, DB_NAME, table_key, param="") +
-                       ht.generate_doctype_footer())
-
-
-def main(mode=0):
-    address, user, password = fs.get_login_data()
-    server = SQLHandler(address, user, password, DB_NAME)
-    server.create_table(TABLE_NAME, TABLE_COLUMNS)
-    server.create_table(HISTORICAL_TABLE_NAME, TABLE_COLUMNS)
-    server.create_table(DAY_DIFF_TABLE_NAME, DIFF_COLUMNS)
-    excluded_channels = []
-    data = []
-    if mode == 0:
-        holodex_organizations = HOLODEX_ORG.split(",")
-        print("Running Holodex API Data Collection\n"+"Found "+str(len(holodex_organizations))+" organizations")
-        try:
-            for org in holodex_organizations:
-                hldex = HolodexAPI(fs.get_api_key("holodex_api_key"), member_count = ORG_MEMBER_COUNT, organization = org)
-                data += hldex.get_data_all_channels()
-                excluded_channels += hldex.get_exclude_channels()
-        except:
-            main(mode=1)
-
-    elif mode == 1:
-        print("Running YouTube API Data Collection")
-        ytapi = YouTubeAPI(fs.get_api_key("youtube_api_key"))
-        data = ytapi.get_data_all_channels(fs.get_local_channels())
-
-    # Updating DB and generating HTML data
-    update_database(server, data)
-    excluded_channels += fs.get_excluded_channels()
-    with open(ROOT_STORAGE_PATH+"index.html", "w", encoding="utf-8") as file:
-        file.write(ht.generate_meta_data(META_DATA_TITLE, META_DATA_DESCRIPTION, META_PROFILE_PIC) +
-                   ht.generate_title_banner(HEADER_TITLE, COLOR_THEME) +
-                   gb.plot_subscriber_count_over_time(address, user, password, DB_NAME, HISTORICAL_TABLE_NAME, gtitle="Subscriber Count Over Time", exclude_channels=excluded_channels) +
-                   "\n"+ht.generate_html_divider("Last Updated: "+time.strftime('%Y-%m-%d %H:%M:%S')+" "+ TIMEZONE) +
-                   tb.sql_to_html_table(address, user, password, DB_NAME, TABLE_NAME, root_url=ROOT_URL) +
-                   ht.build_footer_info(FOOTER_MESSAGE) +
-                   ht.generate_doctype_footer())
-
-    # Generating individual pages
-    generate_individual_stats(server, data)
-
-def generate_channel_files():
-    """
-    Generates the channels.txt and exclude_channels.txt files based on Holodex listings
-    """
-    if not UPDATE_LOCAL_RECORDS:
-        return
-    print("Running Channel Files Update")
-    active_channels = []
-    exclude_channels = []
-    if not os.path.exists("data"):
-        os.mkdir("data")
-    for org in HOLODEX_ORG.split(","):
-        hldex = HolodexAPI(fs.get_api_key("holodex_api_key"), member_count = ORG_MEMBER_COUNT,
-                    organization = org)
-        hldex.get_data_all_channels()
-        active_channels += hldex.get_active_channels()
-        exclude_channels += hldex.get_exclude_channels()
-    with open("data/channels.txt", "w", encoding="utf-8") as file:
-        file.write("\n".join(active_channels))
-    with open("data/exclude_channel.txt", "w", encoding="utf-8") as file:
-        file.write("\n".join(exclude_channels))
-    print("Success! Channel Files Updated!")
-
-if __name__ == "__main2__":
-    MODE = 0
-    if len(sys.argv) > 1:
-        MODE = int(sys.argv[1])
-        ROOT_STORAGE_PATH = sys.argv[2]
-    main(MODE)