From 6d084b98235bf1799e17fc17aad7ab9894621915 Mon Sep 17 00:00:00 2001 From: Pinapelz Date: Sat, 13 Apr 2024 01:20:53 -0700 Subject: change backend API to a submodule --- .gitmodules | 3 + backend/.gitignore | 170 -------------------------- backend/app.py | 154 ----------------------- backend/config.ini | 31 ----- backend/data/channels.txt | 4 - backend/data/exclude_channel.txt | 9 -- backend/data/last_refresh.txt | 1 - backend/decorators.py | 16 --- backend/fileutil.py | 93 -------------- backend/graph.py | 35 ------ backend/member_color.py | 30 ----- backend/nijitrack.py | 176 --------------------------- backend/requirements.txt | 155 ----------------------- backend/sql/pg_handler.py | 169 ------------------------- backend/sql/sql_handler.py | 171 -------------------------- backend/sql_table_config.json | 9 -- backend/webapi/holodex.py | 75 ------------ backend/webapi/web_api.py | 29 ----- backend/webapi/youtube.py | 61 ---------- src/components/CompactTable/CompactTable.tsx | 9 ++ 20 files changed, 12 insertions(+), 1388 deletions(-) create mode 100644 .gitmodules delete mode 100644 backend/.gitignore delete mode 100644 backend/app.py delete mode 100644 backend/config.ini delete mode 100644 backend/data/channels.txt delete mode 100644 backend/data/exclude_channel.txt delete mode 100644 backend/data/last_refresh.txt delete mode 100644 backend/decorators.py delete mode 100644 backend/fileutil.py delete mode 100644 backend/graph.py delete mode 100644 backend/member_color.py delete mode 100644 backend/nijitrack.py delete mode 100644 backend/requirements.txt delete mode 100644 backend/sql/pg_handler.py delete mode 100644 backend/sql/sql_handler.py delete mode 100644 backend/sql_table_config.json delete mode 100644 backend/webapi/holodex.py delete mode 100644 backend/webapi/web_api.py delete mode 100644 backend/webapi/youtube.py create mode 100644 src/components/CompactTable/CompactTable.tsx diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e7ca9bb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "NijiTrack-API"] + path = NijiTrack-API + url = https://github.com/pinapelz/NijiTrack-API diff --git a/backend/.gitignore b/backend/.gitignore deleted file mode 100644 index 51ff7fb..0000000 --- a/backend/.gitignore +++ /dev/null @@ -1,170 +0,0 @@ -__pycache__ -config.json -.venv -.github -main.py -.idea -tables -stats -.vscode - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ diff --git a/backend/app.py b/backend/app.py deleted file mode 100644 index 5e3fa7c..0000000 --- a/backend/app.py +++ /dev/null @@ -1,154 +0,0 @@ -""" -Flask app for serving the static files -""" -from flask import Flask, send_file, jsonify -from flask_cors import CORS -from sql.pg_handler import PostgresHandler -import fileutil as fs -import datetime -import pandas -from sklearn.linear_model import Ridge -import numpy as np -import os -from dotenv import load_dotenv - -load_dotenv() - -app = Flask(__name__) -CORS(app) - -# Optional setting to use any of the custom options below -START_DATE = "2023-04-01" # 2023 April 1st - -# Do not include datapoints before the START_DATE for any /api/subscribers/ endpoint -# For when you only want to serve actual data you collected at those specific endpoints -ALL_EXCLUDE_MANUAL_DATA = False - -# Do not include datapoints before the START_DATE for any /api/subscribers/ endpoint -# For when you only want to serve actual data you collected at those specific endpoints -INDIVIDUAL_EXCLUDE_MANUAL_DATA = True - -def create_database_connection(): - """ - Creates a database connection using the environment variables - :param: auth_append: str = "" - If you want to use a different set of variables for persisitance of sessions - """ - hostname = os.environ.get("POSTGRES_HOST") - user = os.environ.get("POSTGRES_USER") - password = os.environ.get("POSTGRES_PASSWORD") - database = os.environ.get("POSTGRES_DATABASE") - return PostgresHandler(host_name=hostname, username=user, password=password, database=database, port=5432) - -@app.route("/") -def index(): - try: - return send_file("index.html") - except Exception as e: - return jsonify({"error": str(e)}) - -@app.route("/api/subscribers") -def api_subscribers(): - server = create_database_connection() - query = 'SELECT sd.*, h.* FROM subscriber_data sd INNER JOIN "24h_historical" h ON sd.channel_id = h.channel_id ORDER BY sd.subscriber_count DESC' - data = server.execute_query(query) - channel_data_list = [{"channel_name": row[3], "profile_pic": row[2], "subscribers": row[4], "sub_org": row[5], "video_count": row[6], "day_diff": int(row[4] - int(row[10]))} for row in data] - subscriber_data = {"timestamp": datetime.datetime.now(), "channel_data": channel_data_list} - return jsonify(subscriber_data) - -@app.route("/api/subscribers/") -def api_subscribers_channel(channel_name): - server = create_database_connection() - query = "SELECT * FROM subscriber_data_historical WHERE name = %s AND timestamp > %s ORDER BY TO_CHAR(timestamp, 'YYYY-MM-DD')" - data = server.execute_query(query, (channel_name, START_DATE)) - labels = [] - data_points = [] - seen_dates = set() - for row in data: - date_string = row[5].strftime("%Y-%m-%d") - if date_string in seen_dates: - continue - labels.append(date_string) - data_points.append(row[4]) - seen_dates.add(date_string) - return jsonify({"labels": labels, "datasets": data_points}) - -@app.route("/api/subscribers//7d") -def api_subscribers_channel_7d(channel_name): - server = create_database_connection() - query = "SELECT * FROM subscriber_data_historical WHERE name = %s ORDER BY TO_CHAR(timestamp, 'YYYY-MM-DD')" - data = server.execute_query(query, (channel_name,)) - labels = [] - data_points = [] - seen_dates = set() - for row in data: - date_string = row[5].strftime("%Y-%m-%d") - if date_string in seen_dates: - continue - labels.append(date_string) - data_points.append(row[4]) - seen_dates.add(date_string) - return jsonify({"labels": labels[-7:], "datasets": data_points[-7:]}) - -@app.route("/api/channel/") -def get_channel_information(channel_name): - def find_next_milestone(subscriber_count): - if subscriber_count < 100000: - return ((subscriber_count // 10000) + 1) * 10000 - elif subscriber_count < 1000000: - return ((subscriber_count // 100000) + 1) * 100000 - else: - return ((subscriber_count // 1000000) + 1) * 1000000 - server = create_database_connection() - query = "SELECT * FROM subscriber_data WHERE name = %s" - data = server.execute_query(query, (channel_name,)) - channel_data = {"channel_id": data[0][1], "channel_name": data[0][3], "profile_pic": data[0][2], "subscribers": data[0][4], "sub_org": data[0][5], "video_count": data[0][6]} - historical_data = server.execute_query("SELECT * FROM subscriber_data_historical WHERE name = %s", (channel_name,)) - current_subscriber_count = data[0][4] - subscriber_points = [] - date_strings = [] - seen_dates = set() - for row in historical_data: - date_string = row[5].strftime("%Y-%m-%d") - if date_string in seen_dates: - continue - subscriber_points.append(row[4]) - date_strings.append(date_string) - seen_dates.add(date_string) - data = {"subscribers": subscriber_points, "dates": date_strings} - df = pandas.DataFrame(data=data) - df['dates'] = pandas.to_datetime(df['dates']) - df.set_index('dates', inplace=True) - df.sort_index(inplace=True) - three_months_ago = datetime.datetime.now() - datetime.timedelta(days=90) - df = df[df.index > three_months_ago] - try: - model = Ridge(alpha=100) - X = np.array(range(len(df))).reshape(-1, 1) - y = df['subscribers'] - model.fit(X, y) - next_milestone = find_next_milestone(current_subscriber_count) - days_until_next_milestone = (next_milestone - model.intercept_) / model.coef_ - days_until_next_milestone_scalar = int(days_until_next_milestone[0]) - next_milestone_date = (df.index[0] + pandas.Timedelta(days=days_until_next_milestone_scalar)).date() - time_until_next_milestone = (next_milestone_date - datetime.datetime.now().date()).days - if time_until_next_milestone < 0: - raise OverflowError - channel_data["next_milestone_date"] = str(next_milestone_date) - channel_data["days_until_next_milestone"] = str(time_until_next_milestone) - channel_data["next_milestone"] = str(next_milestone) - except OverflowError: - channel_data["next_milestone_date"] = "N/A" - channel_data["days_until_next_milestone"] = "N/A" - channel_data["next_milestone"] = "N/A" - return jsonify(channel_data) -@app.route("/api/announcement") -def api_announcement(): - announcement_data = {"message": "None", "show_message": False} - return jsonify(announcement_data) - -@app.errorhandler(404) -def not_found(error): - return jsonify(error=str(error)), 404 - -if __name__ == "__main__": - app.run(debug=True) diff --git a/backend/config.ini b/backend/config.ini deleted file mode 100644 index 0110dea..0000000 --- a/backend/config.ini +++ /dev/null @@ -1,31 +0,0 @@ -[SQL] -; Database connection details -host = localhost -user = root -password = root -database = Nijitrack_Dev - -[TABLES] -; Names of the tables that should be created in the database - -; Current subscriber data -live = subscriber_data -historical = subscriber_data_historical -daily = 24h_historical - -[API] -holodex = HOLODEXAPI -youtube = YOUTUBEAPI - -[PATH] -root_html = stats - -; Website details -homepage = https://nijitracker.com -title = Nijitracker -description = A site that tracks the historical subscriber data for Nijisanji affiliated livers -icon = https://raw.githubusercontent.com/pinapelz/NijiTrack/master/assets/icon.png -timezone = PST -footer_message = This is a demo of Nijitrack, a way to track historical subscriber data for any set of channels on YouTube.
This webpage is not affiliated with ANYCOLOR or any of the channels listed here in any way
Date Started: 2023-03-26 - - diff --git a/backend/data/channels.txt b/backend/data/channels.txt deleted file mode 100644 index 2b86946..0000000 --- a/backend/data/channels.txt +++ /dev/null @@ -1,4 +0,0 @@ -# DELETE THIS: [Add new line delineated channels here]. One each line -# DELETE THIS: ChannelName: -# DELETE THIS: [Add new line delineated channels here]. One each line with channel id and name seperated by a colon -# DELETE THIS: Channel_ID:Channel_Name diff --git a/backend/data/exclude_channel.txt b/backend/data/exclude_channel.txt deleted file mode 100644 index 2c01310..0000000 --- a/backend/data/exclude_channel.txt +++ /dev/null @@ -1,9 +0,0 @@ -UCRW3qJuYP8aODl7_5TXn_nw -UCx7aPMHI_kK1CP6zSvilHSA -UCL0_lYZwyhbcNX-feoyCcJw -UCkDfBt3R64R2rRIrAQwldeQ -UCa_EIfw5I5uDlwsK6sc-83Q -UC3-Rfh_Ek-s6EUWS4fT5VPw -UCUC1EIq0MtF-kctHPtQzIjQ -UCvmSVAwFc5MJwQowsDvCTuw -UClpOEZoIYhxIY6n5NNGaPbA \ No newline at end of file diff --git a/backend/data/last_refresh.txt b/backend/data/last_refresh.txt deleted file mode 100644 index 0f7ae5c..0000000 --- a/backend/data/last_refresh.txt +++ /dev/null @@ -1 +0,0 @@ -2023-11-26 \ No newline at end of file diff --git a/backend/decorators.py b/backend/decorators.py deleted file mode 100644 index bc0b420..0000000 --- a/backend/decorators.py +++ /dev/null @@ -1,16 +0,0 @@ -import time - - -def log(message: str): - def decorator(func): - def wrapper(*args, **kwargs): - print("TASK: " + message) - start = time.time() - result = func(*args, **kwargs) - end = time.time() - print(f"COMPLETE: {message} {round(end - start, 3)} seconds") - return result - - return wrapper - - return decorator diff --git a/backend/fileutil.py b/backend/fileutil.py deleted file mode 100644 index 325ee0f..0000000 --- a/backend/fileutil.py +++ /dev/null @@ -1,93 +0,0 @@ -import os.path -import urllib.request -import json -import time -import configparser - - -def _read_file(path: str, lines=True) -> list: - # reads a file and returns a list of lines - with open(path, "r", encoding="utf-8") as file: - if not lines: - return file.read() - return file.read().splitlines() - - -def get_excluded_channels(): - # gets excluded channels from exclude_channel.txt - if not os.path.exists(os.path.join("data", "exclude_channel.txt")): - open(os.path.join("data", "exclude_channel.txt"), "w").close() - excluded_channels = _read_file(os.path.join("data", "exclude_channel.txt")) - return excluded_channels - -def update_excluded_channels(channel_ids: list): - # add to exclude_channel.txt if not already there - excluded_channels = get_excluded_channels() - for channel_id in channel_ids: - if channel_id not in excluded_channels: - excluded_channels.append(channel_id) - with open(os.path.join("data", "exclude_channel.txt"), "w", encoding="utf-8") as file: - for channel_id in excluded_channels: - file.write(channel_id + "\n") - -def save_local_channels(data: list, path: str = "data"): - """ - Save the channel names and ids locally for when the API is down - """ - path = os.path.join(path, "channels.txt") - excluded_channels = get_excluded_channels() - if not os.path.exists(path): - os.makedirs(os.path.dirname(path), exist_ok=True) - open(path, "w").close() - with open(path, "w", encoding="utf-8") as file: - for channel in data: - if channel["id"] in excluded_channels: - continue - file.write(f"{channel['id']},{channel['english_name']}\n") - - -def get_local_channels(path: str = "data"): - """ - Get the channel names and ids locally for when the API is down - """ - path = os.path.join(path, "channels.txt") - if not os.path.exists(path): - raise Exception("Local channel data not found") - with open(path, "r", encoding="utf-8") as file: - rows = file.read().splitlines() - return [tuple(row.split(":")) for row in rows] - - -def update_data_files(url: str) -> None: - # Updates the local txt channel data stored in data folder - if not os.path.exists(os.path.join("data", "channels.txt")): - open(os.path.join("data", "channels.txt"), "w").close() - urllib.request.urlretrieve( - url + "channels.txt", os.path.join("data", "channels.txt") - ) - # downloaded txt file from url and write to channels.txt - - if not os.path.exists(os.path.join("data", "exclude_channel.txt")): - open(os.path.join("data", "exclude_channel.txt"), "w").close() - urllib.request.urlretrieve( - url + "exclude_channel.txt", os.path.join("data", "exclude_channel.txt") - ) - - -def load_config(ini_filepath: str) -> dict: - config_object = configparser.ConfigParser() - file = open(ini_filepath, "r") - config_object.read_file(file) - output_dict = {} - sections = config_object.sections() - for section in sections: - output_dict[section] = {} - for key in config_object[section]: - output_dict[section][key] = config_object[section][key] - return output_dict - -def load_json_file(json_file_path: str) -> dict: - with open(json_file_path, "r", encoding="utf-8") as file: - return json.load(file) - - diff --git a/backend/graph.py b/backend/graph.py deleted file mode 100644 index f7d8716..0000000 --- a/backend/graph.py +++ /dev/null @@ -1,35 +0,0 @@ -import plotly.graph_objects as go -import plotly.express as px -import pandas as pd -import warnings -from member_colors import member_color_map -import random - -def plot_subscriber_count_over_time(server, table_name, gtitle="Subscriber Count Over Time for Phase Connect Members", - overrideQuery=None, markers="lines", exclude_channels=[]): - warnings.filterwarnings('ignore') # Ignore pandas warning regarding pyodbc - query = f"SELECT name, subscriber_count, timestamp, channel_id FROM {table_name} ORDER by timestamp DESC" if overrideQuery is None else overrideQuery - df = pd.read_sql_query(query, server.get_connection()) - groups = df.groupby("name") - fig = go.Figure() - config = dict({'responsive': True, 'displaylogo': False, 'modeBarButtonsToAdd': ['pan2d', 'zoomIn2d', 'zoomOut2d']}) - - for channel, group in groups: - if len(exclude_channels) != 0 and group['channel_id'].iloc[0] in exclude_channels: - continue - color = None - color = member_color_map.get(channel, '#' + ''.join(random.choices('0123456789ABCDEF', k=6))) - - fig.add_trace(go.Scattergl( - x=group["timestamp"], y=group["subscriber_count"], name=channel, mode=markers, - showlegend=True, line=dict(color=color))) - - fig.update_layout( - title={'text': gtitle, 'x': 0.5, 'xanchor': 'center', - 'yanchor': 'top', 'font': {'family': 'Droid Sans', 'size': 30}}, - xaxis_title="Date", - yaxis_title="Subscribers", - legend=dict(font=dict(size=16), title=dict(text="Channels")), - height=950, - ) - return fig.to_html(config=config) diff --git a/backend/member_color.py b/backend/member_color.py deleted file mode 100644 index 37149d7..0000000 --- a/backend/member_color.py +++ /dev/null @@ -1,30 +0,0 @@ -# Add specific colors for traces here. Below is an example from https://phase-tracker.com -member_color_map = { - 'Rinkou Ashelia': '#D985B3', - 'Utatane Nasa': '#C69E90', - 'Pipkin Pippa': '#E78CA3', - 'Maemi Tenma': '#A99FAC', - 'Hakushika Iori': '#60C1F1', - 'Fujikura Uruka': '#687199', - 'Shisui Michiru': '#3D2539', - 'Remilia Nephys': '#723838', - 'Chisaka Airi': '#966D7A', - 'Amanogawa Shiina': '#5F5675', - 'Himemiya Rie': '#D168A2', - 'Erina Makina': '#3D4E68', - 'Komachi Panko': '#CCA3A3', - 'Kaneko Lumi': '#C8B8A0 ', - 'Ember Amane': '#ADAEFF', - 'Dizzy Dokuro': '#223268', - 'Jelly Hoshiumi': '#A9D1E6', - 'Saya Sairroxs': '#723838', - 'Runie Ruse': '#FF96B7', - 'Muu Muyu': '#DFAFED', - 'Hikanari Hina': '#6A8FB1', - 'Eimi Isami': '#FF9400', - 'Ayase Yuu': '#292A33', - 'Kokoromo Memory': '#FFB1D5', - 'Kaminari Clara': '#91BCC6', - 'Kannagi Loki': '#982428', - 'Fuura Yuri': '#000000' -} diff --git a/backend/nijitrack.py b/backend/nijitrack.py deleted file mode 100644 index f573bda..0000000 --- a/backend/nijitrack.py +++ /dev/null @@ -1,176 +0,0 @@ -from datetime import datetime - -import fileutil as fs -from sql.pg_handler import PostgresHandler -from webapi.holodex import HolodexAPI -from webapi.youtube import YouTubeAPI -from b2sdk.v2 import * -import graph -from decorators import * -import argparse -import os -import pytz -from dotenv import load_dotenv - -load_dotenv() - -DATA_SETTING = fs.load_json_file("sql_table_config.json") -CONFIG = fs.load_config("config.ini") - -def create_database_connection(): - """ - Creates a database connection using the environment variables - :param: auth_append: str = "" - If you want to use a different set of variables for persisitance of sessions - """ - hostname = os.environ.get("POSTGRES_HOST") - user = os.environ.get("POSTGRES_USER") - password = os.environ.get("POSTGRES_PASSWORD") - database = os.environ.get("POSTGRES_DATABASE") - return PostgresHandler(host_name=hostname, username=user, password=password, database=database, port=5432) - -@log("Initializing Database") -def initialize_database(server: PostgresHandler): - server.create_table(name = CONFIG["TABLES"]["live"], column = DATA_SETTING["LIVE_COLUMNS"]) - server.create_table(name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["HISTORICAL_COLUMNS"]) - server.create_table(name = CONFIG["TABLES"]["daily"], column = DATA_SETTING["DAILY_COLUMNS"]) - - -@log("Inserting Live Data into Database") -def record_subscriber_data(data: list, force_refresh: bool = False): - def transform_sql_string(string: str) -> str: - return string.encode("ascii", "ignore").decode().replace("'", "''") - def record_diff_data(data_tuple: tuple, refresh_daily: bool): - if not server.check_row_exists(CONFIG["TABLES"]["daily"], "channel_id", channel_id): - # data_tuple = (channel_id, pfp, channel_name, sub_count, time.strftime('%Y-%m-%d %H:%M:%S')) - server.insert_row(CONFIG["TABLES"]["daily"], DATA_SETTING["DAILY_HEADER"], (data_tuple[0], data_tuple[3])) - server.insert_row(table_name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["HISTORICAL_HEADER"], data=data_tuple) - return - elif refresh_daily: - server.update_row(CONFIG["TABLES"]["daily"], "channel_id", channel_id, "sub_diff", sub_count) - server.insert_row(table_name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["HISTORICAL_HEADER"], data=data_tuple) - - def check_diff_refresh(): - last_updated = server.get_most_recently_added_row_time(CONFIG["TABLES"]["historical"])[0] - if not last_updated: - print("Failed to get the most recently added row time.") - return False - last_updated = pytz.timezone('US/Pacific').localize(last_updated) - utc_now = datetime.now(pytz.timezone('UTC')) - pst_now = utc_now.astimezone(pytz.timezone('US/Pacific')) - time_diff = pst_now - last_updated - if time_diff.days >= 1: - return True - elif time_diff.days == 0 and time_diff.seconds >= 85800: - return True - else: - print("Skipping Daily Refresh. It has not been a day yet") - return False - exclude_channels = fs.get_excluded_channels() - if force_refresh: - refresh_daily = True - else: - refresh_daily = check_diff_refresh() - for channel in data: - channel_id = channel["id"] - if channel_id in exclude_channels: - continue - pfp = channel["photo"] - sub_count = channel["subscriber_count"] - channel_name = channel["english_name"] - sub_org = channel["group"] - video_count = channel["video_count"] - if channel_name is None: - channel_name = channel["name"] - if sub_org is None: - sub_org = "Unknown" - channel_name = transform_sql_string(channel_name) - utc_now = datetime.now(pytz.timezone('UTC')) - pst_now = utc_now.astimezone(pytz.timezone('US/Pacific')) - formatted_time = pst_now.strftime('%Y-%m-%d %H:%M:%S') - data_tuple = (channel_id, pfp, channel_name, sub_count, sub_org, video_count, formatted_time) - historical_data_tuple = (channel_id, pfp, channel_name, sub_count, formatted_time) - server.insert_row(table_name = CONFIG["TABLES"]["live"], column = DATA_SETTING["LIVE_HEADER"], data=data_tuple) - record_diff_data(historical_data_tuple, refresh_daily) - - -@log("Running Holodex Generation") -def holodex_generation(server: PostgresHandler, force_refresh: bool = False): - """ - Generates the data from the Holodex API - """ - holodex_organizations = DATA_SETTING["HOLODEX_ORGS"].split(",") - server.clear_table(CONFIG["TABLES"]["live"]) - server.reset_auto_increment(CONFIG["TABLES"]["live"]) - holodex = HolodexAPI(os.environ.get("HOLODEX_KEY"), organization="Phase%20Connect") - for organization in holodex_organizations: - holodex.set_organization(organization) - subscriber_data = holodex.get_subscriber_data() - record_subscriber_data(subscriber_data, force_refresh) - return holodex.get_generated_channel_data(), holodex.get_inactive_channels() - -@log("Running YouTube Generation") -def youtube_generation(server: PostgresHandler): - """ - Generates the data from the YouTube API - """ - ytapi = YouTubeAPI(os.environ.get("YOUTUBE_API_KEY")) - server.clear_table(CONFIG["TABLES"]["live"]) - server.reset_auto_increment(CONFIG["TABLES"]["live"]) - data = ytapi.get_data_all_channels(fs.get_local_channels()) - record_subscriber_data(data) - return data - -def combine_excluded_channel_ids(inactive_channel_data: list, excluded_channels: list): - """ - Combines the local excluded channels with the inactive channels from the API - """ - channel_ids = [] - for inactive_channel in inactive_channel_data: - if inactive_channel in excluded_channels: - continue - channel_ids.append(inactive_channel) - return channel_ids - -def uploadFileToBucket(filepath: str) -> bool: - try: - info = InMemoryAccountInfo() - b2_api = B2Api(info) - application_key_id = os.environ.get("B2_APP_ID") - application_key = os.environ.get("B2_APP_KEY") - file_info = {'how': 'good-file'} - b2_api.authorize_account("production", application_key_id, application_key) - b2_file_name = "graph.html" - bucket = b2_api.get_bucket_by_name("vtuber-rabbit-hole-archive") - bucket.upload_local_file(local_file=filepath, file_name=b2_file_name, file_info=file_info) - return True - except Exception as e: - print("An error occured while attempting to upload to B2") - print(e) - return False; - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="NijiTrack - A Subscriber Tracker") - parser.add_argument('--mode', choices=['yt', 'holodex'], help='Specify the data source to use (yt or holodex)') - parser.add_argument('--b2', action='store_true', help="Upload graph html to Backblaze B2") - parser.add_argument('--ff', action='store_true', help="Force a full refresh of all data (override daily refresh)") - args = parser.parse_args() - server = create_database_connection() - initialize_database(server) - if args.mode == 'yt': - print("Using YouTube API") - channel_data = youtube_generation(server) - inactive_channels = fs.get_excluded_channels() - else: - if args.ff: - print("Forcing a full refresh") - channel_data, inactive_channels = holodex_generation(server, force_refresh=True) - else: - channel_data, inactive_channels = holodex_generation(server) - fs.update_excluded_channels(inactive_channels) - graph_html = graph.plot_subscriber_count_over_time(server, CONFIG["TABLES"]["historical"], exclude_channels=combine_excluded_channel_ids(inactive_channels, fs.get_excluded_channels())) - with open("index.html", "w", encoding="utf-8") as file: - file.write(graph_html) - if args.b2: - uploadFileToBucket("index.html") - else: - print("Skipping B2 Upload") \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt deleted file mode 100644 index 5f8e9bb..0000000 --- a/backend/requirements.txt +++ /dev/null @@ -1,155 +0,0 @@ -anyio==4.3.0 -argon2-cffi==23.1.0 -argon2-cffi-bindings==21.2.0 -arrow==1.3.0 -asttokens==2.4.1 -async-lru==2.0.4 -attrs==23.2.0 -b2sdk==1.29.0 -Babel==2.14.0 -beautifulsoup4==4.12.3 -bleach==6.1.0 -blinker==1.7.0 -certifi==2023.11.17 -cffi==1.16.0 -charset-normalizer==3.3.2 -click==8.1.7 -colorama==0.4.6 -comm==0.2.1 -contourpy==1.2.0 -cycler==0.12.1 -debugpy==1.8.1 -decorator==5.1.1 -defusedxml==0.7.1 -docopt==0.6.2 -exceptiongroup==1.2.0 -executing==2.0.1 -fastjsonschema==2.19.1 -filelock==3.13.1 -Flask==3.0.0 -Flask-Cors==4.0.0 -fonttools==4.49.0 -fqdn==1.5.1 -fsspec==2024.2.0 -gitdb==4.0.11 -GitPython==3.1.42 -greenlet==3.0.1 -gunicorn==21.2.0 -h11==0.14.0 -httpcore==1.0.4 -httpx==0.27.0 -idna==3.6 -ipykernel==6.29.3 -ipython==8.22.2 -isoduration==20.11.0 -itsdangerous==2.1.2 -jedi==0.19.1 -Jinja2==3.1.3 -joblib==1.3.2 -json5==0.9.20 -jsonpointer==2.4 -jsonschema==4.21.1 -jsonschema-specifications==2023.12.1 -jupyter-events==0.9.0 -jupyter-lsp==2.2.4 -jupyter-server-mathjax==0.2.6 -jupyter_client==8.6.0 -jupyter_core==5.7.1 -jupyter_server==2.13.0 -jupyter_server_terminals==0.5.2 -jupyterlab==4.1.3 -jupyterlab_git==0.50.0 -jupyterlab_pygments==0.3.0 -jupyterlab_server==2.25.3 -kiwisolver==1.4.5 -logfury==1.0.1 -MarkupSafe==2.1.3 -matplotlib==3.8.3 -matplotlib-inline==0.1.6 -mistune==3.0.2 -mpmath==1.3.0 -mysql-connector-python==8.2.0 -nbclient==0.9.0 -nbconvert==7.16.2 -nbdime==4.0.1 -nbformat==5.9.2 -nest-asyncio==1.6.0 -networkx==3.2.1 -notebook_shim==0.2.4 -numpy==1.26.2 -nvidia-cublas-cu12==12.1.3.1 -nvidia-cuda-cupti-cu12==12.1.105 -nvidia-cuda-nvrtc-cu12==12.1.105 -nvidia-cuda-runtime-cu12==12.1.105 -nvidia-cudnn-cu12==8.9.2.26 -nvidia-cufft-cu12==11.0.2.54 -nvidia-curand-cu12==10.3.2.106 -nvidia-cusolver-cu12==11.4.5.107 -nvidia-cusparse-cu12==12.1.0.106 -nvidia-nccl-cu12==2.19.3 -nvidia-nvjitlink-cu12==12.3.101 -nvidia-nvtx-cu12==12.1.105 -overrides==7.7.0 -packaging==23.2 -pandas==2.1.3 -pandocfilters==1.5.1 -parso==0.8.3 -patsy==0.5.3 -pexpect==4.9.0 -pillow==10.2.0 -pip-review==1.3.0 -pipreqs==0.4.13 -platformdirs==4.2.0 -plotly==5.18.0 -prometheus_client==0.20.0 -prompt-toolkit==3.0.43 -protobuf==4.21.12 -psutil==5.9.8 -psycopg2-binary==2.9.9 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pycparser==2.21 -Pygments==2.17.2 -pyparsing==3.1.1 -python-dateutil==2.8.2 -python-dotenv==1.0.1 -python-json-logger==2.0.7 -pytz==2023.3.post1 -PyYAML==6.0.1 -pyzmq==25.1.2 -referencing==0.33.0 -requests==2.31.0 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rpds-py==0.18.0 -scikit-learn==1.3.2 -scipy==1.11.4 -seaborn==0.13.2 -Send2Trash==1.8.2 -six==1.16.0 -smmap==5.0.1 -sniffio==1.3.1 -soupsieve==2.5 -stack-data==0.6.3 -sympy==1.12 -tenacity==8.2.3 -terminado==0.18.0 -threadpoolctl==3.2.0 -tinycss2==1.2.1 -tomli==2.0.1 -torch==2.2.1 -tornado==6.4 -tqdm==4.66.1 -traitlets==5.14.1 -triton==2.2.0 -types-python-dateutil==2.8.19.20240106 -typing_extensions==4.8.0 -tzdata==2023.3 -uri-template==1.3.0 -urllib3==2.1.0 -wcwidth==0.2.13 -webcolors==1.13 -webencodings==0.5.1 -websocket-client==1.7.0 -Werkzeug==3.0.1 -yarg==0.1.9 diff --git a/backend/sql/pg_handler.py b/backend/sql/pg_handler.py deleted file mode 100644 index 74a9170..0000000 --- a/backend/sql/pg_handler.py +++ /dev/null @@ -1,169 +0,0 @@ -import psycopg2 -from psycopg2 import Error - -class PostgresHandler: - def __init__(self, username: str, password: str, host_name: str, port: int, database: str): - db_params = { - "dbname": database, - "user": username, - "password": password, - "host": host_name, - "port": port - } - self._connection = psycopg2.connect(**db_params) - print("Handler Success") - - def get_connection(self): - return self._connection - - def create_table(self, name: str, column: str): - cursor = self._connection.cursor() - cursor.execute(f'CREATE TABLE IF NOT EXISTS "{name}" ({column})') - self._connection.commit() - cursor.close() - - def clear_table(self, name: str): - cursor = self._connection.cursor() - cursor.execute(f"DELETE FROM {name}") - self._connection.commit() - cursor.close() - - def check_row_exists(self, table_name: str, column_name: str, value: str): - cursor = self._connection.cursor() - query = f'SELECT 1 FROM "{table_name}" WHERE {column_name} = %s' - cursor.execute(query, (value,)) - result = cursor.fetchone() - cursor.close() - - if result is not None: - return True - else: - return False - - def insert_row(self, table_name, column, data): - try: - cursor = self._connection.cursor() - placeholders = ', '.join(['%s'] * len(data)) - query = f'INSERT INTO "{table_name}" ({column}) VALUES ({placeholders})' - cursor.execute(query, data) - self._connection.commit() - print("Data Inserted:", data) - except Error as err: - self._connection.rollback() - print("Error inserting data") - print(err) - if "duplicate key" not in str(err).lower(): - return False - return True - - def update_row(self, table_name: str, column: str, value: str, update_column: str, update_value: str): - try: - cursor = self._connection.cursor() - query = f'UPDATE "{table_name}" SET {update_column} = %s WHERE {column} = %s' - cursor.execute(query, (update_value, value)) - self._connection.commit() - print("Data Updated:", value, update_value) - except Error as e: - self._connection.rollback() - print(f"Failed to update row from {table_name} WHERE {column} is {value}") - print(e) - return False - return True - - def get_rows(self, table_name: str, column: str, value: str): - try: - cursor = self._connection.cursor() - query = f'SELECT * FROM "{table_name}" WHERE {column} = %s' - cursor.execute(query, (value,)) - result = cursor.fetchall() - return result - except Error as e: - self._connection.rollback() - print(f"Failed to fetch row from {table_name} WHERE {column} is {value}") - print(e) - return False - - def get_random_row(self, table_name: str, count: int, condition: str = None): - if condition is None: - condition = "1 = 1" - try: - cursor = self._connection.cursor() - query = f"SELECT * FROM {table_name} WHERE {condition} ORDER BY RANDOM() LIMIT {str(count)}" - cursor.execute(query) - result = cursor.fetchall() - return result - except Error as e: - self._connection.rollback() - print(f"Failed to select random rows from {table_name}") - print(e) - return False - - def check_health(self): - cursor = self._connection.cursor() - cursor.execute("SELECT 1") - result = cursor.fetchone() - cursor.close() - if result is not None: - return True - else: - return False - - def delete_row(self, table_name: str, column: str, value: str): - try: - cursor = self._connection.cursor() - query = f"DELETE FROM {table_name} WHERE {column} = %s" - cursor.execute(query, (value,)) - self._connection.commit() - print("Data Deleted:", value) - except Error as e: - self._connection.rollback() - print(f"Failed to delete row from {table_name} WHERE {column} is {value}") - print(e) - return False - return True - - def execute_query(self, query: str, data: tuple = None): - try: - cursor = self._connection.cursor() - if data is None: - cursor.execute(query) - else: - cursor.execute(query, data) - result = cursor.fetchall() - return result - except Error as e: - self._connection.rollback() - print(f"Failed to execute query: {query}") - print(e) - return False - - def reset_auto_increment(self, table_name: str): - try: - cursor = self._connection.cursor() - query = f"ALTER SEQUENCE {table_name}_id RESTART WITH 1" - cursor.execute(query) - self._connection.commit() - print("Auto Increment Reset") - except Error as e: - self._connection.rollback() - print(f"Failed to reset auto increment for {table_name}") - print(e) - return False - return True - - def get_most_recently_added_row_time(self, table_name: str): - try: - cursor = self._connection.cursor() - query = f"SELECT timestamp FROM {table_name} ORDER BY id DESC LIMIT 1" - cursor.execute(query) - result = cursor.fetchone() - return result - except Error as e: - self._connection.rollback() - print(f"Failed to get most recently added row from {table_name}") - print(e) - return False - - - def close_connection(self): - self._connection.close() \ No newline at end of file diff --git a/backend/sql/sql_handler.py b/backend/sql/sql_handler.py deleted file mode 100644 index 82f071d..0000000 --- a/backend/sql/sql_handler.py +++ /dev/null @@ -1,171 +0,0 @@ -import mysql.connector -from mysql.connector import Error, errorcode - - -class SQLHandler: - def __init__(self, host_name: str, user_name: str, user_password: str, database_name: str): - self.host_name = host_name - self.username = user_name - self.password = user_password - self.database_name = database_name - self.connection = self._create_server_connection( - host_name, user_name, user_password) - self._load_database(database_name) - - def _create_server_connection(self, host_name: str, user_name: str, user_password: str) -> mysql.connector: - connection = None - try: - connection = mysql.connector.connect(host=host_name, user=user_name, passwd=user_password) - print("MySQL Database connection successful") - except Error as err: - print(f"Error: '{err}'") - return connection - - def get_connection(self): - return self.connection - - def _create_database(self, cursor: str, database_name: str): - try: - cursor.execute( - f"CREATE DATABASE {database_name} DEFAULT CHARACTER SET 'utf8'") - except Error as err: - print(f"Failed creating database: {err}") - exit(1) - - def _load_database(self, database_name: str): - cursor = self.connection.cursor(buffered=True) - try: - cursor.execute(f"USE {database_name}") - print(f"Database {database_name} loaded successfully") - except Error as err: - print(f"Database {database_name} does not exist") - if err.errno == errorcode.ER_BAD_DB_ERROR: - self._create_database(cursor, database_name) - print(f"Database {database_name} created successfully") - self.connection.database = database_name - else: - print(err) - exit(1) - - def create_table(self, name: str, column: str): - cursor = self.connection.cursor(buffered=True) - try: - cursor.execute(f"CREATE TABLE {name} ({column})") - print(f"Table {name} created successfully") - except Error as err: - print(err) - - def insert_row(self, name: str, column: str, data: tuple): - cursor = self.connection.cursor(buffered=True) - try: - placeholders = ', '.join(['%s'] * len(data)) - query = f"INSERT INTO {name} ({column}) VALUES ({placeholders})" - cursor.execute(query, data) - self.connection.commit() - print("Data Inserted:", data, "into", name) - except Error as err: - print("Error inserting data") - print(err) - - - def clear_table(self, name: str): - cursor = self.connection.cursor(buffered=True) - try: - cursor.execute(f"DELETE FROM {name}") - self.connection.commit() - print("Table cleared successfully") - except Error as err: - print("Error clearing table") - print(err) - - def reset_auto_increment(self, name: str): - cursor = self.connection.cursor(buffered=True) - try: - cursor.execute(f"ALTER TABLE {name} AUTO_INCREMENT = 1") - self.connection.commit() - print("Table reset successfully") - except Error as err: - print("Error resetting table") - print(err) - - def copy_rows_to_new_table(self, name: str, new_name: str, column: str): - cursor = self.connection.cursor(buffered=True) - try: - cursor.execute( - f"INSERT INTO {new_name} ({column}) SELECT {column} FROM {name}") - cursor.execute( - f"ALTER TABLE {new_name} MODIFY COLUMN id INT AUTO_INCREMENT") - self.connection.commit() - print("Rows copied successfully") - except Error as err: - print("Error copying rows") - print(err) - - def drop_table(self, name: str): - cursor = self.connection.cursor(buffered=True) - try: - cursor.execute(f"DROP TABLE {name}") - self.connection.commit() - print("Table dropped successfully") - except Error as err: - print("Error dropping table") - print(err) - - def check_row_exists(self, name: str, column_name: str, value: str): - """ - Checks if a row exists in a table - """ - cursor = self.connection.cursor(buffered=True) - try: - cursor.execute(f"SELECT * FROM {name} WHERE {column_name} = '{value}'") - result = cursor.fetchone() - if result: - return True - else: - return False - except Error as err: - print("Error checking row") - print(err) - - def update_row(self, name: str, column_name: str, search_val: str, replace_col:str, new_value: str): - """ - Updates a row in a table - """ - cursor = self.connection.cursor(buffered=True) - try: - cursor.execute(f"UPDATE {name} SET {replace_col} = '{new_value}' WHERE {column_name} = '{search_val}'") - self.connection.commit() - print("Row updated successfully") - except Error as err: - print("Error updating row") - print(err) - - def execute_query(self, query: str, data: tuple = None): - cursor = self.connection.cursor(buffered=True) - if data: - try: - cursor.execute(query, data) - result = cursor.fetchall() - return result - except Error as err: - print("Error executing query") - print(err) - return None - try: - cursor.execute(query) - result = cursor.fetchall() - return result - except Error as err: - print("Error executing query") - print(err) - - def get_query_result(self, query: str): - cursor = self.connection.cursor(buffered=True) - try: - cursor.execute(query) - result = cursor.fetchall() - return result - except Error as err: - print("Error executing query") - print(err) - \ No newline at end of file diff --git a/backend/sql_table_config.json b/backend/sql_table_config.json deleted file mode 100644 index 8bc237f..0000000 --- a/backend/sql_table_config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "LIVE_COLUMNS": "id SERIAL PRIMARY KEY, channel_id VARCHAR(255), profile_pic VARCHAR(255), name VARCHAR(255), subscriber_count INT, suborg VARCHAR(255), video_count INT, timestamp TIMESTAMP", - "LIVE_HEADER": "channel_id, profile_pic, name, subscriber_count, suborg, video_count, timestamp", - "DAILY_COLUMNS": "id SERIAL PRIMARY KEY, channel_id VARCHAR(255), sub_diff INT", - "DAILY_HEADER": "channel_id, sub_diff", - "HISTORICAL_COLUMNS": "id SERIAL PRIMARY KEY, channel_id VARCHAR(255), profile_pic VARCHAR(255), name VARCHAR(255), subscriber_count INT, timestamp TIMESTAMP", - "HISTORICAL_HEADER": "channel_id, profile_pic, name, subscriber_count, timestamp", - "HOLODEX_ORGS": "Phase%20Connect" -} diff --git a/backend/webapi/holodex.py b/backend/webapi/holodex.py deleted file mode 100644 index 5f81892..0000000 --- a/backend/webapi/holodex.py +++ /dev/null @@ -1,75 +0,0 @@ -from webapi.web_api import WebAPI -from typing import Iterable - - -class HolodexAPI(WebAPI): - """ - Class for interacting with the Holodex API - """ - - def __init__(self,api_key: str = None,member_count: int = 300,organization: str = "Nijisanji"): - super().__init__(api_key=api_key, base_url="https://holodex.net/api/v2/") - self.member_count = member_count - self.organization = organization - self._inactive_channels = [] - self._channel_data = [] - - def get_subscriber_data(self) -> Iterable: - """ - Gets data for all channels in a particular organization - """ - members = self.member_count - data = [] - active_channels = [] - offset = 0 - while members > 0: - data += self._download_url( - f"channels?type=vtuber&offset={offset}&limit=100&org={self.organization}" - ) - members -= 100 - offset += 100 - for channel in data: - print("DEBUG: ", channel["id"]) - try: - channel["description"] = self.get_channel_description(channel["id"]) - if channel["inactive"]: - self._inactive_channels.append(channel["id"]) - continue - active_channels.append(channel) - except (KeyError, TypeError, ValueError): - print("DEBUG:","An error occured with parsing ", channel["id"], channel["name"]) - continue - self._channel_data = active_channels - return active_channels - - def get_view_count(self, channel_id: str) -> int: - """ - Gets the view count for a particular channel - """ - data = self._download_url(f"channels/{channel_id}") - return data["view_count"] - - def get_channel_description(self, channel_id: str) -> str: - """ - Gets the description for a particular channel - """ - data = self._download_url(f"channels/{channel_id}") - return data["description"] - - def set_organization(self, organization: str): - """ - Sets the organization for the API - """ - self.organization = organization - - def get_inactive_channels(self) -> list: - """ - Gets the list of inactive channels - """ - return self._inactive_channels - - def get_generated_channel_data(self) -> list: - """ - Gets the list of channel data - """ - return self._channel_data diff --git a/backend/webapi/web_api.py b/backend/webapi/web_api.py deleted file mode 100644 index 525994c..0000000 --- a/backend/webapi/web_api.py +++ /dev/null @@ -1,29 +0,0 @@ -import urllib.request -import json - - -class WebAPI: - """ - General class for interacting with web APIs - """ - - def __init__(self, api_key: str, base_url: str) -> None: - self.api_key = api_key - self.base_url = base_url - - def _download_url(self, query: str, header = 'X-APIKEY') -> dict: - """ - Downloads the URL and returns the result as a string - param: - query: str - the query to be appended to the base URL - """ - if self.api_key is None: - raise Exception("API key not set") - opener = urllib.request.build_opener() - opener.addheaders = [(header, self.api_key)] - urllib.request.install_opener(opener) - response = urllib.request.urlopen(self.base_url + query) - json_results = response.read() - r_obj = json.loads(json_results) - response.close() - return r_obj diff --git a/backend/webapi/youtube.py b/backend/webapi/youtube.py deleted file mode 100644 index a25f5ba..0000000 --- a/backend/webapi/youtube.py +++ /dev/null @@ -1,61 +0,0 @@ -from webapi.web_api import WebAPI - - -class YouTubeAPI(WebAPI): - """ - Class for interacting with the YouTube API - """ - - def __init__(self, api_key: str = None): - self.api_key = api_key - self.base_url = "https://www.googleapis.com/youtube/v3/" - - def _search_matching_id(self, id: str, data: list) -> dict: - """ - Searches for a info matching a given ID - param: - id: str - the ID to search for - """ - for entry in data: - if entry['id'] == id: - return entry - return None - - def get_data_all_channels(self, channel_tuples: list) -> list: - data = [] - members = len(channel_tuples) - request_chunks = [channel_tuples[i:i + 50] for i in range(0, members, 50)] - for chunk in request_chunks: - channel_ids = [x[0] for x in chunk] - channel_names = [x[1] for x in chunk] - request_string = ",".join(channel_ids) - stats = self._download_url( - f"channels?part=statistics&id={request_string}&key={self.api_key}") - snippet = self._download_url( - f"channels?part=snippet&id={request_string}&key={self.api_key}") - stats_list = stats['items'] - snippet_list = snippet['items'] - for i in range(len(stats_list)): - try: - # group/sub_org is used to further divide channels into subsets (sorta like teams) - # can't think of a better match via YouTube API rn other than customUrl - data_entry = {'english_name': channel_names[i], 'id': channel_ids[i], - 'subscriber_count': - self._search_matching_id(channel_ids[i], stats_list)['statistics']['subscriberCount'], - 'view_count': - self._search_matching_id(channel_ids[i], stats_list)['statistics']['viewCount'], - 'photo': - self._search_matching_id(channel_ids[i], snippet_list)['snippet']['thumbnails']['default']['url'], - 'description': - self._search_matching_id(channel_ids[i], snippet_list)['snippet']['description'], - 'group': - self._search_matching_id(channel_ids[i], snippet_list)['snippet']['customUrl'], - 'video_count': - self._search_matching_id(channel_ids[i], stats_list)['statistics']['videoCount'] - } - data.append(data_entry) - except TypeError: - print("Error NoneType: " + str(channel_ids[i])) - except KeyError: - print("Error KeyError: " + str(channel_ids[i])) - return data diff --git a/src/components/CompactTable/CompactTable.tsx b/src/components/CompactTable/CompactTable.tsx new file mode 100644 index 0000000..153d5ea --- /dev/null +++ b/src/components/CompactTable/CompactTable.tsx @@ -0,0 +1,9 @@ +interface CompactTableProps{ + subs: number[]; + milestone: string[]; + } + +const CompactTable = (props: CompactTableProps) => { + +} +export default CompactTable; \ No newline at end of file -- cgit v1.2.3