From 96019367e8f72eac26abd3b7a908c2b914bd1ae1 Mon Sep 17 00:00:00 2001 From: Pinapelz Date: Fri, 17 Nov 2023 13:24:42 -0800 Subject: v2: add initial Next JS files, remove static templates --- backend/.gitignore | 170 ++++++++++++++++++++++++++++++++++++ backend/app.py | 22 +++++ backend/config.ini | 31 +++++++ backend/data/channels.txt | 181 +++++++++++++++++++++++++++++++++++++++ backend/data/exclude_channel.txt | 111 ++++++++++++++++++++++++ backend/data/last_refresh.txt | 1 + backend/decorators.py | 16 ++++ backend/fileutil.py | 109 +++++++++++++++++++++++ backend/nijitrack.py | 104 ++++++++++++++++++++++ backend/requirements.txt | 21 +++++ backend/sql/sql_handler.py | 162 +++++++++++++++++++++++++++++++++++ backend/sql_table_config.json | 7 ++ backend/webapi/holodex.py | 75 ++++++++++++++++ backend/webapi/web_api.py | 29 +++++++ backend/webapi/youtube.py | 55 ++++++++++++ 15 files changed, 1094 insertions(+) create mode 100644 backend/.gitignore create mode 100644 backend/app.py create mode 100644 backend/config.ini create mode 100644 backend/data/channels.txt create mode 100644 backend/data/exclude_channel.txt create mode 100644 backend/data/last_refresh.txt create mode 100644 backend/decorators.py create mode 100644 backend/fileutil.py create mode 100644 backend/nijitrack.py create mode 100644 backend/requirements.txt create mode 100644 backend/sql/sql_handler.py create mode 100644 backend/sql_table_config.json create mode 100644 backend/webapi/holodex.py create mode 100644 backend/webapi/web_api.py create mode 100644 backend/webapi/youtube.py (limited to 'backend') diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 0000000..51ff7fb --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1,170 @@ +__pycache__ +config.json +.venv +.github +main.py +.idea +tables +stats +.vscode + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/backend/app.py b/backend/app.py new file mode 100644 index 0000000..51b0dbb --- /dev/null +++ b/backend/app.py @@ -0,0 +1,22 @@ +""" +Flask app for serving the static files +""" +from flask import Flask, send_file, send_from_directory, jsonify, abort +from flask_cors import CORS + +app = Flask(__name__) +CORS(app) + + +@app.route('/') +def main_page(): + return "We are offline at the moment" + + +@app.errorhandler(404) +def not_found(error): + return jsonify(error=str(error)), 404 + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/backend/config.ini b/backend/config.ini new file mode 100644 index 0000000..0110dea --- /dev/null +++ b/backend/config.ini @@ -0,0 +1,31 @@ +[SQL] +; Database connection details +host = localhost +user = root +password = root +database = Nijitrack_Dev + +[TABLES] +; Names of the tables that should be created in the database + +; Current subscriber data +live = subscriber_data +historical = subscriber_data_historical +daily = 24h_historical + +[API] +holodex = HOLODEXAPI +youtube = YOUTUBEAPI + +[PATH] +root_html = stats + +; Website details +homepage = https://nijitracker.com +title = Nijitracker +description = A site that tracks the historical subscriber data for Nijisanji affiliated livers +icon = https://raw.githubusercontent.com/pinapelz/NijiTrack/master/assets/icon.png +timezone = PST +footer_message = This is a demo of Nijitrack, a way to track historical subscriber data for any set of channels on YouTube.
This webpage is not affiliated with ANYCOLOR or any of the channels listed here in any way
Date Started: 2023-03-26 + + diff --git a/backend/data/channels.txt b/backend/data/channels.txt new file mode 100644 index 0000000..480388b --- /dev/null +++ b/backend/data/channels.txt @@ -0,0 +1,181 @@ +UC0g1AE0DOjBYnLhkgoRWN1w,Honma Himawari +UC0WwEfE-jOM2rzjpdfhTzZA,Aizono Manami +UC1QgXt46-GEvtNjEC1paHnw,Gwelu Os Gar +UC1vawzfbCnRpHT9SJ5pHlHw,Akagi Wen +UC1zFJrfEKvCixhsjNSb1toQ,Sister Claire +UC2OacIzd2UxGHRGhdHl1Rhw,Hayase Sou +UC3lNFeJiTq6L3UWoz4g1e-A,Uzuki Kou +UC47rNmkDcNgbOcM-2BwzJTQ,Millie Parfait +UC48jH1ul-6HOrcSSfoR02fQ,Yuuhi Riri +UC4l9gz3q65lTBFfFtW5LLeA,Watarai Hibari +UC_4tXjqecqox5Uc05ncxpxg,Shiina Yuika +UC4WvIIAo89_AzGUh1AZ6Dkg,Rosemi Lovelock +UC4yNIKGvy-YUrwYupVdLDXA,Ike Eveland +UC53UDnhAAYwvNO7j_2Ju1cQ,Dola +UC5dJFf4m-mEcoyJRfhBljoA,Seraph Dazzlegarden +UC5ek2GWKvUKFgnKSHuuCFrw,So Nagi +UC5qSx7KzdRwbsO1QmJc4d-w,Siska Leontyne +UC5yckZliCkuaEFbqzLBD7hQ,Reza Avanluna +UC6oDys1BGgBsIC3WhG1BovQ,Shizuka Rin +UC-6rZgmxZSIbq786j3RD5ow,Leos Vincent +UC6TfqY40Xt1Y0J-N18c85qQ,Azuchi Momo +UC6wvdADTJ88OfIbJYIpAaDA,Fuwa Minato +UC7Gb7Uawe20QyFibhLl1lzA,Luca Kaneshiro +UC7hffDQLKIEG-_zoAQkMIvg,Akira Ray +UC_82HBGtvwN1hcGeOGHzUBQ,Sorahoshi Kirame +UC8C1LLhBhf_E2IBPLSDJXlQ,Sukoya Kana +UC8Snw5i4eOJXEQqURAK17hQ,Rai Galilei +UC9EjSJ8pvxtvPdxLOElv73w,Makaino Ririmu +UC9V3Y3_uzU5e-usObb6IE1w,Hoshikawa Sara +UCA3WE2WRSpoIvtnoVGq4VAw,ZEA Cornelia +UCahgMxSIQ2zIRrPKhM6Mjvg,Mika Melatika +UCambvP8yxNDot4FzQc9cgiw,Usami Rito +UCAQDFeCTVdx90GtwohwjHzQ,Amagase Muyu +UCb6ObE-XGCctO3WrjRZC-cw,Luis Cammy +UCbc8fwhdUNlqi-J99ISYu4A,Belmond Banderas +UCBi8YaVyZpiKWN3_Z0dCTfQ,Akabane Youko +UCBiqkFJljoxAj10SoP2w2Cg,Fumino Tamaki +UCBURM8S4LH7cRZ0Clea9RDA,Reimu Endou +UCC7rRD6P7RQcx0hKv9RQP4w,Fura Kanato +UCcDDxnoQcezyTUzHg5uHaKg,Shikinagi Akira +UCCHH0nWYXFZmtDS_4tvMxHQ,Yang Nari +UCckdfYDGrjojJM28n5SHYrA,Vox Akuma +UCClwIqTUn5LDpFucHyaAhHg,Lee Roha +UCD-miitqNY3nyukJ4Fnf4_A,Tsukino Mito +UCdpUojq0KWZCN9bxXnZwz5w,Ars Almal +UCe22Bcwd_GCpTjLxn83zl7A,Ponto Nei +UCebT4Aq-3XWb5je1S1FvR_A,Todo Kohaku +UCeGendL8CO5RkffB6IFwHow,Seffyna +UCeK9HFcRZoTrvqcUCtccMoQ,Shibuya Hajime +UCe_p3YEuYJb8Np0Ip9dk-FQ,Asahina Akane +UCerkculBD7YLc_vOGrF7tKg,Matsukai Mao +UCeShTCVgZyq2lsBW9QwIJcw,Gundo Mirei +UCFgXWZOUZA2oYHNr6qDmsTQ,Scarle Yonaguni +UCfipDDn7wY-C-SoUChgxCQQ,Hayama Marin +UCfQVs_KuXeNAlGa3fb8rlnQ,Sakura Ritsuki +UCG0rzBZV_QMP4MtWg6IjhEA,Shu Yamino +UCg63a3lk6PNeWhVvMRM_mrQ,Onomachi Haruka +UCgA2jKRkqpY_8eysPUs8sjw,Petra Gurin +UC_GCs6GARLxEHxy1w40d6VQ,Ienaga Mugi +UCGEMpMpll4otCdnnxQ9paFg,SakaYui +UCggO2c1unS-oLwTLT0ICywg,Kotoka Torahime +UCGhqxhovNfaPBpxfCruy9EA,Fulgur Ovid +UCgIfLpQvelloDi8I0Ycbwpg,Hyakumantenbara Salome +UCgmFrRcyH7d1zR9sIVQhFow,Lauren Iroas +UCGw7lrT-rVZCWHfdG9Frcgg,Genzuki Tojiro +UCGYAYLDE7TZiiC8U6teciDQ,Hakase Fuyuki +UCHBhnG2G-qN0JrrWmMO2FTA,Shellin Burgundy +UChJ5FTsHOu72_5OVx0rvsvQ,Uki Violeta +UCHjeZylSgXDSnor8wUnwU_g,Bonnivier Pranaja +UCHK5wkevfaGrPr7j3g56Jmw,Seto Miyako +UChKXd7oqD18qiIYBoRIHTlw,Meloco Kyoran +UChUJbHiTVeGrSkTdBzVfNCQ,Joe Rikiichi +UCHVXbQzkl3rDfsXWo8xi2qw,Ange Katrina +UCHX7YpFG8rVwhsHCx34xt7w,Yukishiro Mahiro +UCiA-trSZfB0i92V_-dyDqBw,Kuramochi Meruto +UCIBj1-d71vKjRftiauF50pg,Hyona Elatiora +UCIeSUTOTkF9Hs7q3SGcO-Ow,Elira Pendora +UCIG9rDtgR45VCZmYnd-4DUw,Ratna Petit +UCijNnZ-6m8g85UGaRAWuw7g,Nagisa Arcinia +UCIM92Ok_spNKLVB5TsgwseQ,Mysta Rias +UCiSRx1a2k-0tOg-fs6gAolQ,Asuka Hina +UCivwPlOp0ojnMPZj5pNOPPA,Sophia Valentine +UCIytNcoz4pWzXfLda0DoULQ,Ex Albio +UCjFu-9GHnabzSFRAYm1B9Dw,Etna Crimson +UCjlmCrq4TP1I4xguOtJ-31w,Debidebi Debiru +UCJubINhCcFXlsBwnHp0wl_g,Maimoto Keisuke +UCk5r533QVMgJUdWwqegH2TA,Azura Cecillia +UCkieJGn3pgJikVW8gmMXE2w,Nina Kosaka +UCkIimWZ9gBJRamKF0rmPU8w,Amamiya Kokoro +UCkL9OLKjIQbKk2CztbpOCFg,Riksa Dhirendra +UCKMYISTJAQ8xTplUPHiABlA,Yashiro Kizuku +UCkngxfPbmGyGl_RIq4FA3MQ,Nishizono Chigusa +UCKu59gTZ_rdEmerdx5rV4Yg,Ren Zotto +UCl1oLKcAq93p-pwKfDGhiYQ,Emma★August +UCL34fAoFim9oHLbVzMKFavQ,Yorumi Rena +UCLjx3lqIkYkPCBJop8czJ2A,Ban Hada +UCllKI7VjyANuS1RXatizfLQ,Yamagami Karuta +UCLO9QDxVL4bnvRRsz6K4bsQ,Yuuki Chihiro +UCL_O_HXgLJx3Auteer0n0pA,Suo Sango +UClrQ7xhRBxS_v_-WuudGKmA,Kaburaki Roco +UClS6k3w1sPwlVFqK3-yID5A,Ryu Hari +UCmeyo5pRj_6PXG-CsGUuWWg,Kuroi Shiba +UCmovZ2th3Sqpd00F5RdeigQ,Kagami Hayato +UCmUjjW5zF1MMOhYUwwwQv9Q,Ushimi Ichigo +UCmZ1Rbthn-6Jm_qOGjYsh5A,Ibrahim +UCMzVa7B8UEdrvUGsPmSgyjA,Derem Kado +UCN68LoM3khS4gdBMiWJO8WA,Aia Amare +UCnRQYHTnRLSF0cLJwMnedCg,Aiba Uiha +UCNW1Ex0r6HsWRD4LCtPwvoQ,Saegusa Akina +UCnzZmBOSrQf2wDBbnsDajVw,Oh Jiyu +UCo2N7C-Z91waaR6lF3LL_jw,Kaida Haru +UCo7TRj3cS-f_1D9ZDmuTsjw,Machita Chima +UCO8WcDsF5znr-qsXlzZNpqg,Ver Vermillion +UC-o-E6I3IC2q8sAoAuM6Umg,Naraka +UCoJ0Ct-jdas4cLPpSp06gZg,Xia Ekavira +UCoM_XmK45j504hfUWvN06Qg,Naruse Naru +UCoWH3sDpeXG1aXmOxveX4KA,Nara Haramaung +UCoztvTULBYd3WmStqYeoHcA,Sasaki Saku +UCP4nMSTdwU1KqYWu3UH5DHQ,Pomu Rainpuff +UCpJtk0myFr5WnyfsmnInP-w,Hana Macchia +UCpNH2Zk2gw3JBjWAKSyZcQQ,Eli Conifer +UCpnvhOIJ6BN-vPkYU9ls-Eg,Suzuya Aki +UCpRXCTyNNa-MnjhK6gisnRw,Gaon +UCPvGypSgfDkVe7JG2KygK7A,Rindou Mikoto +UCpzxZW5kghGnO5TmAFJQAVw,Aster Arcadia +UCQ1zGxHrfEmmW4CPpBx9-qw,Alban Knox +UCqjTqdVlvIipZXIKeCkHKUA,Oliver Evans +UCqXxS-9x9Ha_UiH6hG4kh5Q,Hibachi Mana +UCR6qhsLpn62WVxCBK1dkLow,Enna Alouette +UCRcLAVTbmx2-iNcXSsupdNA,Kurusu Natsume +UCrhhJPNsOqzNIkUfTABoSpg,Ha Yun +UCRm6lqtdxs_Qo6HeL-SRQ-w,Lain Paterson +UCRqBKoKuX30ruKAq05pCeRQ,Kitakoji Hisui +UCrR7JxkbeLY82e8gsj_I0pQ,Amicia Michella +UCRV9d6YCYIMUszK-83TwxVA,Todoroki Kyoko +UCRWOdwLRsenx2jLaiCAIU4A,Amemori Sayo +UCryOPk2GZ1meIDt53tL30Tw,Suzuki Masaru +UCsb-1aJgiJXJH2feV-zlZRw,Kyo Kaneko +UCSFCh5NL4qXrAy9u-u2lX3g,Kuzuha +UCsFn_ueskBkMCEyzCEqAOvg,Hanabatake Chaika +UCsg-YqdqQ-KFF0LNk23BY4A,Higuchi Kaede +UCspv01oxUFf_MTSipURRhkA,Kanae +UCS-XXTgVkotkbkDnGEprXpg,Mashiro +UCt0clH12Xk1-Ej5PXKGfdPA,Mononobe Alice +UCt5-0i4AVHXaWJrL8Wql3mw,Ryushen +UCtAvQ5U0aXyKwm2i4GqFgJg,Harusaki Air +UCtHY-tP0dyykhTRMmnfPs_g,Umise Yotsuha +UCTIE7LM5X15NVugV7Krp9Hw,Yumeoi Kakeru +UCtLfA_qUqCJtjXJM2ZR_keg,Ishigami Nozomi +UCtnO2N4kPTXmyvedjGWdx3Q,Levi Elipha +UCtpB6Bvhs1Um93ziEDACQ8g,Morinaka Kazaki +UCUc8GZfFxtmk7ZwSO7ccQ0g,Nui Sociere +UCuep1JCrMvSxOGgGhBfJuYw,Furen E Lustario +UCu-J8uIXuLZh16gG-cT1naw,Finana Ryugu +UCUP8TmlO7NNra88AMqGU_vQ,Koshimizu Toru +UCu-rV2gPtJ-CsGxe71z_BrQ,Igarashi Rika +UCUtKkGKef8BYMs3h-3zQm9A,Min Suha +UCuuAb_72QzK0M1USPMEl1yw,Sonny Brisko +UCuvk5PilcvDECU7dDZhQiEw,Shirayuki Tomoe +UCUzJ90o1EjqUbk2pBAy0_aw,Gilzaren III +UCv1fFr156jc65EMiLbaLImw,Kenmochi Toya +UCV1xUwfM2v2oBtT3JNvic3w,Selen Tatsuki +UCV5ZZlLjk5MKGg3L0n0vbzw,Takamiya Rion +UCvmppcdYf4HOv-tFQhHHJMA,Moira +UCvzVB-EYuHFXHZrObB8a_Og,Yaguruma Rine +UCwaS8_S7kMiKA3izlTWHbQg,Maria Marionette +UCwokZsOK_uEre70XayaFnzA,Suzuka Utako +UCwrjITPwG4q71HzihV2C7Nw,Fumi +UCWRPqA0ehhWV4Hnp27PJCkQ,Shishido Akari +UCWz0CSYCxf4MhRKPDm220AQ,Kanda Shoichi +UCX88Pe54pxbJDSGIyGrzNdg,Na Sera +UCXRlIK3Cw_TJIQC5kSJJQMg,Inui Toko +UCXU7YYxy_iQd3ulXyO-zC2w,Fushimi Gaku +UCXW4MqCQn-jCaxlX-nn-BYg,Nagao Kei +UCy8P3o5XlMpJGQY4WugzdNA,Saiki Ittetsu +UCy91xBlY_Brh3bnHxKtjrrg,Doppio Dropscythe +UCYKP16oMX9KKPbrNgo_Kgag,Elu the Elf +UCyRkQSuhJILuGOuXk10voPg,Layla Alstroemeria +UCZ1xuCK1kNmn5RzPYIZop3w,Lize Helesta +UCZ5dNZsqBjBzbBl0l_IdmXg,Taka Radjiman +UCz_ZRw6ak4Foy8zZy0kEprQ,Hex Haywire diff --git a/backend/data/exclude_channel.txt b/backend/data/exclude_channel.txt new file mode 100644 index 0000000..314b524 --- /dev/null +++ b/backend/data/exclude_channel.txt @@ -0,0 +1,111 @@ +UCX7YkU9nEeaoZbkVLVajcMg +UCwi4P78SVunSYAGrvC9aKcw +UCNRh9kkByBTYLo0IJupnAug +C4Jyg9gFStHO8r5n4ya7XCQ +UCStzBFfjwFYb0qSYHnzFZvw +UCrhf6HYKnV6bxyaB6ooJzbw +UCfki3lMEF6SGBFiFfo9kvUA +UCtHFXfrn52juTqGBN4WbMVw +UCbLgcjfsUaCUgJh9SVit8kw +UC_D2DNy-KUNQJ_NGMppgmyg +UCxWcO9CLti4uouUIS5IIF-Q +UC4Jyg9gFStHO8r5n4ya7XCQ +UCz6vnIbgiqFT9xUcD6Bp65Q +UC-JSeFfovhNsEhftt1WHMvg +UC0PwyIlUefx1LGWjFI0QjMg +UCRzHROJUp7Wg900p_fXtJtQ +UC2NtHPaDUA5htYsjVrO8Gng +UCLSzgV37Dt24T8p3-TNiSLg +UC8vZcu6W-EJ6UYEy_C31c-A +UCh00mQw8BCrTbchYWGz4JTQ +UCWwwXXd_RPzj3LAyEfBslsg +UCINqoksO3CQPCt8a-mRe-Ew +UCF1JdALrXgub24weQpqDy9Q +UCvQIBipkXlXXVdGga9sn8dw +UCKQi12nOGZsJ5nOuCTHErmA +UCSUZugdxy9Wcrkp781cFt1w +UCTi_rzf5QIkXjhJjkbcAdTg +UCpfjQCCavrO-rnKaAaIF9dg +UCX7YkU9nEeaoZbkVLVajcMg +UCwi4P78SVunSYAGrvC9aKcw +UCNRh9kkByBTYLo0IJupnAug +C4Jyg9gFStHO8r5n4ya7XCQ +UCStzBFfjwFYb0qSYHnzFZvw +UCrhf6HYKnV6bxyaB6ooJzbw +UCfki3lMEF6SGBFiFfo9kvUA +UCtHFXfrn52juTqGBN4WbMVw +UCbLgcjfsUaCUgJh9SVit8kw +UC_D2DNy-KUNQJ_NGMppgmyg +UCxWcO9CLti4uouUIS5IIF-Q +UC4Jyg9gFStHO8r5n4ya7XCQ +UCz6vnIbgiqFT9xUcD6Bp65Q +UC-JSeFfovhNsEhftt1WHMvg +UC0PwyIlUefx1LGWjFI0QjMg +UCRzHROJUp7Wg900p_fXtJtQ +UC2NtHPaDUA5htYsjVrO8Gng +UCLSzgV37Dt24T8p3-TNiSLg +UC8vZcu6W-EJ6UYEy_C31c-A +UCh00mQw8BCrTbchYWGz4JTQ +UCWwwXXd_RPzj3LAyEfBslsg +UCINqoksO3CQPCt8a-mRe-Ew +UCF1JdALrXgub24weQpqDy9Q +UCvQIBipkXlXXVdGga9sn8dw +UCKQi12nOGZsJ5nOuCTHErmA +UCSUZugdxy9Wcrkp781cFt1w +UCTi_rzf5QIkXjhJjkbcAdTg +UCpfjQCCavrO-rnKaAaIF9dg +UC0lik9pHju6ONgkBh7N5wHw +UC1ZV7KBscK0EMoJKFu1DnDg +UC69URn8iP4u8D_zUp-IJ1sg +UC6oW4FXETgEGOFTxWmI2h5Q +UC6WU2SrnG019ucm_bdY6qxQ +UC8oPnditPSp5lZu45fnXWCA +UC9oudjCTHL2BfwxfDwI18eg +UC_a1ZYZ8ZTXpjg9xUY9sj8w +UC_aB_-PHLFHiP61djM0oOiQ +UCb5JxV6vKlYVknoJB8TnyYg +UCbLgcjfsUaCUgJh9SVit8kw +UCCJcCWrbQzz6eDhX4M1CUwg +UCCVwhI5trmaSxfcze_Ovzfw +UCF1JdALrXgub24weQpqDy9Q +UCFaDvgez8USXHiKidt0NtZg +UCfki3lMEF6SGBFiFfo9kvUA +UCfM_A7lE6LkGrzx6_mOtI4g +UCi6nV5Z2dzFuXBzLG3P9zqQ +UCIairB9UMDvqSKfMdv1jmjg +UCIJ9zP6gIkT8BB4Lz4UYPhA +UCJ6LH4jMNy0JN9RSThz1mMQ +UCjGE11ZnF0JSR8egVAwh-3A +UCks41vQN-hN-1KHmpZyPY3A +UCLpYMk5h1bq8_GAFVBgXhPQ +UCLSzgV37Dt24T8p3-TNiSLg +UCmWqYB6y8gSfPONWGspuOWQ +UCNUgrFCo2Hr_VXc9bEwjcHQ +UCOmjciHZ8Au3iKMElKXCF_g +UCP2o-o6u4uX3uq1hXspl0rg +UCqEp6RdtsMbUNrCdCswr6pA +UCqQV8xEBWd5SVZBLlYrS_5Q +UCRzHROJUp7Wg900p_fXtJtQ +UCSc_KzY_9WYAx9LghggjVRA +UCSlv7Z-4q7_7NRkzJB10A5Q +UCStzBFfjwFYb0qSYHnzFZvw +UCufQu4q65z63IgE4cfKs1BQ +UCuz0vzQgC8LRdS6lVV0UkUg +UCveZ9Ic1VtcXbsyaBgxPMvg +UCwcyyxn6h9ex4sMXGtpQE_g +UCwQ9Uv-m8xkE5PzRc7Bqx3Q +UCyqzU2nq7eGNi4kN0uHx7TA +UCZgRRTDMYwHpQ7sCl0aCp2Q +UCzMAP-oh5-pC8j6RlCPf26w +UC5qSx7KzdRwbsO1QmJc4d-w +UC5yckZliCkuaEFbqzLBD7hQ +UCA3WE2WRSpoIvtnoVGq4VAw +UCe_p3YEuYJb8Np0Ip9dk-FQ +UCeShTCVgZyq2lsBW9QwIJcw +UCIBj1-d71vKjRftiauF50pg +UCIM92Ok_spNKLVB5TsgwseQ +UCk5r533QVMgJUdWwqegH2TA +UCkieJGn3pgJikVW8gmMXE2w +UCoWH3sDpeXG1aXmOxveX4KA +UCrR7JxkbeLY82e8gsj_I0pQ +UCZ5dNZsqBjBzbBl0l_IdmXg diff --git a/backend/data/last_refresh.txt b/backend/data/last_refresh.txt new file mode 100644 index 0000000..ae97356 --- /dev/null +++ b/backend/data/last_refresh.txt @@ -0,0 +1 @@ +2023-11-17 \ No newline at end of file diff --git a/backend/decorators.py b/backend/decorators.py new file mode 100644 index 0000000..bc0b420 --- /dev/null +++ b/backend/decorators.py @@ -0,0 +1,16 @@ +import time + + +def log(message: str): + def decorator(func): + def wrapper(*args, **kwargs): + print("TASK: " + message) + start = time.time() + result = func(*args, **kwargs) + end = time.time() + print(f"COMPLETE: {message} {round(end - start, 3)} seconds") + return result + + return wrapper + + return decorator diff --git a/backend/fileutil.py b/backend/fileutil.py new file mode 100644 index 0000000..2aa2d6f --- /dev/null +++ b/backend/fileutil.py @@ -0,0 +1,109 @@ +import os.path +import urllib.request +import json +import time +import configparser + + +def _read_file(path: str, lines=True) -> list: + # reads a file and returns a list of lines + with open(path, "r", encoding="utf-8") as file: + if not lines: + return file.read() + return file.read().splitlines() + + +def get_excluded_channels(): + # gets excluded channels from exclude_channel.txt + if not os.path.exists(os.path.join("data", "exclude_channel.txt")): + open(os.path.join("data", "exclude_channel.txt"), "w").close() + excluded_channels = _read_file(os.path.join("data", "exclude_channel.txt")) + return excluded_channels + +def update_excluded_channels(channel_ids: list): + # add to exclude_channel.txt if not already there + excluded_channels = get_excluded_channels() + for channel_id in channel_ids: + if channel_id not in excluded_channels: + excluded_channels.append(channel_id) + with open(os.path.join("data", "exclude_channel.txt"), "w", encoding="utf-8") as file: + for channel_id in excluded_channels: + file.write(channel_id + "\n") + +def save_local_channels(data: list, path: str = "data"): + """ + Save the channel names and ids locally for when the API is down + """ + path = os.path.join(path, "channels.txt") + excluded_channels = get_excluded_channels() + if not os.path.exists(path): + os.makedirs(os.path.dirname(path), exist_ok=True) + open(path, "w").close() + with open(path, "w", encoding="utf-8") as file: + for channel in data: + if channel["id"] in excluded_channels: + continue + file.write(f"{channel['id']},{channel['english_name']}\n") + + +def get_local_channels(path: str = "data"): + """ + Get the channel names and ids locally for when the API is down + """ + path = os.path.join(path, "channels.txt") + if not os.path.exists(path): + raise Exception("Local channel data not found") + with open(path, "r", encoding="utf-8") as file: + rows = file.read().splitlines() + return [tuple(row.split(",")) for row in rows] + + +def check_diff_refresh(): + if not os.path.exists(os.path.join("data", "last_refresh.txt")): + with open( + os.path.join("data", "last_refresh.txt"), "w", encoding="utf-8") as file: + file.write(time.strftime("%Y-%m-%d")) + return True + with open(os.path.join("data", "last_refresh.txt"), "r", encoding="utf-8") as file: + last_refresh = file.read() + if last_refresh != time.strftime("%Y-%m-%d"): + with open( + os.path.join("data", "last_refresh.txt"), "w", encoding="utf-8" + ) as file: + file.write(time.strftime("%Y-%m-%d")) + return True + + +def update_data_files(url: str) -> None: + # Updates the local txt channel data stored in data folder + if not os.path.exists(os.path.join("data", "channels.txt")): + open(os.path.join("data", "channels.txt"), "w").close() + urllib.request.urlretrieve( + url + "channels.txt", os.path.join("data", "channels.txt") + ) + # downloaded txt file from url and write to channels.txt + + if not os.path.exists(os.path.join("data", "exclude_channel.txt")): + open(os.path.join("data", "exclude_channel.txt"), "w").close() + urllib.request.urlretrieve( + url + "exclude_channel.txt", os.path.join("data", "exclude_channel.txt") + ) + + +def load_config(ini_filepath: str) -> dict: + config_object = configparser.ConfigParser() + file = open(ini_filepath, "r") + config_object.read_file(file) + output_dict = {} + sections = config_object.sections() + for section in sections: + output_dict[section] = {} + for key in config_object[section]: + output_dict[section][key] = config_object[section][key] + return output_dict + +def load_json_file(json_file_path: str) -> dict: + with open(json_file_path, "r", encoding="utf-8") as file: + return json.load(file) + + diff --git a/backend/nijitrack.py b/backend/nijitrack.py new file mode 100644 index 0000000..30b31cc --- /dev/null +++ b/backend/nijitrack.py @@ -0,0 +1,104 @@ +import os +import time + +import fileutil as fs +from sql.sql_handler import SQLHandler +from webapi.holodex import HolodexAPI +from webapi.youtube import YouTubeAPI +from decorators import * +import argparse + + +CONFIG = fs.load_config("config.ini") +DATA_SETTING = fs.load_json_file("sql_table_config.json") + + +@log("Initializing Database") +def initialize_database(server: SQLHandler): + server.create_table(name = CONFIG["TABLES"]["live"], column = DATA_SETTING["LIVE_COLUMNS"]) + server.create_table(name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["LIVE_COLUMNS"]) + server.create_table(name = CONFIG["TABLES"]["daily"], column = DATA_SETTING["DAILY_COLUMNS"]) + + +@log("Inserting Live Data into Database") +def record_subscriber_data(data: list): + def transform_sql_string(string: str) -> str: + return string.encode("ascii", "ignore").decode().replace("'", "''") + def record_diff_data(data_tuple: tuple, refresh_daily: bool): + if not server.check_row_exists(CONFIG["TABLES"]["daily"], "channel_id", channel_id): + # data_tuple = (channel_id, pfp, channel_name, sub_count, time.strftime('%Y-%m-%d %H:%M:%S')) + server.insert_row(CONFIG["TABLES"]["daily"], DATA_SETTING["DAILY_HEADER"], (data_tuple[0], data_tuple[3])) + server.insert_row(name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["LIVE_HEADER"], data=data_tuple) + return + elif refresh_daily: + server.update_row(CONFIG["TABLES"]["daily"], "channel_id", channel_id, "sub_diff", sub_count) + server.insert_row(name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["LIVE_HEADER"], data=data_tuple) + + exclude_channels = fs.get_excluded_channels() + refresh_daily = fs.check_diff_refresh() + for channel in data: + channel_id = channel["id"] + if channel_id in exclude_channels: + continue + pfp = channel["photo"] + sub_count = channel["subscriber_count"] + channel_name = channel["english_name"] + if channel_name is None: + channel_name = channel["name"] + channel_name = transform_sql_string(channel_name) + data_tuple = (channel_id, pfp, channel_name, sub_count, time.strftime('%Y-%m-%d %H:%M:%S')) + server.insert_row(name = CONFIG["TABLES"]["live"], column = DATA_SETTING["LIVE_HEADER"], data=data_tuple) + record_diff_data(data_tuple, refresh_daily) + + +@log("Running Holodex Generation") +def holodex_generation(server: SQLHandler): + """ + Generates the data from the Holodex API + """ + holodex_organizations = DATA_SETTING["HOLODEX_ORGS"].split(",") + server.clear_table(CONFIG["TABLES"]["live"]) + server.reset_auto_increment(CONFIG["TABLES"]["live"]) + holodex = HolodexAPI(CONFIG["API"]["holodex"]) + for organization in holodex_organizations: + holodex.set_organization(organization) + record_subscriber_data(holodex.get_subscriber_data()) + return holodex.get_generated_channel_data(), holodex.get_inactive_channels() + +@log("Running YouTube Generation") +def youtube_generation(server: SQLHandler): + """ + Generates the data from the YouTube API + """ + ytapi = YouTubeAPI(CONFIG["API"]["youtube"]) + server.clear_table(CONFIG["TABLES"]["live"]) + server.reset_auto_increment(CONFIG["TABLES"]["live"]) + data = ytapi.get_data_all_channels(fs.get_local_channels()) + record_subscriber_data(data) + return data + +def combine_excluded_channel_ids(inactive_channel_data: list, excluded_channels: list): + """ + Combines the local excluded channels with the inactive channels from the API + """ + channel_ids = [] + for inactive_channel in inactive_channel_data: + if inactive_channel in excluded_channels: + continue + channel_ids.append(inactive_channel) + return channel_ids + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="NijiTrack - A Subscriber Tracker") + parser.add_argument('--mode', choices=['yt', 'holodex'], help='Specify the data source to use (yt or holodex)') + args = parser.parse_args() + server = SQLHandler(CONFIG["SQL"]["host"], CONFIG["SQL"]["user"], CONFIG["SQL"]["password"], CONFIG["SQL"]["database"]) + initialize_database(server) + if args.mode == 'yt': + print("Using YouTube API") + channel_data = youtube_generation(server) + inactive_channels = fs.get_excluded_channels() + else: + channel_data, inactive_channels = holodex_generation(server) + fs.update_excluded_channels(inactive_channels) \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..2f05f53 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,21 @@ +certifi==2023.7.22 +charset-normalizer==3.1.0 +docopt==0.6.2 +greenlet==2.0.2 +idna==3.4 +mysql-connector-python==8.0.32 +numpy==1.24.2 +packaging==23.0 +pandas==1.5.3 +pipreqs==0.4.12 +plotly==5.13.1 +protobuf==3.20.3 +python-dateutil==2.8.2 +pytz==2022.7.1 +requests==2.31.0 +six==1.16.0 +tenacity==8.2.2 +typing_extensions==4.5.0 +urllib3==1.26.15 +yarg==0.1.9 +Flask~=2.2.3 \ No newline at end of file diff --git a/backend/sql/sql_handler.py b/backend/sql/sql_handler.py new file mode 100644 index 0000000..9d1c10d --- /dev/null +++ b/backend/sql/sql_handler.py @@ -0,0 +1,162 @@ +import mysql.connector +from mysql.connector import Error, errorcode + + +class SQLHandler: + def __init__(self, host_name: str, user_name: str, user_password: str, database_name: str): + self.host_name = host_name + self.username = user_name + self.password = user_password + self.database_name = database_name + self.connection = self._create_server_connection( + host_name, user_name, user_password) + self._load_database(database_name) + + def _create_server_connection(self, host_name: str, user_name: str, user_password: str) -> mysql.connector: + connection = None + try: + connection = mysql.connector.connect(host=host_name, user=user_name, passwd=user_password) + print("MySQL Database connection successful") + except Error as err: + print(f"Error: '{err}'") + return connection + + def get_connection(self): + return self.connection + + def _create_database(self, cursor: str, database_name: str): + try: + cursor.execute( + f"CREATE DATABASE {database_name} DEFAULT CHARACTER SET 'utf8'") + except Error as err: + print(f"Failed creating database: {err}") + exit(1) + + def _load_database(self, database_name: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"USE {database_name}") + print(f"Database {database_name} loaded successfully") + except Error as err: + print(f"Database {database_name} does not exist") + if err.errno == errorcode.ER_BAD_DB_ERROR: + self._create_database(cursor, database_name) + print(f"Database {database_name} created successfully") + self.connection.database = database_name + else: + print(err) + exit(1) + + def create_table(self, name: str, column: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"CREATE TABLE {name} ({column})") + print(f"Table {name} created successfully") + except Error as err: + print(err) + + def insert_row(self, name: str, column: str, data: tuple): + cursor = self.connection.cursor(buffered=True) + try: + placeholders = ', '.join(['%s'] * len(data)) + query = f"INSERT INTO {name} ({column}) VALUES ({placeholders})" + cursor.execute(query, data) + self.connection.commit() + print("Data Inserted:", data, "into", name) + except Error as err: + print("Error inserting data") + print(err) + + + def clear_table(self, name: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"DELETE FROM {name}") + self.connection.commit() + print("Table cleared successfully") + except Error as err: + print("Error clearing table") + print(err) + + def reset_auto_increment(self, name: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"ALTER TABLE {name} AUTO_INCREMENT = 1") + self.connection.commit() + print("Table reset successfully") + except Error as err: + print("Error resetting table") + print(err) + + def copy_rows_to_new_table(self, name: str, new_name: str, column: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute( + f"INSERT INTO {new_name} ({column}) SELECT {column} FROM {name}") + cursor.execute( + f"ALTER TABLE {new_name} MODIFY COLUMN id INT AUTO_INCREMENT") + self.connection.commit() + print("Rows copied successfully") + except Error as err: + print("Error copying rows") + print(err) + + def drop_table(self, name: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"DROP TABLE {name}") + self.connection.commit() + print("Table dropped successfully") + except Error as err: + print("Error dropping table") + print(err) + + def check_row_exists(self, name: str, column_name: str, value: str): + """ + Checks if a row exists in a table + """ + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"SELECT * FROM {name} WHERE {column_name} = '{value}'") + result = cursor.fetchone() + if result: + return True + else: + return False + except Error as err: + print("Error checking row") + print(err) + + def update_row(self, name: str, column_name: str, search_val: str, replace_col:str, new_value: str): + """ + Updates a row in a table + """ + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"UPDATE {name} SET {replace_col} = '{new_value}' WHERE {column_name} = '{search_val}'") + self.connection.commit() + print("Row updated successfully") + except Error as err: + print("Error updating row") + print(err) + + def execute_query(self, query: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(query) + result = cursor.fetchall() + return result + except Error as err: + print("Error executing query") + print(err) + + def get_query_result(self, query: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(query) + result = cursor.fetchall() + return result + except Error as err: + print("Error executing query") + print(err) + \ No newline at end of file diff --git a/backend/sql_table_config.json b/backend/sql_table_config.json new file mode 100644 index 0000000..0e2e6e5 --- /dev/null +++ b/backend/sql_table_config.json @@ -0,0 +1,7 @@ +{ + "LIVE_COLUMNS": "id INT PRIMARY KEY AUTO_INCREMENT, channel_id VARCHAR(255), profile_pic VARCHAR(255), name VARCHAR(255), subscriber_count INT, timestamp DATETIME", + "LIVE_HEADER": "channel_id, profile_pic, name, subscriber_count, timestamp", + "DAILY_COLUMNS": "id INT PRIMARY KEY AUTO_INCREMENT, channel_id VARCHAR(255), sub_diff INT", + "DAILY_HEADER": "channel_id, sub_diff", + "HOLODEX_ORGS": "Nijisanji" +} diff --git a/backend/webapi/holodex.py b/backend/webapi/holodex.py new file mode 100644 index 0000000..5f81892 --- /dev/null +++ b/backend/webapi/holodex.py @@ -0,0 +1,75 @@ +from webapi.web_api import WebAPI +from typing import Iterable + + +class HolodexAPI(WebAPI): + """ + Class for interacting with the Holodex API + """ + + def __init__(self,api_key: str = None,member_count: int = 300,organization: str = "Nijisanji"): + super().__init__(api_key=api_key, base_url="https://holodex.net/api/v2/") + self.member_count = member_count + self.organization = organization + self._inactive_channels = [] + self._channel_data = [] + + def get_subscriber_data(self) -> Iterable: + """ + Gets data for all channels in a particular organization + """ + members = self.member_count + data = [] + active_channels = [] + offset = 0 + while members > 0: + data += self._download_url( + f"channels?type=vtuber&offset={offset}&limit=100&org={self.organization}" + ) + members -= 100 + offset += 100 + for channel in data: + print("DEBUG: ", channel["id"]) + try: + channel["description"] = self.get_channel_description(channel["id"]) + if channel["inactive"]: + self._inactive_channels.append(channel["id"]) + continue + active_channels.append(channel) + except (KeyError, TypeError, ValueError): + print("DEBUG:","An error occured with parsing ", channel["id"], channel["name"]) + continue + self._channel_data = active_channels + return active_channels + + def get_view_count(self, channel_id: str) -> int: + """ + Gets the view count for a particular channel + """ + data = self._download_url(f"channels/{channel_id}") + return data["view_count"] + + def get_channel_description(self, channel_id: str) -> str: + """ + Gets the description for a particular channel + """ + data = self._download_url(f"channels/{channel_id}") + return data["description"] + + def set_organization(self, organization: str): + """ + Sets the organization for the API + """ + self.organization = organization + + def get_inactive_channels(self) -> list: + """ + Gets the list of inactive channels + """ + return self._inactive_channels + + def get_generated_channel_data(self) -> list: + """ + Gets the list of channel data + """ + return self._channel_data diff --git a/backend/webapi/web_api.py b/backend/webapi/web_api.py new file mode 100644 index 0000000..525994c --- /dev/null +++ b/backend/webapi/web_api.py @@ -0,0 +1,29 @@ +import urllib.request +import json + + +class WebAPI: + """ + General class for interacting with web APIs + """ + + def __init__(self, api_key: str, base_url: str) -> None: + self.api_key = api_key + self.base_url = base_url + + def _download_url(self, query: str, header = 'X-APIKEY') -> dict: + """ + Downloads the URL and returns the result as a string + param: + query: str - the query to be appended to the base URL + """ + if self.api_key is None: + raise Exception("API key not set") + opener = urllib.request.build_opener() + opener.addheaders = [(header, self.api_key)] + urllib.request.install_opener(opener) + response = urllib.request.urlopen(self.base_url + query) + json_results = response.read() + r_obj = json.loads(json_results) + response.close() + return r_obj diff --git a/backend/webapi/youtube.py b/backend/webapi/youtube.py new file mode 100644 index 0000000..44d786f --- /dev/null +++ b/backend/webapi/youtube.py @@ -0,0 +1,55 @@ +from webapi.web_api import WebAPI + + +class YouTubeAPI(WebAPI): + """ + Class for interacting with the YouTube API + """ + + def __init__(self, api_key: str = None): + self.api_key = api_key + self.base_url = "https://www.googleapis.com/youtube/v3/" + + def _search_matching_id(self, id: str, data: list) -> dict: + """ + Searches for a info matching a given ID + param: + id: str - the ID to search for + """ + for entry in data: + if entry['id'] == id: + return entry + return None + + def get_data_all_channels(self, channel_tuples: list) -> list: + data = [] + members = len(channel_tuples) + request_chunks = [channel_tuples[i:i + 50] for i in range(0, members, 50)] + for chunk in request_chunks: + channel_ids = [x[0] for x in chunk] + channel_names = [x[1] for x in chunk] + request_string = ",".join(channel_ids) + stats = self._download_url( + f"channels?part=statistics&id={request_string}&key={self.api_key}") + snippet = self._download_url( + f"channels?part=snippet&id={request_string}&key={self.api_key}") + stats_list = stats['items'] + snippet_list = snippet['items'] + for i in range(len(stats_list)): + try: + data_entry = {'english_name': channel_names[i], 'id': channel_ids[i], + 'subscriber_count': + self._search_matching_id(channel_ids[i], stats_list)[ + 'statistics']['subscriberCount'], 'view_count': + self._search_matching_id(channel_ids[i], stats_list)[ + 'statistics']['viewCount'], 'photo': + self._search_matching_id(channel_ids[i], snippet_list)[ + 'snippet']['thumbnails']['default']['url'], 'description': + self._search_matching_id(channel_ids[i], snippet_list)[ + 'snippet']['description']} + data.append(data_entry) + except TypeError: + print("Error NoneType: " + str(channel_ids[i])) + except KeyError: + print("Error KeyError: " + str(channel_ids[i])) + return data -- cgit v1.2.3