diff options
Diffstat (limited to 'backend')
| -rw-r--r-- | backend/.gitignore | 170 | ||||
| -rw-r--r-- | backend/app.py | 22 | ||||
| -rw-r--r-- | backend/config.ini | 31 | ||||
| -rw-r--r-- | backend/data/channels.txt | 181 | ||||
| -rw-r--r-- | backend/data/exclude_channel.txt | 111 | ||||
| -rw-r--r-- | backend/data/last_refresh.txt | 1 | ||||
| -rw-r--r-- | backend/decorators.py | 16 | ||||
| -rw-r--r-- | backend/fileutil.py | 109 | ||||
| -rw-r--r-- | backend/nijitrack.py | 104 | ||||
| -rw-r--r-- | backend/requirements.txt | 21 | ||||
| -rw-r--r-- | backend/sql/sql_handler.py | 162 | ||||
| -rw-r--r-- | backend/sql_table_config.json | 7 | ||||
| -rw-r--r-- | backend/webapi/holodex.py | 75 | ||||
| -rw-r--r-- | backend/webapi/web_api.py | 29 | ||||
| -rw-r--r-- | backend/webapi/youtube.py | 55 |
15 files changed, 1094 insertions, 0 deletions
diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 0000000..51ff7fb --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1,170 @@ +__pycache__ +config.json +.venv +.github +main.py +.idea +tables +stats +.vscode + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/backend/app.py b/backend/app.py new file mode 100644 index 0000000..51b0dbb --- /dev/null +++ b/backend/app.py @@ -0,0 +1,22 @@ +""" +Flask app for serving the static files +""" +from flask import Flask, send_file, send_from_directory, jsonify, abort +from flask_cors import CORS + +app = Flask(__name__) +CORS(app) + + +@app.route('/') +def main_page(): + return "We are offline at the moment" + + +@app.errorhandler(404) +def not_found(error): + return jsonify(error=str(error)), 404 + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/backend/config.ini b/backend/config.ini new file mode 100644 index 0000000..0110dea --- /dev/null +++ b/backend/config.ini @@ -0,0 +1,31 @@ +[SQL] +; Database connection details +host = localhost +user = root +password = root +database = Nijitrack_Dev + +[TABLES] +; Names of the tables that should be created in the database + +; Current subscriber data +live = subscriber_data +historical = subscriber_data_historical +daily = 24h_historical + +[API] +holodex = HOLODEXAPI +youtube = YOUTUBEAPI + +[PATH] +root_html = stats + +; Website details +homepage = https://nijitracker.com +title = Nijitracker +description = A site that tracks the historical subscriber data for Nijisanji affiliated livers +icon = https://raw.githubusercontent.com/pinapelz/NijiTrack/master/assets/icon.png +timezone = PST +footer_message = This is a demo of Nijitrack, a way to track historical subscriber data for any set of channels on YouTube.<br>This webpage is not affiliated with ANYCOLOR or any of the channels listed here in any way<br>Date Started: 2023-03-26 + + diff --git a/backend/data/channels.txt b/backend/data/channels.txt new file mode 100644 index 0000000..480388b --- /dev/null +++ b/backend/data/channels.txt @@ -0,0 +1,181 @@ +UC0g1AE0DOjBYnLhkgoRWN1w,Honma Himawari +UC0WwEfE-jOM2rzjpdfhTzZA,Aizono Manami +UC1QgXt46-GEvtNjEC1paHnw,Gwelu Os Gar +UC1vawzfbCnRpHT9SJ5pHlHw,Akagi Wen +UC1zFJrfEKvCixhsjNSb1toQ,Sister Claire +UC2OacIzd2UxGHRGhdHl1Rhw,Hayase Sou +UC3lNFeJiTq6L3UWoz4g1e-A,Uzuki Kou +UC47rNmkDcNgbOcM-2BwzJTQ,Millie Parfait +UC48jH1ul-6HOrcSSfoR02fQ,Yuuhi Riri +UC4l9gz3q65lTBFfFtW5LLeA,Watarai Hibari +UC_4tXjqecqox5Uc05ncxpxg,Shiina Yuika +UC4WvIIAo89_AzGUh1AZ6Dkg,Rosemi Lovelock +UC4yNIKGvy-YUrwYupVdLDXA,Ike Eveland +UC53UDnhAAYwvNO7j_2Ju1cQ,Dola +UC5dJFf4m-mEcoyJRfhBljoA,Seraph Dazzlegarden +UC5ek2GWKvUKFgnKSHuuCFrw,So Nagi +UC5qSx7KzdRwbsO1QmJc4d-w,Siska Leontyne +UC5yckZliCkuaEFbqzLBD7hQ,Reza Avanluna +UC6oDys1BGgBsIC3WhG1BovQ,Shizuka Rin +UC-6rZgmxZSIbq786j3RD5ow,Leos Vincent +UC6TfqY40Xt1Y0J-N18c85qQ,Azuchi Momo +UC6wvdADTJ88OfIbJYIpAaDA,Fuwa Minato +UC7Gb7Uawe20QyFibhLl1lzA,Luca Kaneshiro +UC7hffDQLKIEG-_zoAQkMIvg,Akira Ray +UC_82HBGtvwN1hcGeOGHzUBQ,Sorahoshi Kirame +UC8C1LLhBhf_E2IBPLSDJXlQ,Sukoya Kana +UC8Snw5i4eOJXEQqURAK17hQ,Rai Galilei +UC9EjSJ8pvxtvPdxLOElv73w,Makaino Ririmu +UC9V3Y3_uzU5e-usObb6IE1w,Hoshikawa Sara +UCA3WE2WRSpoIvtnoVGq4VAw,ZEA Cornelia +UCahgMxSIQ2zIRrPKhM6Mjvg,Mika Melatika +UCambvP8yxNDot4FzQc9cgiw,Usami Rito +UCAQDFeCTVdx90GtwohwjHzQ,Amagase Muyu +UCb6ObE-XGCctO3WrjRZC-cw,Luis Cammy +UCbc8fwhdUNlqi-J99ISYu4A,Belmond Banderas +UCBi8YaVyZpiKWN3_Z0dCTfQ,Akabane Youko +UCBiqkFJljoxAj10SoP2w2Cg,Fumino Tamaki +UCBURM8S4LH7cRZ0Clea9RDA,Reimu Endou +UCC7rRD6P7RQcx0hKv9RQP4w,Fura Kanato +UCcDDxnoQcezyTUzHg5uHaKg,Shikinagi Akira +UCCHH0nWYXFZmtDS_4tvMxHQ,Yang Nari +UCckdfYDGrjojJM28n5SHYrA,Vox Akuma +UCClwIqTUn5LDpFucHyaAhHg,Lee Roha +UCD-miitqNY3nyukJ4Fnf4_A,Tsukino Mito +UCdpUojq0KWZCN9bxXnZwz5w,Ars Almal +UCe22Bcwd_GCpTjLxn83zl7A,Ponto Nei +UCebT4Aq-3XWb5je1S1FvR_A,Todo Kohaku +UCeGendL8CO5RkffB6IFwHow,Seffyna +UCeK9HFcRZoTrvqcUCtccMoQ,Shibuya Hajime +UCe_p3YEuYJb8Np0Ip9dk-FQ,Asahina Akane +UCerkculBD7YLc_vOGrF7tKg,Matsukai Mao +UCeShTCVgZyq2lsBW9QwIJcw,Gundo Mirei +UCFgXWZOUZA2oYHNr6qDmsTQ,Scarle Yonaguni +UCfipDDn7wY-C-SoUChgxCQQ,Hayama Marin +UCfQVs_KuXeNAlGa3fb8rlnQ,Sakura Ritsuki +UCG0rzBZV_QMP4MtWg6IjhEA,Shu Yamino +UCg63a3lk6PNeWhVvMRM_mrQ,Onomachi Haruka +UCgA2jKRkqpY_8eysPUs8sjw,Petra Gurin +UC_GCs6GARLxEHxy1w40d6VQ,Ienaga Mugi +UCGEMpMpll4otCdnnxQ9paFg,SakaYui +UCggO2c1unS-oLwTLT0ICywg,Kotoka Torahime +UCGhqxhovNfaPBpxfCruy9EA,Fulgur Ovid +UCgIfLpQvelloDi8I0Ycbwpg,Hyakumantenbara Salome +UCgmFrRcyH7d1zR9sIVQhFow,Lauren Iroas +UCGw7lrT-rVZCWHfdG9Frcgg,Genzuki Tojiro +UCGYAYLDE7TZiiC8U6teciDQ,Hakase Fuyuki +UCHBhnG2G-qN0JrrWmMO2FTA,Shellin Burgundy +UChJ5FTsHOu72_5OVx0rvsvQ,Uki Violeta +UCHjeZylSgXDSnor8wUnwU_g,Bonnivier Pranaja +UCHK5wkevfaGrPr7j3g56Jmw,Seto Miyako +UChKXd7oqD18qiIYBoRIHTlw,Meloco Kyoran +UChUJbHiTVeGrSkTdBzVfNCQ,Joe Rikiichi +UCHVXbQzkl3rDfsXWo8xi2qw,Ange Katrina +UCHX7YpFG8rVwhsHCx34xt7w,Yukishiro Mahiro +UCiA-trSZfB0i92V_-dyDqBw,Kuramochi Meruto +UCIBj1-d71vKjRftiauF50pg,Hyona Elatiora +UCIeSUTOTkF9Hs7q3SGcO-Ow,Elira Pendora +UCIG9rDtgR45VCZmYnd-4DUw,Ratna Petit +UCijNnZ-6m8g85UGaRAWuw7g,Nagisa Arcinia +UCIM92Ok_spNKLVB5TsgwseQ,Mysta Rias +UCiSRx1a2k-0tOg-fs6gAolQ,Asuka Hina +UCivwPlOp0ojnMPZj5pNOPPA,Sophia Valentine +UCIytNcoz4pWzXfLda0DoULQ,Ex Albio +UCjFu-9GHnabzSFRAYm1B9Dw,Etna Crimson +UCjlmCrq4TP1I4xguOtJ-31w,Debidebi Debiru +UCJubINhCcFXlsBwnHp0wl_g,Maimoto Keisuke +UCk5r533QVMgJUdWwqegH2TA,Azura Cecillia +UCkieJGn3pgJikVW8gmMXE2w,Nina Kosaka +UCkIimWZ9gBJRamKF0rmPU8w,Amamiya Kokoro +UCkL9OLKjIQbKk2CztbpOCFg,Riksa Dhirendra +UCKMYISTJAQ8xTplUPHiABlA,Yashiro Kizuku +UCkngxfPbmGyGl_RIq4FA3MQ,Nishizono Chigusa +UCKu59gTZ_rdEmerdx5rV4Yg,Ren Zotto +UCl1oLKcAq93p-pwKfDGhiYQ,Emma★August +UCL34fAoFim9oHLbVzMKFavQ,Yorumi Rena +UCLjx3lqIkYkPCBJop8czJ2A,Ban Hada +UCllKI7VjyANuS1RXatizfLQ,Yamagami Karuta +UCLO9QDxVL4bnvRRsz6K4bsQ,Yuuki Chihiro +UCL_O_HXgLJx3Auteer0n0pA,Suo Sango +UClrQ7xhRBxS_v_-WuudGKmA,Kaburaki Roco +UClS6k3w1sPwlVFqK3-yID5A,Ryu Hari +UCmeyo5pRj_6PXG-CsGUuWWg,Kuroi Shiba +UCmovZ2th3Sqpd00F5RdeigQ,Kagami Hayato +UCmUjjW5zF1MMOhYUwwwQv9Q,Ushimi Ichigo +UCmZ1Rbthn-6Jm_qOGjYsh5A,Ibrahim +UCMzVa7B8UEdrvUGsPmSgyjA,Derem Kado +UCN68LoM3khS4gdBMiWJO8WA,Aia Amare +UCnRQYHTnRLSF0cLJwMnedCg,Aiba Uiha +UCNW1Ex0r6HsWRD4LCtPwvoQ,Saegusa Akina +UCnzZmBOSrQf2wDBbnsDajVw,Oh Jiyu +UCo2N7C-Z91waaR6lF3LL_jw,Kaida Haru +UCo7TRj3cS-f_1D9ZDmuTsjw,Machita Chima +UCO8WcDsF5znr-qsXlzZNpqg,Ver Vermillion +UC-o-E6I3IC2q8sAoAuM6Umg,Naraka +UCoJ0Ct-jdas4cLPpSp06gZg,Xia Ekavira +UCoM_XmK45j504hfUWvN06Qg,Naruse Naru +UCoWH3sDpeXG1aXmOxveX4KA,Nara Haramaung +UCoztvTULBYd3WmStqYeoHcA,Sasaki Saku +UCP4nMSTdwU1KqYWu3UH5DHQ,Pomu Rainpuff +UCpJtk0myFr5WnyfsmnInP-w,Hana Macchia +UCpNH2Zk2gw3JBjWAKSyZcQQ,Eli Conifer +UCpnvhOIJ6BN-vPkYU9ls-Eg,Suzuya Aki +UCpRXCTyNNa-MnjhK6gisnRw,Gaon +UCPvGypSgfDkVe7JG2KygK7A,Rindou Mikoto +UCpzxZW5kghGnO5TmAFJQAVw,Aster Arcadia +UCQ1zGxHrfEmmW4CPpBx9-qw,Alban Knox +UCqjTqdVlvIipZXIKeCkHKUA,Oliver Evans +UCqXxS-9x9Ha_UiH6hG4kh5Q,Hibachi Mana +UCR6qhsLpn62WVxCBK1dkLow,Enna Alouette +UCRcLAVTbmx2-iNcXSsupdNA,Kurusu Natsume +UCrhhJPNsOqzNIkUfTABoSpg,Ha Yun +UCRm6lqtdxs_Qo6HeL-SRQ-w,Lain Paterson +UCRqBKoKuX30ruKAq05pCeRQ,Kitakoji Hisui +UCrR7JxkbeLY82e8gsj_I0pQ,Amicia Michella +UCRV9d6YCYIMUszK-83TwxVA,Todoroki Kyoko +UCRWOdwLRsenx2jLaiCAIU4A,Amemori Sayo +UCryOPk2GZ1meIDt53tL30Tw,Suzuki Masaru +UCsb-1aJgiJXJH2feV-zlZRw,Kyo Kaneko +UCSFCh5NL4qXrAy9u-u2lX3g,Kuzuha +UCsFn_ueskBkMCEyzCEqAOvg,Hanabatake Chaika +UCsg-YqdqQ-KFF0LNk23BY4A,Higuchi Kaede +UCspv01oxUFf_MTSipURRhkA,Kanae +UCS-XXTgVkotkbkDnGEprXpg,Mashiro +UCt0clH12Xk1-Ej5PXKGfdPA,Mononobe Alice +UCt5-0i4AVHXaWJrL8Wql3mw,Ryushen +UCtAvQ5U0aXyKwm2i4GqFgJg,Harusaki Air +UCtHY-tP0dyykhTRMmnfPs_g,Umise Yotsuha +UCTIE7LM5X15NVugV7Krp9Hw,Yumeoi Kakeru +UCtLfA_qUqCJtjXJM2ZR_keg,Ishigami Nozomi +UCtnO2N4kPTXmyvedjGWdx3Q,Levi Elipha +UCtpB6Bvhs1Um93ziEDACQ8g,Morinaka Kazaki +UCUc8GZfFxtmk7ZwSO7ccQ0g,Nui Sociere +UCuep1JCrMvSxOGgGhBfJuYw,Furen E Lustario +UCu-J8uIXuLZh16gG-cT1naw,Finana Ryugu +UCUP8TmlO7NNra88AMqGU_vQ,Koshimizu Toru +UCu-rV2gPtJ-CsGxe71z_BrQ,Igarashi Rika +UCUtKkGKef8BYMs3h-3zQm9A,Min Suha +UCuuAb_72QzK0M1USPMEl1yw,Sonny Brisko +UCuvk5PilcvDECU7dDZhQiEw,Shirayuki Tomoe +UCUzJ90o1EjqUbk2pBAy0_aw,Gilzaren III +UCv1fFr156jc65EMiLbaLImw,Kenmochi Toya +UCV1xUwfM2v2oBtT3JNvic3w,Selen Tatsuki +UCV5ZZlLjk5MKGg3L0n0vbzw,Takamiya Rion +UCvmppcdYf4HOv-tFQhHHJMA,Moira +UCvzVB-EYuHFXHZrObB8a_Og,Yaguruma Rine +UCwaS8_S7kMiKA3izlTWHbQg,Maria Marionette +UCwokZsOK_uEre70XayaFnzA,Suzuka Utako +UCwrjITPwG4q71HzihV2C7Nw,Fumi +UCWRPqA0ehhWV4Hnp27PJCkQ,Shishido Akari +UCWz0CSYCxf4MhRKPDm220AQ,Kanda Shoichi +UCX88Pe54pxbJDSGIyGrzNdg,Na Sera +UCXRlIK3Cw_TJIQC5kSJJQMg,Inui Toko +UCXU7YYxy_iQd3ulXyO-zC2w,Fushimi Gaku +UCXW4MqCQn-jCaxlX-nn-BYg,Nagao Kei +UCy8P3o5XlMpJGQY4WugzdNA,Saiki Ittetsu +UCy91xBlY_Brh3bnHxKtjrrg,Doppio Dropscythe +UCYKP16oMX9KKPbrNgo_Kgag,Elu the Elf +UCyRkQSuhJILuGOuXk10voPg,Layla Alstroemeria +UCZ1xuCK1kNmn5RzPYIZop3w,Lize Helesta +UCZ5dNZsqBjBzbBl0l_IdmXg,Taka Radjiman +UCz_ZRw6ak4Foy8zZy0kEprQ,Hex Haywire diff --git a/backend/data/exclude_channel.txt b/backend/data/exclude_channel.txt new file mode 100644 index 0000000..314b524 --- /dev/null +++ b/backend/data/exclude_channel.txt @@ -0,0 +1,111 @@ +UCX7YkU9nEeaoZbkVLVajcMg +UCwi4P78SVunSYAGrvC9aKcw +UCNRh9kkByBTYLo0IJupnAug +C4Jyg9gFStHO8r5n4ya7XCQ +UCStzBFfjwFYb0qSYHnzFZvw +UCrhf6HYKnV6bxyaB6ooJzbw +UCfki3lMEF6SGBFiFfo9kvUA +UCtHFXfrn52juTqGBN4WbMVw +UCbLgcjfsUaCUgJh9SVit8kw +UC_D2DNy-KUNQJ_NGMppgmyg +UCxWcO9CLti4uouUIS5IIF-Q +UC4Jyg9gFStHO8r5n4ya7XCQ +UCz6vnIbgiqFT9xUcD6Bp65Q +UC-JSeFfovhNsEhftt1WHMvg +UC0PwyIlUefx1LGWjFI0QjMg +UCRzHROJUp7Wg900p_fXtJtQ +UC2NtHPaDUA5htYsjVrO8Gng +UCLSzgV37Dt24T8p3-TNiSLg +UC8vZcu6W-EJ6UYEy_C31c-A +UCh00mQw8BCrTbchYWGz4JTQ +UCWwwXXd_RPzj3LAyEfBslsg +UCINqoksO3CQPCt8a-mRe-Ew +UCF1JdALrXgub24weQpqDy9Q +UCvQIBipkXlXXVdGga9sn8dw +UCKQi12nOGZsJ5nOuCTHErmA +UCSUZugdxy9Wcrkp781cFt1w +UCTi_rzf5QIkXjhJjkbcAdTg +UCpfjQCCavrO-rnKaAaIF9dg +UCX7YkU9nEeaoZbkVLVajcMg +UCwi4P78SVunSYAGrvC9aKcw +UCNRh9kkByBTYLo0IJupnAug +C4Jyg9gFStHO8r5n4ya7XCQ +UCStzBFfjwFYb0qSYHnzFZvw +UCrhf6HYKnV6bxyaB6ooJzbw +UCfki3lMEF6SGBFiFfo9kvUA +UCtHFXfrn52juTqGBN4WbMVw +UCbLgcjfsUaCUgJh9SVit8kw +UC_D2DNy-KUNQJ_NGMppgmyg +UCxWcO9CLti4uouUIS5IIF-Q +UC4Jyg9gFStHO8r5n4ya7XCQ +UCz6vnIbgiqFT9xUcD6Bp65Q +UC-JSeFfovhNsEhftt1WHMvg +UC0PwyIlUefx1LGWjFI0QjMg +UCRzHROJUp7Wg900p_fXtJtQ +UC2NtHPaDUA5htYsjVrO8Gng +UCLSzgV37Dt24T8p3-TNiSLg +UC8vZcu6W-EJ6UYEy_C31c-A +UCh00mQw8BCrTbchYWGz4JTQ +UCWwwXXd_RPzj3LAyEfBslsg +UCINqoksO3CQPCt8a-mRe-Ew +UCF1JdALrXgub24weQpqDy9Q +UCvQIBipkXlXXVdGga9sn8dw +UCKQi12nOGZsJ5nOuCTHErmA +UCSUZugdxy9Wcrkp781cFt1w +UCTi_rzf5QIkXjhJjkbcAdTg +UCpfjQCCavrO-rnKaAaIF9dg +UC0lik9pHju6ONgkBh7N5wHw +UC1ZV7KBscK0EMoJKFu1DnDg +UC69URn8iP4u8D_zUp-IJ1sg +UC6oW4FXETgEGOFTxWmI2h5Q +UC6WU2SrnG019ucm_bdY6qxQ +UC8oPnditPSp5lZu45fnXWCA +UC9oudjCTHL2BfwxfDwI18eg +UC_a1ZYZ8ZTXpjg9xUY9sj8w +UC_aB_-PHLFHiP61djM0oOiQ +UCb5JxV6vKlYVknoJB8TnyYg +UCbLgcjfsUaCUgJh9SVit8kw +UCCJcCWrbQzz6eDhX4M1CUwg +UCCVwhI5trmaSxfcze_Ovzfw +UCF1JdALrXgub24weQpqDy9Q +UCFaDvgez8USXHiKidt0NtZg +UCfki3lMEF6SGBFiFfo9kvUA +UCfM_A7lE6LkGrzx6_mOtI4g +UCi6nV5Z2dzFuXBzLG3P9zqQ +UCIairB9UMDvqSKfMdv1jmjg +UCIJ9zP6gIkT8BB4Lz4UYPhA +UCJ6LH4jMNy0JN9RSThz1mMQ +UCjGE11ZnF0JSR8egVAwh-3A +UCks41vQN-hN-1KHmpZyPY3A +UCLpYMk5h1bq8_GAFVBgXhPQ +UCLSzgV37Dt24T8p3-TNiSLg +UCmWqYB6y8gSfPONWGspuOWQ +UCNUgrFCo2Hr_VXc9bEwjcHQ +UCOmjciHZ8Au3iKMElKXCF_g +UCP2o-o6u4uX3uq1hXspl0rg +UCqEp6RdtsMbUNrCdCswr6pA +UCqQV8xEBWd5SVZBLlYrS_5Q +UCRzHROJUp7Wg900p_fXtJtQ +UCSc_KzY_9WYAx9LghggjVRA +UCSlv7Z-4q7_7NRkzJB10A5Q +UCStzBFfjwFYb0qSYHnzFZvw +UCufQu4q65z63IgE4cfKs1BQ +UCuz0vzQgC8LRdS6lVV0UkUg +UCveZ9Ic1VtcXbsyaBgxPMvg +UCwcyyxn6h9ex4sMXGtpQE_g +UCwQ9Uv-m8xkE5PzRc7Bqx3Q +UCyqzU2nq7eGNi4kN0uHx7TA +UCZgRRTDMYwHpQ7sCl0aCp2Q +UCzMAP-oh5-pC8j6RlCPf26w +UC5qSx7KzdRwbsO1QmJc4d-w +UC5yckZliCkuaEFbqzLBD7hQ +UCA3WE2WRSpoIvtnoVGq4VAw +UCe_p3YEuYJb8Np0Ip9dk-FQ +UCeShTCVgZyq2lsBW9QwIJcw +UCIBj1-d71vKjRftiauF50pg +UCIM92Ok_spNKLVB5TsgwseQ +UCk5r533QVMgJUdWwqegH2TA +UCkieJGn3pgJikVW8gmMXE2w +UCoWH3sDpeXG1aXmOxveX4KA +UCrR7JxkbeLY82e8gsj_I0pQ +UCZ5dNZsqBjBzbBl0l_IdmXg diff --git a/backend/data/last_refresh.txt b/backend/data/last_refresh.txt new file mode 100644 index 0000000..ae97356 --- /dev/null +++ b/backend/data/last_refresh.txt @@ -0,0 +1 @@ +2023-11-17
\ No newline at end of file diff --git a/backend/decorators.py b/backend/decorators.py new file mode 100644 index 0000000..bc0b420 --- /dev/null +++ b/backend/decorators.py @@ -0,0 +1,16 @@ +import time + + +def log(message: str): + def decorator(func): + def wrapper(*args, **kwargs): + print("TASK: " + message) + start = time.time() + result = func(*args, **kwargs) + end = time.time() + print(f"COMPLETE: {message} {round(end - start, 3)} seconds") + return result + + return wrapper + + return decorator diff --git a/backend/fileutil.py b/backend/fileutil.py new file mode 100644 index 0000000..2aa2d6f --- /dev/null +++ b/backend/fileutil.py @@ -0,0 +1,109 @@ +import os.path +import urllib.request +import json +import time +import configparser + + +def _read_file(path: str, lines=True) -> list: + # reads a file and returns a list of lines + with open(path, "r", encoding="utf-8") as file: + if not lines: + return file.read() + return file.read().splitlines() + + +def get_excluded_channels(): + # gets excluded channels from exclude_channel.txt + if not os.path.exists(os.path.join("data", "exclude_channel.txt")): + open(os.path.join("data", "exclude_channel.txt"), "w").close() + excluded_channels = _read_file(os.path.join("data", "exclude_channel.txt")) + return excluded_channels + +def update_excluded_channels(channel_ids: list): + # add to exclude_channel.txt if not already there + excluded_channels = get_excluded_channels() + for channel_id in channel_ids: + if channel_id not in excluded_channels: + excluded_channels.append(channel_id) + with open(os.path.join("data", "exclude_channel.txt"), "w", encoding="utf-8") as file: + for channel_id in excluded_channels: + file.write(channel_id + "\n") + +def save_local_channels(data: list, path: str = "data"): + """ + Save the channel names and ids locally for when the API is down + """ + path = os.path.join(path, "channels.txt") + excluded_channels = get_excluded_channels() + if not os.path.exists(path): + os.makedirs(os.path.dirname(path), exist_ok=True) + open(path, "w").close() + with open(path, "w", encoding="utf-8") as file: + for channel in data: + if channel["id"] in excluded_channels: + continue + file.write(f"{channel['id']},{channel['english_name']}\n") + + +def get_local_channels(path: str = "data"): + """ + Get the channel names and ids locally for when the API is down + """ + path = os.path.join(path, "channels.txt") + if not os.path.exists(path): + raise Exception("Local channel data not found") + with open(path, "r", encoding="utf-8") as file: + rows = file.read().splitlines() + return [tuple(row.split(",")) for row in rows] + + +def check_diff_refresh(): + if not os.path.exists(os.path.join("data", "last_refresh.txt")): + with open( + os.path.join("data", "last_refresh.txt"), "w", encoding="utf-8") as file: + file.write(time.strftime("%Y-%m-%d")) + return True + with open(os.path.join("data", "last_refresh.txt"), "r", encoding="utf-8") as file: + last_refresh = file.read() + if last_refresh != time.strftime("%Y-%m-%d"): + with open( + os.path.join("data", "last_refresh.txt"), "w", encoding="utf-8" + ) as file: + file.write(time.strftime("%Y-%m-%d")) + return True + + +def update_data_files(url: str) -> None: + # Updates the local txt channel data stored in data folder + if not os.path.exists(os.path.join("data", "channels.txt")): + open(os.path.join("data", "channels.txt"), "w").close() + urllib.request.urlretrieve( + url + "channels.txt", os.path.join("data", "channels.txt") + ) + # downloaded txt file from url and write to channels.txt + + if not os.path.exists(os.path.join("data", "exclude_channel.txt")): + open(os.path.join("data", "exclude_channel.txt"), "w").close() + urllib.request.urlretrieve( + url + "exclude_channel.txt", os.path.join("data", "exclude_channel.txt") + ) + + +def load_config(ini_filepath: str) -> dict: + config_object = configparser.ConfigParser() + file = open(ini_filepath, "r") + config_object.read_file(file) + output_dict = {} + sections = config_object.sections() + for section in sections: + output_dict[section] = {} + for key in config_object[section]: + output_dict[section][key] = config_object[section][key] + return output_dict + +def load_json_file(json_file_path: str) -> dict: + with open(json_file_path, "r", encoding="utf-8") as file: + return json.load(file) + + diff --git a/backend/nijitrack.py b/backend/nijitrack.py new file mode 100644 index 0000000..30b31cc --- /dev/null +++ b/backend/nijitrack.py @@ -0,0 +1,104 @@ +import os +import time + +import fileutil as fs +from sql.sql_handler import SQLHandler +from webapi.holodex import HolodexAPI +from webapi.youtube import YouTubeAPI +from decorators import * +import argparse + + +CONFIG = fs.load_config("config.ini") +DATA_SETTING = fs.load_json_file("sql_table_config.json") + + +@log("Initializing Database") +def initialize_database(server: SQLHandler): + server.create_table(name = CONFIG["TABLES"]["live"], column = DATA_SETTING["LIVE_COLUMNS"]) + server.create_table(name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["LIVE_COLUMNS"]) + server.create_table(name = CONFIG["TABLES"]["daily"], column = DATA_SETTING["DAILY_COLUMNS"]) + + +@log("Inserting Live Data into Database") +def record_subscriber_data(data: list): + def transform_sql_string(string: str) -> str: + return string.encode("ascii", "ignore").decode().replace("'", "''") + def record_diff_data(data_tuple: tuple, refresh_daily: bool): + if not server.check_row_exists(CONFIG["TABLES"]["daily"], "channel_id", channel_id): + # data_tuple = (channel_id, pfp, channel_name, sub_count, time.strftime('%Y-%m-%d %H:%M:%S')) + server.insert_row(CONFIG["TABLES"]["daily"], DATA_SETTING["DAILY_HEADER"], (data_tuple[0], data_tuple[3])) + server.insert_row(name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["LIVE_HEADER"], data=data_tuple) + return + elif refresh_daily: + server.update_row(CONFIG["TABLES"]["daily"], "channel_id", channel_id, "sub_diff", sub_count) + server.insert_row(name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["LIVE_HEADER"], data=data_tuple) + + exclude_channels = fs.get_excluded_channels() + refresh_daily = fs.check_diff_refresh() + for channel in data: + channel_id = channel["id"] + if channel_id in exclude_channels: + continue + pfp = channel["photo"] + sub_count = channel["subscriber_count"] + channel_name = channel["english_name"] + if channel_name is None: + channel_name = channel["name"] + channel_name = transform_sql_string(channel_name) + data_tuple = (channel_id, pfp, channel_name, sub_count, time.strftime('%Y-%m-%d %H:%M:%S')) + server.insert_row(name = CONFIG["TABLES"]["live"], column = DATA_SETTING["LIVE_HEADER"], data=data_tuple) + record_diff_data(data_tuple, refresh_daily) + + +@log("Running Holodex Generation") +def holodex_generation(server: SQLHandler): + """ + Generates the data from the Holodex API + """ + holodex_organizations = DATA_SETTING["HOLODEX_ORGS"].split(",") + server.clear_table(CONFIG["TABLES"]["live"]) + server.reset_auto_increment(CONFIG["TABLES"]["live"]) + holodex = HolodexAPI(CONFIG["API"]["holodex"]) + for organization in holodex_organizations: + holodex.set_organization(organization) + record_subscriber_data(holodex.get_subscriber_data()) + return holodex.get_generated_channel_data(), holodex.get_inactive_channels() + +@log("Running YouTube Generation") +def youtube_generation(server: SQLHandler): + """ + Generates the data from the YouTube API + """ + ytapi = YouTubeAPI(CONFIG["API"]["youtube"]) + server.clear_table(CONFIG["TABLES"]["live"]) + server.reset_auto_increment(CONFIG["TABLES"]["live"]) + data = ytapi.get_data_all_channels(fs.get_local_channels()) + record_subscriber_data(data) + return data + +def combine_excluded_channel_ids(inactive_channel_data: list, excluded_channels: list): + """ + Combines the local excluded channels with the inactive channels from the API + """ + channel_ids = [] + for inactive_channel in inactive_channel_data: + if inactive_channel in excluded_channels: + continue + channel_ids.append(inactive_channel) + return channel_ids + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="NijiTrack - A Subscriber Tracker") + parser.add_argument('--mode', choices=['yt', 'holodex'], help='Specify the data source to use (yt or holodex)') + args = parser.parse_args() + server = SQLHandler(CONFIG["SQL"]["host"], CONFIG["SQL"]["user"], CONFIG["SQL"]["password"], CONFIG["SQL"]["database"]) + initialize_database(server) + if args.mode == 'yt': + print("Using YouTube API") + channel_data = youtube_generation(server) + inactive_channels = fs.get_excluded_channels() + else: + channel_data, inactive_channels = holodex_generation(server) + fs.update_excluded_channels(inactive_channels)
\ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..2f05f53 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,21 @@ +certifi==2023.7.22 +charset-normalizer==3.1.0 +docopt==0.6.2 +greenlet==2.0.2 +idna==3.4 +mysql-connector-python==8.0.32 +numpy==1.24.2 +packaging==23.0 +pandas==1.5.3 +pipreqs==0.4.12 +plotly==5.13.1 +protobuf==3.20.3 +python-dateutil==2.8.2 +pytz==2022.7.1 +requests==2.31.0 +six==1.16.0 +tenacity==8.2.2 +typing_extensions==4.5.0 +urllib3==1.26.15 +yarg==0.1.9 +Flask~=2.2.3
\ No newline at end of file diff --git a/backend/sql/sql_handler.py b/backend/sql/sql_handler.py new file mode 100644 index 0000000..9d1c10d --- /dev/null +++ b/backend/sql/sql_handler.py @@ -0,0 +1,162 @@ +import mysql.connector +from mysql.connector import Error, errorcode + + +class SQLHandler: + def __init__(self, host_name: str, user_name: str, user_password: str, database_name: str): + self.host_name = host_name + self.username = user_name + self.password = user_password + self.database_name = database_name + self.connection = self._create_server_connection( + host_name, user_name, user_password) + self._load_database(database_name) + + def _create_server_connection(self, host_name: str, user_name: str, user_password: str) -> mysql.connector: + connection = None + try: + connection = mysql.connector.connect(host=host_name, user=user_name, passwd=user_password) + print("MySQL Database connection successful") + except Error as err: + print(f"Error: '{err}'") + return connection + + def get_connection(self): + return self.connection + + def _create_database(self, cursor: str, database_name: str): + try: + cursor.execute( + f"CREATE DATABASE {database_name} DEFAULT CHARACTER SET 'utf8'") + except Error as err: + print(f"Failed creating database: {err}") + exit(1) + + def _load_database(self, database_name: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"USE {database_name}") + print(f"Database {database_name} loaded successfully") + except Error as err: + print(f"Database {database_name} does not exist") + if err.errno == errorcode.ER_BAD_DB_ERROR: + self._create_database(cursor, database_name) + print(f"Database {database_name} created successfully") + self.connection.database = database_name + else: + print(err) + exit(1) + + def create_table(self, name: str, column: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"CREATE TABLE {name} ({column})") + print(f"Table {name} created successfully") + except Error as err: + print(err) + + def insert_row(self, name: str, column: str, data: tuple): + cursor = self.connection.cursor(buffered=True) + try: + placeholders = ', '.join(['%s'] * len(data)) + query = f"INSERT INTO {name} ({column}) VALUES ({placeholders})" + cursor.execute(query, data) + self.connection.commit() + print("Data Inserted:", data, "into", name) + except Error as err: + print("Error inserting data") + print(err) + + + def clear_table(self, name: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"DELETE FROM {name}") + self.connection.commit() + print("Table cleared successfully") + except Error as err: + print("Error clearing table") + print(err) + + def reset_auto_increment(self, name: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"ALTER TABLE {name} AUTO_INCREMENT = 1") + self.connection.commit() + print("Table reset successfully") + except Error as err: + print("Error resetting table") + print(err) + + def copy_rows_to_new_table(self, name: str, new_name: str, column: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute( + f"INSERT INTO {new_name} ({column}) SELECT {column} FROM {name}") + cursor.execute( + f"ALTER TABLE {new_name} MODIFY COLUMN id INT AUTO_INCREMENT") + self.connection.commit() + print("Rows copied successfully") + except Error as err: + print("Error copying rows") + print(err) + + def drop_table(self, name: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"DROP TABLE {name}") + self.connection.commit() + print("Table dropped successfully") + except Error as err: + print("Error dropping table") + print(err) + + def check_row_exists(self, name: str, column_name: str, value: str): + """ + Checks if a row exists in a table + """ + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"SELECT * FROM {name} WHERE {column_name} = '{value}'") + result = cursor.fetchone() + if result: + return True + else: + return False + except Error as err: + print("Error checking row") + print(err) + + def update_row(self, name: str, column_name: str, search_val: str, replace_col:str, new_value: str): + """ + Updates a row in a table + """ + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(f"UPDATE {name} SET {replace_col} = '{new_value}' WHERE {column_name} = '{search_val}'") + self.connection.commit() + print("Row updated successfully") + except Error as err: + print("Error updating row") + print(err) + + def execute_query(self, query: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(query) + result = cursor.fetchall() + return result + except Error as err: + print("Error executing query") + print(err) + + def get_query_result(self, query: str): + cursor = self.connection.cursor(buffered=True) + try: + cursor.execute(query) + result = cursor.fetchall() + return result + except Error as err: + print("Error executing query") + print(err) +
\ No newline at end of file diff --git a/backend/sql_table_config.json b/backend/sql_table_config.json new file mode 100644 index 0000000..0e2e6e5 --- /dev/null +++ b/backend/sql_table_config.json @@ -0,0 +1,7 @@ +{ + "LIVE_COLUMNS": "id INT PRIMARY KEY AUTO_INCREMENT, channel_id VARCHAR(255), profile_pic VARCHAR(255), name VARCHAR(255), subscriber_count INT, timestamp DATETIME", + "LIVE_HEADER": "channel_id, profile_pic, name, subscriber_count, timestamp", + "DAILY_COLUMNS": "id INT PRIMARY KEY AUTO_INCREMENT, channel_id VARCHAR(255), sub_diff INT", + "DAILY_HEADER": "channel_id, sub_diff", + "HOLODEX_ORGS": "Nijisanji" +} diff --git a/backend/webapi/holodex.py b/backend/webapi/holodex.py new file mode 100644 index 0000000..5f81892 --- /dev/null +++ b/backend/webapi/holodex.py @@ -0,0 +1,75 @@ +from webapi.web_api import WebAPI +from typing import Iterable + + +class HolodexAPI(WebAPI): + """ + Class for interacting with the Holodex API + """ + + def __init__(self,api_key: str = None,member_count: int = 300,organization: str = "Nijisanji"): + super().__init__(api_key=api_key, base_url="https://holodex.net/api/v2/") + self.member_count = member_count + self.organization = organization + self._inactive_channels = [] + self._channel_data = [] + + def get_subscriber_data(self) -> Iterable: + """ + Gets data for all channels in a particular organization + """ + members = self.member_count + data = [] + active_channels = [] + offset = 0 + while members > 0: + data += self._download_url( + f"channels?type=vtuber&offset={offset}&limit=100&org={self.organization}" + ) + members -= 100 + offset += 100 + for channel in data: + print("DEBUG: ", channel["id"]) + try: + channel["description"] = self.get_channel_description(channel["id"]) + if channel["inactive"]: + self._inactive_channels.append(channel["id"]) + continue + active_channels.append(channel) + except (KeyError, TypeError, ValueError): + print("DEBUG:","An error occured with parsing ", channel["id"], channel["name"]) + continue + self._channel_data = active_channels + return active_channels + + def get_view_count(self, channel_id: str) -> int: + """ + Gets the view count for a particular channel + """ + data = self._download_url(f"channels/{channel_id}") + return data["view_count"] + + def get_channel_description(self, channel_id: str) -> str: + """ + Gets the description for a particular channel + """ + data = self._download_url(f"channels/{channel_id}") + return data["description"] + + def set_organization(self, organization: str): + """ + Sets the organization for the API + """ + self.organization = organization + + def get_inactive_channels(self) -> list: + """ + Gets the list of inactive channels + """ + return self._inactive_channels + + def get_generated_channel_data(self) -> list: + """ + Gets the list of channel data + """ + return self._channel_data diff --git a/backend/webapi/web_api.py b/backend/webapi/web_api.py new file mode 100644 index 0000000..525994c --- /dev/null +++ b/backend/webapi/web_api.py @@ -0,0 +1,29 @@ +import urllib.request +import json + + +class WebAPI: + """ + General class for interacting with web APIs + """ + + def __init__(self, api_key: str, base_url: str) -> None: + self.api_key = api_key + self.base_url = base_url + + def _download_url(self, query: str, header = 'X-APIKEY') -> dict: + """ + Downloads the URL and returns the result as a string + param: + query: str - the query to be appended to the base URL + """ + if self.api_key is None: + raise Exception("API key not set") + opener = urllib.request.build_opener() + opener.addheaders = [(header, self.api_key)] + urllib.request.install_opener(opener) + response = urllib.request.urlopen(self.base_url + query) + json_results = response.read() + r_obj = json.loads(json_results) + response.close() + return r_obj diff --git a/backend/webapi/youtube.py b/backend/webapi/youtube.py new file mode 100644 index 0000000..44d786f --- /dev/null +++ b/backend/webapi/youtube.py @@ -0,0 +1,55 @@ +from webapi.web_api import WebAPI + + +class YouTubeAPI(WebAPI): + """ + Class for interacting with the YouTube API + """ + + def __init__(self, api_key: str = None): + self.api_key = api_key + self.base_url = "https://www.googleapis.com/youtube/v3/" + + def _search_matching_id(self, id: str, data: list) -> dict: + """ + Searches for a info matching a given ID + param: + id: str - the ID to search for + """ + for entry in data: + if entry['id'] == id: + return entry + return None + + def get_data_all_channels(self, channel_tuples: list) -> list: + data = [] + members = len(channel_tuples) + request_chunks = [channel_tuples[i:i + 50] for i in range(0, members, 50)] + for chunk in request_chunks: + channel_ids = [x[0] for x in chunk] + channel_names = [x[1] for x in chunk] + request_string = ",".join(channel_ids) + stats = self._download_url( + f"channels?part=statistics&id={request_string}&key={self.api_key}") + snippet = self._download_url( + f"channels?part=snippet&id={request_string}&key={self.api_key}") + stats_list = stats['items'] + snippet_list = snippet['items'] + for i in range(len(stats_list)): + try: + data_entry = {'english_name': channel_names[i], 'id': channel_ids[i], + 'subscriber_count': + self._search_matching_id(channel_ids[i], stats_list)[ + 'statistics']['subscriberCount'], 'view_count': + self._search_matching_id(channel_ids[i], stats_list)[ + 'statistics']['viewCount'], 'photo': + self._search_matching_id(channel_ids[i], snippet_list)[ + 'snippet']['thumbnails']['default']['url'], 'description': + self._search_matching_id(channel_ids[i], snippet_list)[ + 'snippet']['description']} + data.append(data_entry) + except TypeError: + print("Error NoneType: " + str(channel_ids[i])) + except KeyError: + print("Error KeyError: " + str(channel_ids[i])) + return data |