aboutsummaryrefslogtreecommitdiffstats
path: root/backend
diff options
context:
space:
mode:
authorPinapelz <yukais@pinapelz.com>2023-11-17 13:24:42 -0800
committerPinapelz <yukais@pinapelz.com>2023-11-17 13:24:42 -0800
commit96019367e8f72eac26abd3b7a908c2b914bd1ae1 (patch)
treefa7dbe66ea8b12bde0970dc8db27f7601464b3b8 /backend
parentc159e86a3487dbae79de8d0887f288625084bfcd (diff)
v2: add initial Next JS files, remove static templates
Diffstat (limited to 'backend')
-rw-r--r--backend/.gitignore170
-rw-r--r--backend/app.py22
-rw-r--r--backend/config.ini31
-rw-r--r--backend/data/channels.txt181
-rw-r--r--backend/data/exclude_channel.txt111
-rw-r--r--backend/data/last_refresh.txt1
-rw-r--r--backend/decorators.py16
-rw-r--r--backend/fileutil.py109
-rw-r--r--backend/nijitrack.py104
-rw-r--r--backend/requirements.txt21
-rw-r--r--backend/sql/sql_handler.py162
-rw-r--r--backend/sql_table_config.json7
-rw-r--r--backend/webapi/holodex.py75
-rw-r--r--backend/webapi/web_api.py29
-rw-r--r--backend/webapi/youtube.py55
15 files changed, 1094 insertions, 0 deletions
diff --git a/backend/.gitignore b/backend/.gitignore
new file mode 100644
index 0000000..51ff7fb
--- /dev/null
+++ b/backend/.gitignore
@@ -0,0 +1,170 @@
+__pycache__
+config.json
+.venv
+.github
+main.py
+.idea
+tables
+stats
+.vscode
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+# in version control.
+# https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
diff --git a/backend/app.py b/backend/app.py
new file mode 100644
index 0000000..51b0dbb
--- /dev/null
+++ b/backend/app.py
@@ -0,0 +1,22 @@
+"""
+Flask app for serving the static files
+"""
+from flask import Flask, send_file, send_from_directory, jsonify, abort
+from flask_cors import CORS
+
+app = Flask(__name__)
+CORS(app)
+
+
+@app.route('/')
+def main_page():
+ return "We are offline at the moment"
+
+
+@app.errorhandler(404)
+def not_found(error):
+ return jsonify(error=str(error)), 404
+
+
+if __name__ == "__main__":
+ app.run(debug=True)
diff --git a/backend/config.ini b/backend/config.ini
new file mode 100644
index 0000000..0110dea
--- /dev/null
+++ b/backend/config.ini
@@ -0,0 +1,31 @@
+[SQL]
+; Database connection details
+host = localhost
+user = root
+password = root
+database = Nijitrack_Dev
+
+[TABLES]
+; Names of the tables that should be created in the database
+
+; Current subscriber data
+live = subscriber_data
+historical = subscriber_data_historical
+daily = 24h_historical
+
+[API]
+holodex = HOLODEXAPI
+youtube = YOUTUBEAPI
+
+[PATH]
+root_html = stats
+
+; Website details
+homepage = https://nijitracker.com
+title = Nijitracker
+description = A site that tracks the historical subscriber data for Nijisanji affiliated livers
+icon = https://raw.githubusercontent.com/pinapelz/NijiTrack/master/assets/icon.png
+timezone = PST
+footer_message = This is a demo of Nijitrack, a way to track historical subscriber data for any set of channels on YouTube.<br>This webpage is not affiliated with ANYCOLOR or any of the channels listed here in any way<br>Date Started: 2023-03-26
+
+
diff --git a/backend/data/channels.txt b/backend/data/channels.txt
new file mode 100644
index 0000000..480388b
--- /dev/null
+++ b/backend/data/channels.txt
@@ -0,0 +1,181 @@
+UC0g1AE0DOjBYnLhkgoRWN1w,Honma Himawari
+UC0WwEfE-jOM2rzjpdfhTzZA,Aizono Manami
+UC1QgXt46-GEvtNjEC1paHnw,Gwelu Os Gar
+UC1vawzfbCnRpHT9SJ5pHlHw,Akagi Wen
+UC1zFJrfEKvCixhsjNSb1toQ,Sister Claire
+UC2OacIzd2UxGHRGhdHl1Rhw,Hayase Sou
+UC3lNFeJiTq6L3UWoz4g1e-A,Uzuki Kou
+UC47rNmkDcNgbOcM-2BwzJTQ,Millie Parfait
+UC48jH1ul-6HOrcSSfoR02fQ,Yuuhi Riri
+UC4l9gz3q65lTBFfFtW5LLeA,Watarai Hibari
+UC_4tXjqecqox5Uc05ncxpxg,Shiina Yuika
+UC4WvIIAo89_AzGUh1AZ6Dkg,Rosemi Lovelock
+UC4yNIKGvy-YUrwYupVdLDXA,Ike Eveland
+UC53UDnhAAYwvNO7j_2Ju1cQ,Dola
+UC5dJFf4m-mEcoyJRfhBljoA,Seraph Dazzlegarden
+UC5ek2GWKvUKFgnKSHuuCFrw,So Nagi
+UC5qSx7KzdRwbsO1QmJc4d-w,Siska Leontyne
+UC5yckZliCkuaEFbqzLBD7hQ,Reza Avanluna
+UC6oDys1BGgBsIC3WhG1BovQ,Shizuka Rin
+UC-6rZgmxZSIbq786j3RD5ow,Leos Vincent
+UC6TfqY40Xt1Y0J-N18c85qQ,Azuchi Momo
+UC6wvdADTJ88OfIbJYIpAaDA,Fuwa Minato
+UC7Gb7Uawe20QyFibhLl1lzA,Luca Kaneshiro
+UC7hffDQLKIEG-_zoAQkMIvg,Akira Ray
+UC_82HBGtvwN1hcGeOGHzUBQ,Sorahoshi Kirame
+UC8C1LLhBhf_E2IBPLSDJXlQ,Sukoya Kana
+UC8Snw5i4eOJXEQqURAK17hQ,Rai Galilei
+UC9EjSJ8pvxtvPdxLOElv73w,Makaino Ririmu
+UC9V3Y3_uzU5e-usObb6IE1w,Hoshikawa Sara
+UCA3WE2WRSpoIvtnoVGq4VAw,ZEA Cornelia
+UCahgMxSIQ2zIRrPKhM6Mjvg,Mika Melatika
+UCambvP8yxNDot4FzQc9cgiw,Usami Rito
+UCAQDFeCTVdx90GtwohwjHzQ,Amagase Muyu
+UCb6ObE-XGCctO3WrjRZC-cw,Luis Cammy
+UCbc8fwhdUNlqi-J99ISYu4A,Belmond Banderas
+UCBi8YaVyZpiKWN3_Z0dCTfQ,Akabane Youko
+UCBiqkFJljoxAj10SoP2w2Cg,Fumino Tamaki
+UCBURM8S4LH7cRZ0Clea9RDA,Reimu Endou
+UCC7rRD6P7RQcx0hKv9RQP4w,Fura Kanato
+UCcDDxnoQcezyTUzHg5uHaKg,Shikinagi Akira
+UCCHH0nWYXFZmtDS_4tvMxHQ,Yang Nari
+UCckdfYDGrjojJM28n5SHYrA,Vox Akuma
+UCClwIqTUn5LDpFucHyaAhHg,Lee Roha
+UCD-miitqNY3nyukJ4Fnf4_A,Tsukino Mito
+UCdpUojq0KWZCN9bxXnZwz5w,Ars Almal
+UCe22Bcwd_GCpTjLxn83zl7A,Ponto Nei
+UCebT4Aq-3XWb5je1S1FvR_A,Todo Kohaku
+UCeGendL8CO5RkffB6IFwHow,Seffyna
+UCeK9HFcRZoTrvqcUCtccMoQ,Shibuya Hajime
+UCe_p3YEuYJb8Np0Ip9dk-FQ,Asahina Akane
+UCerkculBD7YLc_vOGrF7tKg,Matsukai Mao
+UCeShTCVgZyq2lsBW9QwIJcw,Gundo Mirei
+UCFgXWZOUZA2oYHNr6qDmsTQ,Scarle Yonaguni
+UCfipDDn7wY-C-SoUChgxCQQ,Hayama Marin
+UCfQVs_KuXeNAlGa3fb8rlnQ,Sakura Ritsuki
+UCG0rzBZV_QMP4MtWg6IjhEA,Shu Yamino
+UCg63a3lk6PNeWhVvMRM_mrQ,Onomachi Haruka
+UCgA2jKRkqpY_8eysPUs8sjw,Petra Gurin
+UC_GCs6GARLxEHxy1w40d6VQ,Ienaga Mugi
+UCGEMpMpll4otCdnnxQ9paFg,SakaYui
+UCggO2c1unS-oLwTLT0ICywg,Kotoka Torahime
+UCGhqxhovNfaPBpxfCruy9EA,Fulgur Ovid
+UCgIfLpQvelloDi8I0Ycbwpg,Hyakumantenbara Salome
+UCgmFrRcyH7d1zR9sIVQhFow,Lauren Iroas
+UCGw7lrT-rVZCWHfdG9Frcgg,Genzuki Tojiro
+UCGYAYLDE7TZiiC8U6teciDQ,Hakase Fuyuki
+UCHBhnG2G-qN0JrrWmMO2FTA,Shellin Burgundy
+UChJ5FTsHOu72_5OVx0rvsvQ,Uki Violeta
+UCHjeZylSgXDSnor8wUnwU_g,Bonnivier Pranaja
+UCHK5wkevfaGrPr7j3g56Jmw,Seto Miyako
+UChKXd7oqD18qiIYBoRIHTlw,Meloco Kyoran
+UChUJbHiTVeGrSkTdBzVfNCQ,Joe Rikiichi
+UCHVXbQzkl3rDfsXWo8xi2qw,Ange Katrina
+UCHX7YpFG8rVwhsHCx34xt7w,Yukishiro Mahiro
+UCiA-trSZfB0i92V_-dyDqBw,Kuramochi Meruto
+UCIBj1-d71vKjRftiauF50pg,Hyona Elatiora
+UCIeSUTOTkF9Hs7q3SGcO-Ow,Elira Pendora
+UCIG9rDtgR45VCZmYnd-4DUw,Ratna Petit
+UCijNnZ-6m8g85UGaRAWuw7g,Nagisa Arcinia
+UCIM92Ok_spNKLVB5TsgwseQ,Mysta Rias
+UCiSRx1a2k-0tOg-fs6gAolQ,Asuka Hina
+UCivwPlOp0ojnMPZj5pNOPPA,Sophia Valentine
+UCIytNcoz4pWzXfLda0DoULQ,Ex Albio
+UCjFu-9GHnabzSFRAYm1B9Dw,Etna Crimson
+UCjlmCrq4TP1I4xguOtJ-31w,Debidebi Debiru
+UCJubINhCcFXlsBwnHp0wl_g,Maimoto Keisuke
+UCk5r533QVMgJUdWwqegH2TA,Azura Cecillia
+UCkieJGn3pgJikVW8gmMXE2w,Nina Kosaka
+UCkIimWZ9gBJRamKF0rmPU8w,Amamiya Kokoro
+UCkL9OLKjIQbKk2CztbpOCFg,Riksa Dhirendra
+UCKMYISTJAQ8xTplUPHiABlA,Yashiro Kizuku
+UCkngxfPbmGyGl_RIq4FA3MQ,Nishizono Chigusa
+UCKu59gTZ_rdEmerdx5rV4Yg,Ren Zotto
+UCl1oLKcAq93p-pwKfDGhiYQ,Emma★August
+UCL34fAoFim9oHLbVzMKFavQ,Yorumi Rena
+UCLjx3lqIkYkPCBJop8czJ2A,Ban Hada
+UCllKI7VjyANuS1RXatizfLQ,Yamagami Karuta
+UCLO9QDxVL4bnvRRsz6K4bsQ,Yuuki Chihiro
+UCL_O_HXgLJx3Auteer0n0pA,Suo Sango
+UClrQ7xhRBxS_v_-WuudGKmA,Kaburaki Roco
+UClS6k3w1sPwlVFqK3-yID5A,Ryu Hari
+UCmeyo5pRj_6PXG-CsGUuWWg,Kuroi Shiba
+UCmovZ2th3Sqpd00F5RdeigQ,Kagami Hayato
+UCmUjjW5zF1MMOhYUwwwQv9Q,Ushimi Ichigo
+UCmZ1Rbthn-6Jm_qOGjYsh5A,Ibrahim
+UCMzVa7B8UEdrvUGsPmSgyjA,Derem Kado
+UCN68LoM3khS4gdBMiWJO8WA,Aia Amare
+UCnRQYHTnRLSF0cLJwMnedCg,Aiba Uiha
+UCNW1Ex0r6HsWRD4LCtPwvoQ,Saegusa Akina
+UCnzZmBOSrQf2wDBbnsDajVw,Oh Jiyu
+UCo2N7C-Z91waaR6lF3LL_jw,Kaida Haru
+UCo7TRj3cS-f_1D9ZDmuTsjw,Machita Chima
+UCO8WcDsF5znr-qsXlzZNpqg,Ver Vermillion
+UC-o-E6I3IC2q8sAoAuM6Umg,Naraka
+UCoJ0Ct-jdas4cLPpSp06gZg,Xia Ekavira
+UCoM_XmK45j504hfUWvN06Qg,Naruse Naru
+UCoWH3sDpeXG1aXmOxveX4KA,Nara Haramaung
+UCoztvTULBYd3WmStqYeoHcA,Sasaki Saku
+UCP4nMSTdwU1KqYWu3UH5DHQ,Pomu Rainpuff
+UCpJtk0myFr5WnyfsmnInP-w,Hana Macchia
+UCpNH2Zk2gw3JBjWAKSyZcQQ,Eli Conifer
+UCpnvhOIJ6BN-vPkYU9ls-Eg,Suzuya Aki
+UCpRXCTyNNa-MnjhK6gisnRw,Gaon
+UCPvGypSgfDkVe7JG2KygK7A,Rindou Mikoto
+UCpzxZW5kghGnO5TmAFJQAVw,Aster Arcadia
+UCQ1zGxHrfEmmW4CPpBx9-qw,Alban Knox
+UCqjTqdVlvIipZXIKeCkHKUA,Oliver Evans
+UCqXxS-9x9Ha_UiH6hG4kh5Q,Hibachi Mana
+UCR6qhsLpn62WVxCBK1dkLow,Enna Alouette
+UCRcLAVTbmx2-iNcXSsupdNA,Kurusu Natsume
+UCrhhJPNsOqzNIkUfTABoSpg,Ha Yun
+UCRm6lqtdxs_Qo6HeL-SRQ-w,Lain Paterson
+UCRqBKoKuX30ruKAq05pCeRQ,Kitakoji Hisui
+UCrR7JxkbeLY82e8gsj_I0pQ,Amicia Michella
+UCRV9d6YCYIMUszK-83TwxVA,Todoroki Kyoko
+UCRWOdwLRsenx2jLaiCAIU4A,Amemori Sayo
+UCryOPk2GZ1meIDt53tL30Tw,Suzuki Masaru
+UCsb-1aJgiJXJH2feV-zlZRw,Kyo Kaneko
+UCSFCh5NL4qXrAy9u-u2lX3g,Kuzuha
+UCsFn_ueskBkMCEyzCEqAOvg,Hanabatake Chaika
+UCsg-YqdqQ-KFF0LNk23BY4A,Higuchi Kaede
+UCspv01oxUFf_MTSipURRhkA,Kanae
+UCS-XXTgVkotkbkDnGEprXpg,Mashiro
+UCt0clH12Xk1-Ej5PXKGfdPA,Mononobe Alice
+UCt5-0i4AVHXaWJrL8Wql3mw,Ryushen
+UCtAvQ5U0aXyKwm2i4GqFgJg,Harusaki Air
+UCtHY-tP0dyykhTRMmnfPs_g,Umise Yotsuha
+UCTIE7LM5X15NVugV7Krp9Hw,Yumeoi Kakeru
+UCtLfA_qUqCJtjXJM2ZR_keg,Ishigami Nozomi
+UCtnO2N4kPTXmyvedjGWdx3Q,Levi Elipha
+UCtpB6Bvhs1Um93ziEDACQ8g,Morinaka Kazaki
+UCUc8GZfFxtmk7ZwSO7ccQ0g,Nui Sociere
+UCuep1JCrMvSxOGgGhBfJuYw,Furen E Lustario
+UCu-J8uIXuLZh16gG-cT1naw,Finana Ryugu
+UCUP8TmlO7NNra88AMqGU_vQ,Koshimizu Toru
+UCu-rV2gPtJ-CsGxe71z_BrQ,Igarashi Rika
+UCUtKkGKef8BYMs3h-3zQm9A,Min Suha
+UCuuAb_72QzK0M1USPMEl1yw,Sonny Brisko
+UCuvk5PilcvDECU7dDZhQiEw,Shirayuki Tomoe
+UCUzJ90o1EjqUbk2pBAy0_aw,Gilzaren III
+UCv1fFr156jc65EMiLbaLImw,Kenmochi Toya
+UCV1xUwfM2v2oBtT3JNvic3w,Selen Tatsuki
+UCV5ZZlLjk5MKGg3L0n0vbzw,Takamiya Rion
+UCvmppcdYf4HOv-tFQhHHJMA,Moira
+UCvzVB-EYuHFXHZrObB8a_Og,Yaguruma Rine
+UCwaS8_S7kMiKA3izlTWHbQg,Maria Marionette
+UCwokZsOK_uEre70XayaFnzA,Suzuka Utako
+UCwrjITPwG4q71HzihV2C7Nw,Fumi
+UCWRPqA0ehhWV4Hnp27PJCkQ,Shishido Akari
+UCWz0CSYCxf4MhRKPDm220AQ,Kanda Shoichi
+UCX88Pe54pxbJDSGIyGrzNdg,Na Sera
+UCXRlIK3Cw_TJIQC5kSJJQMg,Inui Toko
+UCXU7YYxy_iQd3ulXyO-zC2w,Fushimi Gaku
+UCXW4MqCQn-jCaxlX-nn-BYg,Nagao Kei
+UCy8P3o5XlMpJGQY4WugzdNA,Saiki Ittetsu
+UCy91xBlY_Brh3bnHxKtjrrg,Doppio Dropscythe
+UCYKP16oMX9KKPbrNgo_Kgag,Elu the Elf
+UCyRkQSuhJILuGOuXk10voPg,Layla Alstroemeria
+UCZ1xuCK1kNmn5RzPYIZop3w,Lize Helesta
+UCZ5dNZsqBjBzbBl0l_IdmXg,Taka Radjiman
+UCz_ZRw6ak4Foy8zZy0kEprQ,Hex Haywire
diff --git a/backend/data/exclude_channel.txt b/backend/data/exclude_channel.txt
new file mode 100644
index 0000000..314b524
--- /dev/null
+++ b/backend/data/exclude_channel.txt
@@ -0,0 +1,111 @@
+UCX7YkU9nEeaoZbkVLVajcMg
+UCwi4P78SVunSYAGrvC9aKcw
+UCNRh9kkByBTYLo0IJupnAug
+C4Jyg9gFStHO8r5n4ya7XCQ
+UCStzBFfjwFYb0qSYHnzFZvw
+UCrhf6HYKnV6bxyaB6ooJzbw
+UCfki3lMEF6SGBFiFfo9kvUA
+UCtHFXfrn52juTqGBN4WbMVw
+UCbLgcjfsUaCUgJh9SVit8kw
+UC_D2DNy-KUNQJ_NGMppgmyg
+UCxWcO9CLti4uouUIS5IIF-Q
+UC4Jyg9gFStHO8r5n4ya7XCQ
+UCz6vnIbgiqFT9xUcD6Bp65Q
+UC-JSeFfovhNsEhftt1WHMvg
+UC0PwyIlUefx1LGWjFI0QjMg
+UCRzHROJUp7Wg900p_fXtJtQ
+UC2NtHPaDUA5htYsjVrO8Gng
+UCLSzgV37Dt24T8p3-TNiSLg
+UC8vZcu6W-EJ6UYEy_C31c-A
+UCh00mQw8BCrTbchYWGz4JTQ
+UCWwwXXd_RPzj3LAyEfBslsg
+UCINqoksO3CQPCt8a-mRe-Ew
+UCF1JdALrXgub24weQpqDy9Q
+UCvQIBipkXlXXVdGga9sn8dw
+UCKQi12nOGZsJ5nOuCTHErmA
+UCSUZugdxy9Wcrkp781cFt1w
+UCTi_rzf5QIkXjhJjkbcAdTg
+UCpfjQCCavrO-rnKaAaIF9dg
+UCX7YkU9nEeaoZbkVLVajcMg
+UCwi4P78SVunSYAGrvC9aKcw
+UCNRh9kkByBTYLo0IJupnAug
+C4Jyg9gFStHO8r5n4ya7XCQ
+UCStzBFfjwFYb0qSYHnzFZvw
+UCrhf6HYKnV6bxyaB6ooJzbw
+UCfki3lMEF6SGBFiFfo9kvUA
+UCtHFXfrn52juTqGBN4WbMVw
+UCbLgcjfsUaCUgJh9SVit8kw
+UC_D2DNy-KUNQJ_NGMppgmyg
+UCxWcO9CLti4uouUIS5IIF-Q
+UC4Jyg9gFStHO8r5n4ya7XCQ
+UCz6vnIbgiqFT9xUcD6Bp65Q
+UC-JSeFfovhNsEhftt1WHMvg
+UC0PwyIlUefx1LGWjFI0QjMg
+UCRzHROJUp7Wg900p_fXtJtQ
+UC2NtHPaDUA5htYsjVrO8Gng
+UCLSzgV37Dt24T8p3-TNiSLg
+UC8vZcu6W-EJ6UYEy_C31c-A
+UCh00mQw8BCrTbchYWGz4JTQ
+UCWwwXXd_RPzj3LAyEfBslsg
+UCINqoksO3CQPCt8a-mRe-Ew
+UCF1JdALrXgub24weQpqDy9Q
+UCvQIBipkXlXXVdGga9sn8dw
+UCKQi12nOGZsJ5nOuCTHErmA
+UCSUZugdxy9Wcrkp781cFt1w
+UCTi_rzf5QIkXjhJjkbcAdTg
+UCpfjQCCavrO-rnKaAaIF9dg
+UC0lik9pHju6ONgkBh7N5wHw
+UC1ZV7KBscK0EMoJKFu1DnDg
+UC69URn8iP4u8D_zUp-IJ1sg
+UC6oW4FXETgEGOFTxWmI2h5Q
+UC6WU2SrnG019ucm_bdY6qxQ
+UC8oPnditPSp5lZu45fnXWCA
+UC9oudjCTHL2BfwxfDwI18eg
+UC_a1ZYZ8ZTXpjg9xUY9sj8w
+UC_aB_-PHLFHiP61djM0oOiQ
+UCb5JxV6vKlYVknoJB8TnyYg
+UCbLgcjfsUaCUgJh9SVit8kw
+UCCJcCWrbQzz6eDhX4M1CUwg
+UCCVwhI5trmaSxfcze_Ovzfw
+UCF1JdALrXgub24weQpqDy9Q
+UCFaDvgez8USXHiKidt0NtZg
+UCfki3lMEF6SGBFiFfo9kvUA
+UCfM_A7lE6LkGrzx6_mOtI4g
+UCi6nV5Z2dzFuXBzLG3P9zqQ
+UCIairB9UMDvqSKfMdv1jmjg
+UCIJ9zP6gIkT8BB4Lz4UYPhA
+UCJ6LH4jMNy0JN9RSThz1mMQ
+UCjGE11ZnF0JSR8egVAwh-3A
+UCks41vQN-hN-1KHmpZyPY3A
+UCLpYMk5h1bq8_GAFVBgXhPQ
+UCLSzgV37Dt24T8p3-TNiSLg
+UCmWqYB6y8gSfPONWGspuOWQ
+UCNUgrFCo2Hr_VXc9bEwjcHQ
+UCOmjciHZ8Au3iKMElKXCF_g
+UCP2o-o6u4uX3uq1hXspl0rg
+UCqEp6RdtsMbUNrCdCswr6pA
+UCqQV8xEBWd5SVZBLlYrS_5Q
+UCRzHROJUp7Wg900p_fXtJtQ
+UCSc_KzY_9WYAx9LghggjVRA
+UCSlv7Z-4q7_7NRkzJB10A5Q
+UCStzBFfjwFYb0qSYHnzFZvw
+UCufQu4q65z63IgE4cfKs1BQ
+UCuz0vzQgC8LRdS6lVV0UkUg
+UCveZ9Ic1VtcXbsyaBgxPMvg
+UCwcyyxn6h9ex4sMXGtpQE_g
+UCwQ9Uv-m8xkE5PzRc7Bqx3Q
+UCyqzU2nq7eGNi4kN0uHx7TA
+UCZgRRTDMYwHpQ7sCl0aCp2Q
+UCzMAP-oh5-pC8j6RlCPf26w
+UC5qSx7KzdRwbsO1QmJc4d-w
+UC5yckZliCkuaEFbqzLBD7hQ
+UCA3WE2WRSpoIvtnoVGq4VAw
+UCe_p3YEuYJb8Np0Ip9dk-FQ
+UCeShTCVgZyq2lsBW9QwIJcw
+UCIBj1-d71vKjRftiauF50pg
+UCIM92Ok_spNKLVB5TsgwseQ
+UCk5r533QVMgJUdWwqegH2TA
+UCkieJGn3pgJikVW8gmMXE2w
+UCoWH3sDpeXG1aXmOxveX4KA
+UCrR7JxkbeLY82e8gsj_I0pQ
+UCZ5dNZsqBjBzbBl0l_IdmXg
diff --git a/backend/data/last_refresh.txt b/backend/data/last_refresh.txt
new file mode 100644
index 0000000..ae97356
--- /dev/null
+++ b/backend/data/last_refresh.txt
@@ -0,0 +1 @@
+2023-11-17 \ No newline at end of file
diff --git a/backend/decorators.py b/backend/decorators.py
new file mode 100644
index 0000000..bc0b420
--- /dev/null
+++ b/backend/decorators.py
@@ -0,0 +1,16 @@
+import time
+
+
+def log(message: str):
+ def decorator(func):
+ def wrapper(*args, **kwargs):
+ print("TASK: " + message)
+ start = time.time()
+ result = func(*args, **kwargs)
+ end = time.time()
+ print(f"COMPLETE: {message} {round(end - start, 3)} seconds")
+ return result
+
+ return wrapper
+
+ return decorator
diff --git a/backend/fileutil.py b/backend/fileutil.py
new file mode 100644
index 0000000..2aa2d6f
--- /dev/null
+++ b/backend/fileutil.py
@@ -0,0 +1,109 @@
+import os.path
+import urllib.request
+import json
+import time
+import configparser
+
+
+def _read_file(path: str, lines=True) -> list:
+ # reads a file and returns a list of lines
+ with open(path, "r", encoding="utf-8") as file:
+ if not lines:
+ return file.read()
+ return file.read().splitlines()
+
+
+def get_excluded_channels():
+ # gets excluded channels from exclude_channel.txt
+ if not os.path.exists(os.path.join("data", "exclude_channel.txt")):
+ open(os.path.join("data", "exclude_channel.txt"), "w").close()
+ excluded_channels = _read_file(os.path.join("data", "exclude_channel.txt"))
+ return excluded_channels
+
+def update_excluded_channels(channel_ids: list):
+ # add to exclude_channel.txt if not already there
+ excluded_channels = get_excluded_channels()
+ for channel_id in channel_ids:
+ if channel_id not in excluded_channels:
+ excluded_channels.append(channel_id)
+ with open(os.path.join("data", "exclude_channel.txt"), "w", encoding="utf-8") as file:
+ for channel_id in excluded_channels:
+ file.write(channel_id + "\n")
+
+def save_local_channels(data: list, path: str = "data"):
+ """
+ Save the channel names and ids locally for when the API is down
+ """
+ path = os.path.join(path, "channels.txt")
+ excluded_channels = get_excluded_channels()
+ if not os.path.exists(path):
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ open(path, "w").close()
+ with open(path, "w", encoding="utf-8") as file:
+ for channel in data:
+ if channel["id"] in excluded_channels:
+ continue
+ file.write(f"{channel['id']},{channel['english_name']}\n")
+
+
+def get_local_channels(path: str = "data"):
+ """
+ Get the channel names and ids locally for when the API is down
+ """
+ path = os.path.join(path, "channels.txt")
+ if not os.path.exists(path):
+ raise Exception("Local channel data not found")
+ with open(path, "r", encoding="utf-8") as file:
+ rows = file.read().splitlines()
+ return [tuple(row.split(",")) for row in rows]
+
+
+def check_diff_refresh():
+ if not os.path.exists(os.path.join("data", "last_refresh.txt")):
+ with open(
+ os.path.join("data", "last_refresh.txt"), "w", encoding="utf-8") as file:
+ file.write(time.strftime("%Y-%m-%d"))
+ return True
+ with open(os.path.join("data", "last_refresh.txt"), "r", encoding="utf-8") as file:
+ last_refresh = file.read()
+ if last_refresh != time.strftime("%Y-%m-%d"):
+ with open(
+ os.path.join("data", "last_refresh.txt"), "w", encoding="utf-8"
+ ) as file:
+ file.write(time.strftime("%Y-%m-%d"))
+ return True
+
+
+def update_data_files(url: str) -> None:
+ # Updates the local txt channel data stored in data folder
+ if not os.path.exists(os.path.join("data", "channels.txt")):
+ open(os.path.join("data", "channels.txt"), "w").close()
+ urllib.request.urlretrieve(
+ url + "channels.txt", os.path.join("data", "channels.txt")
+ )
+ # downloaded txt file from url and write to channels.txt
+
+ if not os.path.exists(os.path.join("data", "exclude_channel.txt")):
+ open(os.path.join("data", "exclude_channel.txt"), "w").close()
+ urllib.request.urlretrieve(
+ url + "exclude_channel.txt", os.path.join("data", "exclude_channel.txt")
+ )
+
+
+def load_config(ini_filepath: str) -> dict:
+ config_object = configparser.ConfigParser()
+ file = open(ini_filepath, "r")
+ config_object.read_file(file)
+ output_dict = {}
+ sections = config_object.sections()
+ for section in sections:
+ output_dict[section] = {}
+ for key in config_object[section]:
+ output_dict[section][key] = config_object[section][key]
+ return output_dict
+
+def load_json_file(json_file_path: str) -> dict:
+ with open(json_file_path, "r", encoding="utf-8") as file:
+ return json.load(file)
+
+
diff --git a/backend/nijitrack.py b/backend/nijitrack.py
new file mode 100644
index 0000000..30b31cc
--- /dev/null
+++ b/backend/nijitrack.py
@@ -0,0 +1,104 @@
+import os
+import time
+
+import fileutil as fs
+from sql.sql_handler import SQLHandler
+from webapi.holodex import HolodexAPI
+from webapi.youtube import YouTubeAPI
+from decorators import *
+import argparse
+
+
+CONFIG = fs.load_config("config.ini")
+DATA_SETTING = fs.load_json_file("sql_table_config.json")
+
+
+@log("Initializing Database")
+def initialize_database(server: SQLHandler):
+ server.create_table(name = CONFIG["TABLES"]["live"], column = DATA_SETTING["LIVE_COLUMNS"])
+ server.create_table(name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["LIVE_COLUMNS"])
+ server.create_table(name = CONFIG["TABLES"]["daily"], column = DATA_SETTING["DAILY_COLUMNS"])
+
+
+@log("Inserting Live Data into Database")
+def record_subscriber_data(data: list):
+ def transform_sql_string(string: str) -> str:
+ return string.encode("ascii", "ignore").decode().replace("'", "''")
+ def record_diff_data(data_tuple: tuple, refresh_daily: bool):
+ if not server.check_row_exists(CONFIG["TABLES"]["daily"], "channel_id", channel_id):
+ # data_tuple = (channel_id, pfp, channel_name, sub_count, time.strftime('%Y-%m-%d %H:%M:%S'))
+ server.insert_row(CONFIG["TABLES"]["daily"], DATA_SETTING["DAILY_HEADER"], (data_tuple[0], data_tuple[3]))
+ server.insert_row(name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["LIVE_HEADER"], data=data_tuple)
+ return
+ elif refresh_daily:
+ server.update_row(CONFIG["TABLES"]["daily"], "channel_id", channel_id, "sub_diff", sub_count)
+ server.insert_row(name = CONFIG["TABLES"]["historical"], column = DATA_SETTING["LIVE_HEADER"], data=data_tuple)
+
+ exclude_channels = fs.get_excluded_channels()
+ refresh_daily = fs.check_diff_refresh()
+ for channel in data:
+ channel_id = channel["id"]
+ if channel_id in exclude_channels:
+ continue
+ pfp = channel["photo"]
+ sub_count = channel["subscriber_count"]
+ channel_name = channel["english_name"]
+ if channel_name is None:
+ channel_name = channel["name"]
+ channel_name = transform_sql_string(channel_name)
+ data_tuple = (channel_id, pfp, channel_name, sub_count, time.strftime('%Y-%m-%d %H:%M:%S'))
+ server.insert_row(name = CONFIG["TABLES"]["live"], column = DATA_SETTING["LIVE_HEADER"], data=data_tuple)
+ record_diff_data(data_tuple, refresh_daily)
+
+
+@log("Running Holodex Generation")
+def holodex_generation(server: SQLHandler):
+ """
+ Generates the data from the Holodex API
+ """
+ holodex_organizations = DATA_SETTING["HOLODEX_ORGS"].split(",")
+ server.clear_table(CONFIG["TABLES"]["live"])
+ server.reset_auto_increment(CONFIG["TABLES"]["live"])
+ holodex = HolodexAPI(CONFIG["API"]["holodex"])
+ for organization in holodex_organizations:
+ holodex.set_organization(organization)
+ record_subscriber_data(holodex.get_subscriber_data())
+ return holodex.get_generated_channel_data(), holodex.get_inactive_channels()
+
+@log("Running YouTube Generation")
+def youtube_generation(server: SQLHandler):
+ """
+ Generates the data from the YouTube API
+ """
+ ytapi = YouTubeAPI(CONFIG["API"]["youtube"])
+ server.clear_table(CONFIG["TABLES"]["live"])
+ server.reset_auto_increment(CONFIG["TABLES"]["live"])
+ data = ytapi.get_data_all_channels(fs.get_local_channels())
+ record_subscriber_data(data)
+ return data
+
+def combine_excluded_channel_ids(inactive_channel_data: list, excluded_channels: list):
+ """
+ Combines the local excluded channels with the inactive channels from the API
+ """
+ channel_ids = []
+ for inactive_channel in inactive_channel_data:
+ if inactive_channel in excluded_channels:
+ continue
+ channel_ids.append(inactive_channel)
+ return channel_ids
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="NijiTrack - A Subscriber Tracker")
+ parser.add_argument('--mode', choices=['yt', 'holodex'], help='Specify the data source to use (yt or holodex)')
+ args = parser.parse_args()
+ server = SQLHandler(CONFIG["SQL"]["host"], CONFIG["SQL"]["user"], CONFIG["SQL"]["password"], CONFIG["SQL"]["database"])
+ initialize_database(server)
+ if args.mode == 'yt':
+ print("Using YouTube API")
+ channel_data = youtube_generation(server)
+ inactive_channels = fs.get_excluded_channels()
+ else:
+ channel_data, inactive_channels = holodex_generation(server)
+ fs.update_excluded_channels(inactive_channels) \ No newline at end of file
diff --git a/backend/requirements.txt b/backend/requirements.txt
new file mode 100644
index 0000000..2f05f53
--- /dev/null
+++ b/backend/requirements.txt
@@ -0,0 +1,21 @@
+certifi==2023.7.22
+charset-normalizer==3.1.0
+docopt==0.6.2
+greenlet==2.0.2
+idna==3.4
+mysql-connector-python==8.0.32
+numpy==1.24.2
+packaging==23.0
+pandas==1.5.3
+pipreqs==0.4.12
+plotly==5.13.1
+protobuf==3.20.3
+python-dateutil==2.8.2
+pytz==2022.7.1
+requests==2.31.0
+six==1.16.0
+tenacity==8.2.2
+typing_extensions==4.5.0
+urllib3==1.26.15
+yarg==0.1.9
+Flask~=2.2.3 \ No newline at end of file
diff --git a/backend/sql/sql_handler.py b/backend/sql/sql_handler.py
new file mode 100644
index 0000000..9d1c10d
--- /dev/null
+++ b/backend/sql/sql_handler.py
@@ -0,0 +1,162 @@
+import mysql.connector
+from mysql.connector import Error, errorcode
+
+
+class SQLHandler:
+ def __init__(self, host_name: str, user_name: str, user_password: str, database_name: str):
+ self.host_name = host_name
+ self.username = user_name
+ self.password = user_password
+ self.database_name = database_name
+ self.connection = self._create_server_connection(
+ host_name, user_name, user_password)
+ self._load_database(database_name)
+
+ def _create_server_connection(self, host_name: str, user_name: str, user_password: str) -> mysql.connector:
+ connection = None
+ try:
+ connection = mysql.connector.connect(host=host_name, user=user_name, passwd=user_password)
+ print("MySQL Database connection successful")
+ except Error as err:
+ print(f"Error: '{err}'")
+ return connection
+
+ def get_connection(self):
+ return self.connection
+
+ def _create_database(self, cursor: str, database_name: str):
+ try:
+ cursor.execute(
+ f"CREATE DATABASE {database_name} DEFAULT CHARACTER SET 'utf8'")
+ except Error as err:
+ print(f"Failed creating database: {err}")
+ exit(1)
+
+ def _load_database(self, database_name: str):
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ cursor.execute(f"USE {database_name}")
+ print(f"Database {database_name} loaded successfully")
+ except Error as err:
+ print(f"Database {database_name} does not exist")
+ if err.errno == errorcode.ER_BAD_DB_ERROR:
+ self._create_database(cursor, database_name)
+ print(f"Database {database_name} created successfully")
+ self.connection.database = database_name
+ else:
+ print(err)
+ exit(1)
+
+ def create_table(self, name: str, column: str):
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ cursor.execute(f"CREATE TABLE {name} ({column})")
+ print(f"Table {name} created successfully")
+ except Error as err:
+ print(err)
+
+ def insert_row(self, name: str, column: str, data: tuple):
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ placeholders = ', '.join(['%s'] * len(data))
+ query = f"INSERT INTO {name} ({column}) VALUES ({placeholders})"
+ cursor.execute(query, data)
+ self.connection.commit()
+ print("Data Inserted:", data, "into", name)
+ except Error as err:
+ print("Error inserting data")
+ print(err)
+
+
+ def clear_table(self, name: str):
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ cursor.execute(f"DELETE FROM {name}")
+ self.connection.commit()
+ print("Table cleared successfully")
+ except Error as err:
+ print("Error clearing table")
+ print(err)
+
+ def reset_auto_increment(self, name: str):
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ cursor.execute(f"ALTER TABLE {name} AUTO_INCREMENT = 1")
+ self.connection.commit()
+ print("Table reset successfully")
+ except Error as err:
+ print("Error resetting table")
+ print(err)
+
+ def copy_rows_to_new_table(self, name: str, new_name: str, column: str):
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ cursor.execute(
+ f"INSERT INTO {new_name} ({column}) SELECT {column} FROM {name}")
+ cursor.execute(
+ f"ALTER TABLE {new_name} MODIFY COLUMN id INT AUTO_INCREMENT")
+ self.connection.commit()
+ print("Rows copied successfully")
+ except Error as err:
+ print("Error copying rows")
+ print(err)
+
+ def drop_table(self, name: str):
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ cursor.execute(f"DROP TABLE {name}")
+ self.connection.commit()
+ print("Table dropped successfully")
+ except Error as err:
+ print("Error dropping table")
+ print(err)
+
+ def check_row_exists(self, name: str, column_name: str, value: str):
+ """
+ Checks if a row exists in a table
+ """
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ cursor.execute(f"SELECT * FROM {name} WHERE {column_name} = '{value}'")
+ result = cursor.fetchone()
+ if result:
+ return True
+ else:
+ return False
+ except Error as err:
+ print("Error checking row")
+ print(err)
+
+ def update_row(self, name: str, column_name: str, search_val: str, replace_col:str, new_value: str):
+ """
+ Updates a row in a table
+ """
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ cursor.execute(f"UPDATE {name} SET {replace_col} = '{new_value}' WHERE {column_name} = '{search_val}'")
+ self.connection.commit()
+ print("Row updated successfully")
+ except Error as err:
+ print("Error updating row")
+ print(err)
+
+ def execute_query(self, query: str):
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ cursor.execute(query)
+ result = cursor.fetchall()
+ return result
+ except Error as err:
+ print("Error executing query")
+ print(err)
+
+ def get_query_result(self, query: str):
+ cursor = self.connection.cursor(buffered=True)
+ try:
+ cursor.execute(query)
+ result = cursor.fetchall()
+ return result
+ except Error as err:
+ print("Error executing query")
+ print(err)
+ \ No newline at end of file
diff --git a/backend/sql_table_config.json b/backend/sql_table_config.json
new file mode 100644
index 0000000..0e2e6e5
--- /dev/null
+++ b/backend/sql_table_config.json
@@ -0,0 +1,7 @@
+{
+ "LIVE_COLUMNS": "id INT PRIMARY KEY AUTO_INCREMENT, channel_id VARCHAR(255), profile_pic VARCHAR(255), name VARCHAR(255), subscriber_count INT, timestamp DATETIME",
+ "LIVE_HEADER": "channel_id, profile_pic, name, subscriber_count, timestamp",
+ "DAILY_COLUMNS": "id INT PRIMARY KEY AUTO_INCREMENT, channel_id VARCHAR(255), sub_diff INT",
+ "DAILY_HEADER": "channel_id, sub_diff",
+ "HOLODEX_ORGS": "Nijisanji"
+}
diff --git a/backend/webapi/holodex.py b/backend/webapi/holodex.py
new file mode 100644
index 0000000..5f81892
--- /dev/null
+++ b/backend/webapi/holodex.py
@@ -0,0 +1,75 @@
+from webapi.web_api import WebAPI
+from typing import Iterable
+
+
+class HolodexAPI(WebAPI):
+ """
+ Class for interacting with the Holodex API
+ """
+
+ def __init__(self,api_key: str = None,member_count: int = 300,organization: str = "Nijisanji"):
+ super().__init__(api_key=api_key, base_url="https://holodex.net/api/v2/")
+ self.member_count = member_count
+ self.organization = organization
+ self._inactive_channels = []
+ self._channel_data = []
+
+ def get_subscriber_data(self) -> Iterable:
+ """
+ Gets data for all channels in a particular organization
+ """
+ members = self.member_count
+ data = []
+ active_channels = []
+ offset = 0
+ while members > 0:
+ data += self._download_url(
+ f"channels?type=vtuber&offset={offset}&limit=100&org={self.organization}"
+ )
+ members -= 100
+ offset += 100
+ for channel in data:
+ print("DEBUG: ", channel["id"])
+ try:
+ channel["description"] = self.get_channel_description(channel["id"])
+ if channel["inactive"]:
+ self._inactive_channels.append(channel["id"])
+ continue
+ active_channels.append(channel)
+ except (KeyError, TypeError, ValueError):
+ print("DEBUG:","An error occured with parsing ", channel["id"], channel["name"])
+ continue
+ self._channel_data = active_channels
+ return active_channels
+
+ def get_view_count(self, channel_id: str) -> int:
+ """
+ Gets the view count for a particular channel
+ """
+ data = self._download_url(f"channels/{channel_id}")
+ return data["view_count"]
+
+ def get_channel_description(self, channel_id: str) -> str:
+ """
+ Gets the description for a particular channel
+ """
+ data = self._download_url(f"channels/{channel_id}")
+ return data["description"]
+
+ def set_organization(self, organization: str):
+ """
+ Sets the organization for the API
+ """
+ self.organization = organization
+
+ def get_inactive_channels(self) -> list:
+ """
+ Gets the list of inactive channels
+ """
+ return self._inactive_channels
+
+ def get_generated_channel_data(self) -> list:
+ """
+ Gets the list of channel data
+ """
+ return self._channel_data
diff --git a/backend/webapi/web_api.py b/backend/webapi/web_api.py
new file mode 100644
index 0000000..525994c
--- /dev/null
+++ b/backend/webapi/web_api.py
@@ -0,0 +1,29 @@
+import urllib.request
+import json
+
+
+class WebAPI:
+ """
+ General class for interacting with web APIs
+ """
+
+ def __init__(self, api_key: str, base_url: str) -> None:
+ self.api_key = api_key
+ self.base_url = base_url
+
+ def _download_url(self, query: str, header = 'X-APIKEY') -> dict:
+ """
+ Downloads the URL and returns the result as a string
+ param:
+ query: str - the query to be appended to the base URL
+ """
+ if self.api_key is None:
+ raise Exception("API key not set")
+ opener = urllib.request.build_opener()
+ opener.addheaders = [(header, self.api_key)]
+ urllib.request.install_opener(opener)
+ response = urllib.request.urlopen(self.base_url + query)
+ json_results = response.read()
+ r_obj = json.loads(json_results)
+ response.close()
+ return r_obj
diff --git a/backend/webapi/youtube.py b/backend/webapi/youtube.py
new file mode 100644
index 0000000..44d786f
--- /dev/null
+++ b/backend/webapi/youtube.py
@@ -0,0 +1,55 @@
+from webapi.web_api import WebAPI
+
+
+class YouTubeAPI(WebAPI):
+ """
+ Class for interacting with the YouTube API
+ """
+
+ def __init__(self, api_key: str = None):
+ self.api_key = api_key
+ self.base_url = "https://www.googleapis.com/youtube/v3/"
+
+ def _search_matching_id(self, id: str, data: list) -> dict:
+ """
+ Searches for a info matching a given ID
+ param:
+ id: str - the ID to search for
+ """
+ for entry in data:
+ if entry['id'] == id:
+ return entry
+ return None
+
+ def get_data_all_channels(self, channel_tuples: list) -> list:
+ data = []
+ members = len(channel_tuples)
+ request_chunks = [channel_tuples[i:i + 50] for i in range(0, members, 50)]
+ for chunk in request_chunks:
+ channel_ids = [x[0] for x in chunk]
+ channel_names = [x[1] for x in chunk]
+ request_string = ",".join(channel_ids)
+ stats = self._download_url(
+ f"channels?part=statistics&id={request_string}&key={self.api_key}")
+ snippet = self._download_url(
+ f"channels?part=snippet&id={request_string}&key={self.api_key}")
+ stats_list = stats['items']
+ snippet_list = snippet['items']
+ for i in range(len(stats_list)):
+ try:
+ data_entry = {'english_name': channel_names[i], 'id': channel_ids[i],
+ 'subscriber_count':
+ self._search_matching_id(channel_ids[i], stats_list)[
+ 'statistics']['subscriberCount'], 'view_count':
+ self._search_matching_id(channel_ids[i], stats_list)[
+ 'statistics']['viewCount'], 'photo':
+ self._search_matching_id(channel_ids[i], snippet_list)[
+ 'snippet']['thumbnails']['default']['url'], 'description':
+ self._search_matching_id(channel_ids[i], snippet_list)[
+ 'snippet']['description']}
+ data.append(data_entry)
+ except TypeError:
+ print("Error NoneType: " + str(channel_ids[i]))
+ except KeyError:
+ print("Error KeyError: " + str(channel_ids[i]))
+ return data
send patches to the email below
yukais@pinapelz.com
include the subject [PATCH repo_name]
pinapelz.com
homepage