1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
import os
import sys
import time
import sql.graph_builder as gb
import sql.table_builder as tb
import fileutil as fs
import html_gen.html_gen as ht
from sql.sql_handler import SQLHandler
from webapi.holodex import HolodexAPI
from webapi.youtube import YouTubeAPI
from config import *
def update_database(server: SQLHandler, data):
server.clear_table(TABLE_NAME)
server.reset_auto_increment(TABLE_NAME)
excluded_channels = fs.get_excluded_channels()
refresh_diff_table = fs.check_diff_refresh()
for channel in data:
channel_id = channel['id']
if channel_id in excluded_channels:
continue
pfp = channel['photo']
name = channel['english_name']
sub_count = channel['subscriber_count']
if name is None: # if the channel doesn't have an english name, use the japanese name
name = channel['name']
name = name.encode("ascii", "ignore").decode()
data_row = f"'{channel_id}', '{pfp}','{name}', {sub_count}, '{time.strftime('%Y-%m-%d %H:%M:%S')}'"
server.create_table(DAY_DIFF_TABLE_NAME, DIFF_COLUMNS)
# Difference should only be calculated every 24 hours
# If the channel is new then calculate now, else then make sure 24 hours has passed since last reading
if refresh_diff_table or not server.check_row_exists(DAY_DIFF_TABLE_NAME, "channel_id", channel_id):
if not server.check_row_exists(DAY_DIFF_TABLE_NAME, "channel_id", channel_id):
server.insert_data(DAY_DIFF_TABLE_NAME,
diff_columns, f"'{channel_id}', {sub_count}")
else:
server.update_row(DAY_DIFF_TABLE_NAME, "channel_id", channel_id, "sub_diff", sub_count)
server.insert_data(HISTORICAL_TABLE_NAME, table_columns, data_row)
# make updates to the main table
server.insert_data(TABLE_NAME, table_columns, data_row)
name_table = "channel_"+channel_id.lower()+"_subscriber_data"
name_table = name_table.replace("-", "$")
server.create_table(name_table, TABLE_COLUMNS)
server.insert_data(name_table, table_columns, data_row)
# server.drop_table(name_table)
def generate_individual_stats(server: SQLHandler, data):
for channel in data:
channel_id = channel['id']
if channel_id in fs.get_excluded_channels():
continue
name = channel['english_name']
desc = channel['description']
pfp = channel['photo']
sub_count = channel['subscriber_count']
sub_count_str = "{:,.0f}".format(int(sub_count))
if name is None:
name = channel['name']
name = name.encode("ascii", "ignore").decode()
print("GENERATING PAGE FOR "+name+"...")
# Calculate table key name stored in the database
table_key = "channel_"+channel_id.lower()+"_subscriber_data"
table_key = table_key.replace("-", "$")
# Generate missing directories
if not os.path.exists("stats"):
os.mkdir("stats")
if not os.path.exists("tables"):
os.mkdir("tables")
with open(ROOT_STORAGE_PATH+"stats/"+name+".html", "w", encoding="utf-8") as file:
range_query = f"SELECT name, subscriber_count, timestamp FROM {table_key} WHERE timestamp >= DATE_SUB(CURRENT_DATE(), INTERVAL 7 DAY) ORDER BY timestamp DESC"
group_query = f"SELECT name, subscriber_count, timestamp FROM {table_key} GROUP BY DATE(timestamp) ORDER BY timestamp DESC"
file.write(ht.generate_meta_data(name + " - "+ META_DATA_TITLE, META_DATA_DESCRIPTION, META_PROFILE_PIC)+"\n"+
ht.side_swipe_header(sub_count_str + " Subscribers", COLOR_THEME, ROOT_URL)+"\n"+
ht.generate_info_card(name, channel_id, pfp, desc)+
gb.generate_projection(server.host_name, server.username, server.password, DB_NAME, table_key, int(sub_count), TIMEZONE_STR)+
gb.plot_subscriber_count_over_time(server.host_name, server.username, server.password, DB_NAME, table_name=table_key, gtitle=name+" Subscriber Trend", overrideQuery=group_query) +
ht.generate_html_divider("Recent Subscriber Data:") +
gb.plot_subscriber_count_over_time(server.host_name, server.username, server.password, DB_NAME, table_name=table_key, gtitle=name+" Last 7 Days Trend", overrideQuery=range_query, markers="lines+markers") +
tb.generate_individual_table(server.host_name, server.username, server.password, DB_NAME, table_key)+
ht.generate_full_table_button(ROOT_URL+"/tables/"+name) +
ht.generate_doctype_footer()
)
with open(ROOT_STORAGE_PATH+"tables/"+name+".html", "w", encoding="utf-8") as file:
file.write(ht.generate_meta_data(name + " - "+ META_DATA_TITLE, META_DATA_DESCRIPTION, META_PROFILE_PIC)+"\n"+
ht.side_swipe_header(name, COLOR_THEME, ROOT_URL)+"\n"+
ht.generate_info_card(name, channel_id, pfp, desc)+
tb.generate_individual_table(server.host_name, server.username, server.password, DB_NAME, table_key, param="") +
ht.generate_doctype_footer())
def main(mode=0):
address, user, password = fs.get_login_data()
server = SQLHandler(address, user, password, DB_NAME)
server.create_table(TABLE_NAME, TABLE_COLUMNS)
server.create_table(HISTORICAL_TABLE_NAME, TABLE_COLUMNS)
server.create_table(DAY_DIFF_TABLE_NAME, DIFF_COLUMNS)
excluded_channels = []
if mode == 0:
print("Running Holodex API Data Collection")
hldex = HolodexAPI(fs.get_api_key("holodex_api_key"), member_count=ORG_MEMBER_COUNT, organization=HOLODEX_ORG)
data = hldex.get_data_all_channels()
excluded_channels += hldex.get_exclude_channels()
elif mode == 1:
print("Running YouTube API Data Collection")
ytapi = YouTubeAPI(fs.get_api_key("youtube_api_key"))
data = ytapi.get_data_all_channels(fs.get_local_channels())
# Updating DB and generating HTML data
update_database(server, data)
excluded_channels += fs.get_excluded_channels()
with open(ROOT_STORAGE_PATH+"index.html", "w", encoding="utf-8") as file:
file.write(ht.generate_meta_data(META_DATA_TITLE, META_DATA_DESCRIPTION, META_PROFILE_PIC) +
ht.generate_title_banner(HEADER_TITLE, COLOR_THEME) +
gb.plot_subscriber_count_over_time(address, user, password, DB_NAME, HISTORICAL_TABLE_NAME, gtitle="Subscriber Count Over Time", exclude_channels=excluded_channels) +
"\n"+ht.generate_html_divider("Last Updated: "+time.strftime('%Y-%m-%d %H:%M:%S')+" "+ TIMEZONE) +
tb.sql_to_html_table(address, user, password, DB_NAME, TABLE_NAME, root_url=ROOT_URL) +
ht.build_footer_info(FOOTER_MESSAGE) +
ht.generate_doctype_footer())
# Generating individual pages
generate_individual_stats(server, data)
def generate_channel_files():
"""
Generates the channels.txt and exclude_channels.txt files based on Holodex listings
"""
if not UPDATE_LOCAL_RECORDS:
return
print("Running Channel Files Update")
hldex = HolodexAPI(fs.get_api_key("holodex_api_key"), member_count = ORG_MEMBER_COUNT,
organization = HOLODEX_ORG)
hldex.get_data_all_channels()
if not os.path.exists("data"):
os.mkdir("data")
with open("data/channels.txt", "w", encoding="utf-8") as file:
file.write("\n".join(hldex.get_active_channels()))
with open("data/exclude_channels.txt", "w", encoding="utf-8") as file:
file.write("\n".join(hldex.get_exclude_channels()))
print("Success! Channel Files Updated!")
if __name__ == "__main__":
MODE = 1
if len(sys.argv) > 1:
MODE = int(sys.argv[1])
ROOT_STORAGE_PATH = sys.argv[2]
main(MODE)
|
