From 22714994d2b7cfa238c8b2d54a3639cd6417e9b6 Mon Sep 17 00:00:00 2001 From: Pinapelz Date: Fri, 21 Nov 2025 22:30:47 -0800 Subject: scaffold implementation for remote sqlite db --- .env.template | 3 + .gitignore | 4 + common.py | 15 +++ community/wacca_plus/wacca_plus.py | 4 +- database.py | 163 -------------------------- database/base_database.py | 40 +++++++ database/local_database.py | 164 ++++++++++++++++++++++++++ database/remote_database.py | 159 +++++++++++++++++++++++++ generate.py | 7 +- middleware/package.json | 1 + middleware/pnpm-lock.yaml | 234 +++++++++++++++++++++++++++++++++++++ requirements.txt | 6 +- summarizer.py | 4 +- translate.py | 4 +- 14 files changed, 632 insertions(+), 176 deletions(-) create mode 100644 common.py delete mode 100644 database.py create mode 100644 database/base_database.py create mode 100644 database/local_database.py create mode 100644 database/remote_database.py diff --git a/.env.template b/.env.template index bef2005..4ea5b80 100644 --- a/.env.template +++ b/.env.template @@ -6,3 +6,6 @@ GOOGLE_TRANSLATE_API_KEY= KV_REST_API_TOKEN= KV_REST_API_URL= FIREBASE_SERVICE_ACCOUNT_JSON_PATH=serviceAccount.json + +SQLITE_DB="news.db" # leave empty to use local sqlite3 (news.db), otherwise provide a libsql compatible URL or path to a local .db +REMOTE_AUTH_TOKEN= # only used if remote_db diff --git a/.gitignore b/.gitignore index 0921af8..a2be204 100644 --- a/.gitignore +++ b/.gitignore @@ -180,3 +180,7 @@ summarization_cache.json wac_result_cache.json tl_cache.json key.json +serviceAccount.json +.db-info +.db-wal +.db-shm diff --git a/common.py b/common.py new file mode 100644 index 0000000..e45b5f0 --- /dev/null +++ b/common.py @@ -0,0 +1,15 @@ +from database.local_database import Database as LocalDatabase +from database.remote_database import RemoteDatabase +import os + +def create_database_connection(): + sqlite_db = os.getenv("SQLITE_DB") + if sqlite_db is None or sqlite_db == "": + return LocalDatabase("news.db") + elif sqlite_db.endswith(".db"): + return LocalDatabase(sqlite_db) + else: + auth_token = os.getenv("REMOTE_AUTH_TOKEN") + if not auth_token: + raise RuntimeError("Remote DB was specified but no auth token was provided") + return RemoteDatabase(url=sqlite_db, auth_token=auth_token) diff --git a/community/wacca_plus/wacca_plus.py b/community/wacca_plus/wacca_plus.py index 0f34814..90ec661 100644 --- a/community/wacca_plus/wacca_plus.py +++ b/community/wacca_plus/wacca_plus.py @@ -1,6 +1,6 @@ from datetime import datetime from dotenv import load_dotenv -from database import Database +from common import create_database_connection import os import time import requests @@ -78,7 +78,7 @@ def _convert_image_to_base64(img_url: str): def parse_announcement_messages(message_json: dict): news_posts = [] - database = Database() + database = create_database_connection() for message in message_json: type = None message_content = message.get("content", "") diff --git a/database.py b/database.py deleted file mode 100644 index fc476b1..0000000 --- a/database.py +++ /dev/null @@ -1,163 +0,0 @@ -import json -import os -import sqlite3 - - -class Database: - def __init__(self): - self._conn = sqlite3.connect("news.db") - self._cursor = self._conn.cursor() - self._initialize_db() - self._migrate_old_data() - - def _initialize_db(self): - with open("schema.sql") as f: - self._cursor.executescript(f.read()) - self._conn.commit() - - def close(self): - """Close the database connection""" - if self._conn: - self._conn.close() - - def _migrate_old_data(self): - """ - Migrates old summarization, tl and wac files into DB - """ - if os.path.exists("summarization_cache.json"): - print("[Database] Migrating old summarization_cache to DB") - with open("summarization_cache.json", "r") as file: - summ_cache = json.load(file) - for key, val in summ_cache.items(): - self.add_new_summary(key, val["headline"], val["content"]) - os.rename("summarization_cache.json", "summarization_cache.json.bak") - - if os.path.exists("tl_cache.json"): - print("[Database] Migrating old translation cache (tl_cache.json) to DB") - with open("tl_cache.json", "r") as file: - tl_cache = json.load(file) - import hashlib - - for entry in tl_cache: - key = hashlib.sha256( - ( - entry["source_lang"] - + entry["target_lang"] - + entry["source_txt"] - ).encode("utf-8") - ).hexdigest() - self.add_new_translation( - key, - entry["source_lang"], - entry["target_lang"], - entry["source_txt"], - entry["result_txt"], - ) - os.rename("tl_cache.json", "tl_cache.json.bak") - - if os.path.exists("wac_result_cache.json"): - print("[Database] Migrating old WAC Data cache to DB") - with open("wac_result_cache.json", "r") as file: - wac_cache = json.load(file) - import hashlib - - for key, value in wac_cache.items(): - self.add_new_wac_entry(key, value[0], value[1]) - os.rename("wac_result_cache.json", "wac_result_cache.json.bak") - - def add_new_wac_entry(self, key: str, is_news: bool, post_type: str): - news_var = 0 if is_news is False else 1 - self._cursor.execute( - "INSERT OR REPLACE INTO wacplus (id, isNews, type) VALUES (?, ?, ?)", - (key, news_var, post_type), - ) - self._conn.commit() - - def add_new_translation( - self, - key: str, - source_lang: str, - target_lang: str, - source_txt: str, - result_txt: str, - ): - self._cursor.execute( - "INSERT OR REPLACE INTO translation (id, source_lang, target_lang, source, result) VALUES (?, ?, ?, ?, ?)", - (key, source_lang, target_lang, source_txt, result_txt), - ) - self._conn.commit() - - def add_new_summary(self, key: str, headline: str, content: str): - self._cursor.execute( - "INSERT OR REPLACE INTO summarization (id, headline, content) VALUES (?, ?, ?)", - (key, headline, content), - ) - self._conn.commit() - - def add_news_entry(self, key: str, news_entry: dict): - is_ai_summary = 1 if news_entry.get("is_ai_summary", False) else 0 - en_headline = news_entry.get("en_headline", None) - en_content = news_entry.get("en_content", None) - headline = news_entry.get("headline", None) - url = news_entry.get("url", None) - self._cursor.execute( - "INSERT OR REPLACE INTO news (news_id, date, identifier, type, timestamp, headline, content, url, is_ai_summary, en_headline, en_content) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - ( - key, - news_entry["date"], - news_entry["identifier"], - news_entry["type"], - news_entry["timestamp"], - headline, - news_entry["content"], - url, - is_ai_summary, - en_headline, - en_content, - ), - ) - for image_entry in news_entry["images"]: - if image_entry["image"].startswith("data:"): - continue - link_url = image_entry.get("link", None) - self._cursor.execute( - "INSERT OR REPLACE INTO news_images (news_id, image_url, link_url) VALUES (?, ?, ?)", - (key, image_entry["image"], link_url), - ) - self._conn.commit() - - - def get_summary(self, key: str): - self._cursor.execute( - "SELECT headline, content FROM summarization WHERE id = ?", (key,) - ) - result = self._cursor.fetchone() - if result is None: - return None - return {"headline": result[0], "content": result[1]} - - def get_translation(self, key: str): - self._cursor.execute("SELECT result FROM translation WHERE id = ?", (key,)) - result = self._cursor.fetchone() - if result is None: - return None - return result[0] - - def get_wac_entry(self, key: str): - self._cursor.execute("SELECT isNews, type FROM wacplus WHERE id = ?", (key,)) - result = self._cursor.fetchone() - if result is None: - return None - is_news = True if result[0] == 1 else False - return is_news, result[1] - - def check_news_id_exists(self, key: str) -> bool: - """ - Check if a news entry with the given ID exists in the database. - - :param key: The ID of the news entry to check. - :return: True if the news entry exists, False otherwise. - """ - self._cursor.execute("SELECT 1 FROM news WHERE news_id = ?", (key,)) - result = self._cursor.fetchone() - return result is not None diff --git a/database/base_database.py b/database/base_database.py new file mode 100644 index 0000000..6b03755 --- /dev/null +++ b/database/base_database.py @@ -0,0 +1,40 @@ +from abc import ABC, abstractmethod + +class BaseDatabase(ABC): + + @abstractmethod + def close(self): + pass + + @abstractmethod + def add_new_wac_entry(self, key: str, is_news: bool, post_type: str): + pass + + @abstractmethod + def add_new_translation(self, key: str, source_lang: str, + target_lang: str, source_txt: str, result_txt: str): + pass + + @abstractmethod + def add_new_summary(self, key: str, headline: str, content: str): + pass + + @abstractmethod + def add_news_entry(self, key: str, news_entry: dict): + pass + + @abstractmethod + def get_summary(self, key: str): + pass + + @abstractmethod + def get_translation(self, key: str): + pass + + @abstractmethod + def get_wac_entry(self, key: str): + pass + + @abstractmethod + def check_news_id_exists(self, key: str) -> bool: + pass diff --git a/database/local_database.py b/database/local_database.py new file mode 100644 index 0000000..9914132 --- /dev/null +++ b/database/local_database.py @@ -0,0 +1,164 @@ +from database.base_database import BaseDatabase +import json +import os +import sqlite3 + + +class Database(BaseDatabase): + def __init__(self, local_db_path: str="news.db"): + self._conn = sqlite3.connect(local_db_path) + self._cursor = self._conn.cursor() + self._initialize_db() + self._migrate_old_data() + + def _initialize_db(self): + with open("schema.sql") as f: + self._cursor.executescript(f.read()) + self._conn.commit() + + def close(self): + """Close the database connection""" + if self._conn: + self._conn.close() + + def _migrate_old_data(self): + """ + Migrates old summarization, tl and wac files into DB + """ + if os.path.exists("summarization_cache.json"): + print("[Database] Migrating old summarization_cache to DB") + with open("summarization_cache.json", "r") as file: + summ_cache = json.load(file) + for key, val in summ_cache.items(): + self.add_new_summary(key, val["headline"], val["content"]) + os.rename("summarization_cache.json", "summarization_cache.json.bak") + + if os.path.exists("tl_cache.json"): + print("[Database] Migrating old translation cache (tl_cache.json) to DB") + with open("tl_cache.json", "r") as file: + tl_cache = json.load(file) + import hashlib + + for entry in tl_cache: + key = hashlib.sha256( + ( + entry["source_lang"] + + entry["target_lang"] + + entry["source_txt"] + ).encode("utf-8") + ).hexdigest() + self.add_new_translation( + key, + entry["source_lang"], + entry["target_lang"], + entry["source_txt"], + entry["result_txt"], + ) + os.rename("tl_cache.json", "tl_cache.json.bak") + + if os.path.exists("wac_result_cache.json"): + print("[Database] Migrating old WAC Data cache to DB") + with open("wac_result_cache.json", "r") as file: + wac_cache = json.load(file) + import hashlib + + for key, value in wac_cache.items(): + self.add_new_wac_entry(key, value[0], value[1]) + os.rename("wac_result_cache.json", "wac_result_cache.json.bak") + + def add_new_wac_entry(self, key: str, is_news: bool, post_type: str): + news_var = 0 if is_news is False else 1 + self._cursor.execute( + "INSERT OR REPLACE INTO wacplus (id, isNews, type) VALUES (?, ?, ?)", + (key, news_var, post_type), + ) + self._conn.commit() + + def add_new_translation( + self, + key: str, + source_lang: str, + target_lang: str, + source_txt: str, + result_txt: str, + ): + self._cursor.execute( + "INSERT OR REPLACE INTO translation (id, source_lang, target_lang, source, result) VALUES (?, ?, ?, ?, ?)", + (key, source_lang, target_lang, source_txt, result_txt), + ) + self._conn.commit() + + def add_new_summary(self, key: str, headline: str, content: str): + self._cursor.execute( + "INSERT OR REPLACE INTO summarization (id, headline, content) VALUES (?, ?, ?)", + (key, headline, content), + ) + self._conn.commit() + + def add_news_entry(self, key: str, news_entry: dict): + is_ai_summary = 1 if news_entry.get("is_ai_summary", False) else 0 + en_headline = news_entry.get("en_headline", None) + en_content = news_entry.get("en_content", None) + headline = news_entry.get("headline", None) + url = news_entry.get("url", None) + self._cursor.execute( + "INSERT OR REPLACE INTO news (news_id, date, identifier, type, timestamp, headline, content, url, is_ai_summary, en_headline, en_content) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + key, + news_entry["date"], + news_entry["identifier"], + news_entry["type"], + news_entry["timestamp"], + headline, + news_entry["content"], + url, + is_ai_summary, + en_headline, + en_content, + ), + ) + for image_entry in news_entry["images"]: + if image_entry["image"].startswith("data:"): + continue + link_url = image_entry.get("link", None) + self._cursor.execute( + "INSERT OR REPLACE INTO news_images (news_id, image_url, link_url) VALUES (?, ?, ?)", + (key, image_entry["image"], link_url), + ) + self._conn.commit() + + + def get_summary(self, key: str): + self._cursor.execute( + "SELECT headline, content FROM summarization WHERE id = ?", (key,) + ) + result = self._cursor.fetchone() + if result is None: + return None + return {"headline": result[0], "content": result[1]} + + def get_translation(self, key: str): + self._cursor.execute("SELECT result FROM translation WHERE id = ?", (key,)) + result = self._cursor.fetchone() + if result is None: + return None + return result[0] + + def get_wac_entry(self, key: str): + self._cursor.execute("SELECT isNews, type FROM wacplus WHERE id = ?", (key,)) + result = self._cursor.fetchone() + if result is None: + return None + is_news = True if result[0] == 1 else False + return is_news, result[1] + + def check_news_id_exists(self, key: str) -> bool: + """ + Check if a news entry with the given ID exists in the database. + + :param key: The ID of the news entry to check. + :return: True if the news entry exists, False otherwise. + """ + self._cursor.execute("SELECT 1 FROM news WHERE news_id = ?", (key,)) + result = self._cursor.fetchone() + return result is not None diff --git a/database/remote_database.py b/database/remote_database.py new file mode 100644 index 0000000..958d815 --- /dev/null +++ b/database/remote_database.py @@ -0,0 +1,159 @@ +from database.base_database import BaseDatabase +import libsql + + +class RemoteDatabase(BaseDatabase): + def __init__(self, url: str = None, auth_token: str = None, local_replica: str = None): + """ + Initialize connection to Remote database using libsql (designed for Turso) + """ + self._url = url + self._auth_token = auth_token + self._local_replica = local_replica or "new_db.db" + + if not self._url: + raise ValueError("Database URL must be provided either as parameter or TURSO_DATABASE_URL environment variable") + + if not self._auth_token: + raise ValueError("Auth token must be provided either as parameter or TURSO_AUTH_TOKEN environment variable") + + self._conn = libsql.connect( + self._local_replica, + sync_url=self._url, + auth_token=self._auth_token + ) + + # Initial sync to get latest data + self._conn.sync() + self._initialize_db() + + def _initialize_db(self): + """Initialize database schema""" + with open("schema.sql") as f: + # Execute schema creation + self._conn.executescript(f.read()) + self._conn.commit() + + def close(self): + """Close the database connection""" + if self._conn: + self._conn.close() + + def add_new_wac_entry(self, key: str, is_news: bool, post_type: str): + """Add a new WAC entry to the database""" + news_var = 0 if is_news is False else 1 + self._conn.execute( + "INSERT OR REPLACE INTO wacplus (id, isNews, type) VALUES (?, ?, ?)", + (key, news_var, post_type), + ) + self._conn.commit() + + def add_new_translation( + self, + key: str, + source_lang: str, + target_lang: str, + source_txt: str, + result_txt: str, + ): + """Add a new translation to the database""" + self._conn.execute( + "INSERT OR REPLACE INTO translation (id, source_lang, target_lang, source, result) VALUES (?, ?, ?, ?, ?)", + (key, source_lang, target_lang, source_txt, result_txt), + ) + self._conn.commit() + + def add_new_summary(self, key: str, headline: str, content: str): + """Add a new summary to the database""" + self._conn.execute( + "INSERT OR REPLACE INTO summarization (id, headline, content) VALUES (?, ?, ?)", + (key, headline, content), + ) + self._conn.commit() + + def add_news_entry(self, key: str, news_entry: dict): + """Add a new news entry to the database""" + is_ai_summary = 1 if news_entry.get("is_ai_summary", False) else 0 + en_headline = news_entry.get("en_headline", None) + en_content = news_entry.get("en_content", None) + headline = news_entry.get("headline", None) + url = news_entry.get("url", None) + + self._conn.execute( + "INSERT OR REPLACE INTO news (news_id, date, identifier, type, timestamp, headline, content, url, is_ai_summary, en_headline, en_content) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + key, + news_entry["date"], + news_entry["identifier"], + news_entry["type"], + news_entry["timestamp"], + headline, + news_entry["content"], + url, + is_ai_summary, + en_headline, + en_content, + ), + ) + + # Add associated images + for image_entry in news_entry["images"]: + if image_entry["image"].startswith("data:"): + continue + link_url = image_entry.get("link", None) + self._conn.execute( + "INSERT OR REPLACE INTO news_images (news_id, image_url, link_url) VALUES (?, ?, ?)", + (key, image_entry["image"], link_url), + ) + self._conn.commit() + + def get_summary(self, key: str): + """Get a summary by key""" + result = self._conn.execute( + "SELECT headline, content FROM summarization WHERE id = ?", (key,) + ).fetchone() + if result is None: + return None + return {"headline": result[0], "content": result[1]} + + def get_translation(self, key: str): + """Get a translation by key""" + result = self._conn.execute("SELECT result FROM translation WHERE id = ?", (key,)).fetchone() + if result is None: + return None + return result[0] + + def get_wac_entry(self, key: str): + """Get a WAC entry by key""" + result = self._conn.execute("SELECT isNews, type FROM wacplus WHERE id = ?", (key,)).fetchone() + if result is None: + return None + is_news = True if result[0] == 1 else False + return is_news, result[1] + + def check_news_id_exists(self, key: str) -> bool: + """ + Check if a news entry with the given ID exists in the database. + + :param key: The ID of the news entry to check. + :return: True if the news entry exists, False otherwise. + """ + result = self._conn.execute("SELECT 1 FROM news WHERE news_id = ?", (key,)).fetchone() + return result is not None + + def sync(self): + """Sync local changes with the remote database (Turso specific)""" + self._conn.sync() + + def get_stats(self): + """Get database statistics (useful for monitoring)""" + stats = {} + + # Count records in each table + tables = ['news', 'news_images', 'summarization', 'translation', 'wacplus'] + for table in tables: + result = self._conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone() + count = result[0] if result else 0 + stats[f"{table}_count"] = count + + return stats diff --git a/generate.py b/generate.py index 4fd5735..2531f1c 100644 --- a/generate.py +++ b/generate.py @@ -10,7 +10,7 @@ import hashlib import os from dotenv import load_dotenv from datetime import datetime, timedelta -from database import Database +from common import create_database_connection from feed import build_rss_from_news_feed from notifications import check_can_send_notifs, broadcast_to_topic @@ -27,10 +27,9 @@ def compute_json_hash(data): json.dumps(data, sort_keys=True).encode("utf-8") ).hexdigest() - def save_news_to_db(news_feed: list): log_output("Writing news to local save database. This is purely for archival reasons") - database = Database() + database = create_database_connection() for entry in news_feed: key = compute_json_hash(entry) database.add_news_entry(key, entry) @@ -68,7 +67,7 @@ def attempt_broadcast_notifications(news_data: list, title: str, topic: str, ima if not check_can_send_notifs: print("[WARNING] Skipping notifications as env vars are not properly configured. See template") else: - database = Database() + database = create_database_connection() cutoff = datetime.now() - timedelta(days=limit) for entry in news_data: if datetime.fromtimestamp(entry["timestamp"]) < cutoff: diff --git a/middleware/package.json b/middleware/package.json index 573ce36..f2d772e 100644 --- a/middleware/package.json +++ b/middleware/package.json @@ -9,6 +9,7 @@ "lint": "next lint" }, "dependencies": { + "@libsql/client": "^0.15.15", "@next/font": "14.2.15", "@types/node": "24.0.8", "@types/react": "19.1.8", diff --git a/middleware/pnpm-lock.yaml b/middleware/pnpm-lock.yaml index 03cf243..2f92b7b 100644 --- a/middleware/pnpm-lock.yaml +++ b/middleware/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@libsql/client': + specifier: ^0.15.15 + version: 0.15.15 '@next/font': specifier: 14.2.15 version: 14.2.15(next@15.3.4(react-dom@19.1.0(react@19.1.0))(react@19.1.0)) @@ -240,9 +243,73 @@ packages: cpu: [x64] os: [win32] + '@libsql/client@0.15.15': + resolution: {integrity: sha512-twC0hQxPNHPKfeOv3sNT6u2pturQjLcI+CnpTM0SjRpocEGgfiZ7DWKXLNnsothjyJmDqEsBQJ5ztq9Wlu470w==} + + '@libsql/core@0.15.15': + resolution: {integrity: sha512-C88Z6UKl+OyuKKPwz224riz02ih/zHYI3Ho/LAcVOgjsunIRZoBw7fjRfaH9oPMmSNeQfhGklSG2il1URoOIsA==} + + '@libsql/darwin-arm64@0.5.22': + resolution: {integrity: sha512-4B8ZlX3nIDPndfct7GNe0nI3Yw6ibocEicWdC4fvQbSs/jdq/RC2oCsoJxJ4NzXkvktX70C1J4FcmmoBy069UA==} + cpu: [arm64] + os: [darwin] + + '@libsql/darwin-x64@0.5.22': + resolution: {integrity: sha512-ny2HYWt6lFSIdNFzUFIJ04uiW6finXfMNJ7wypkAD8Pqdm6nAByO+Fdqu8t7sD0sqJGeUCiOg480icjyQ2/8VA==} + cpu: [x64] + os: [darwin] + + '@libsql/hrana-client@0.7.0': + resolution: {integrity: sha512-OF8fFQSkbL7vJY9rfuegK1R7sPgQ6kFMkDamiEccNUvieQ+3urzfDFI616oPl8V7T9zRmnTkSjMOImYCAVRVuw==} + + '@libsql/isomorphic-fetch@0.3.1': + resolution: {integrity: sha512-6kK3SUK5Uu56zPq/Las620n5aS9xJq+jMBcNSOmjhNf/MUvdyji4vrMTqD7ptY7/4/CAVEAYDeotUz60LNQHtw==} + engines: {node: '>=18.0.0'} + + '@libsql/isomorphic-ws@0.1.5': + resolution: {integrity: sha512-DtLWIH29onUYR00i0GlQ3UdcTRC6EP4u9w/h9LxpUZJWRMARk6dQwZ6Jkd+QdwVpuAOrdxt18v0K2uIYR3fwFg==} + + '@libsql/linux-arm-gnueabihf@0.5.22': + resolution: {integrity: sha512-3Uo3SoDPJe/zBnyZKosziRGtszXaEtv57raWrZIahtQDsjxBVjuzYQinCm9LRCJCUT5t2r5Z5nLDPJi2CwZVoA==} + cpu: [arm] + os: [linux] + + '@libsql/linux-arm-musleabihf@0.5.22': + resolution: {integrity: sha512-LCsXh07jvSojTNJptT9CowOzwITznD+YFGGW+1XxUr7fS+7/ydUrpDfsMX7UqTqjm7xG17eq86VkWJgHJfvpNg==} + cpu: [arm] + os: [linux] + + '@libsql/linux-arm64-gnu@0.5.22': + resolution: {integrity: sha512-KSdnOMy88c9mpOFKUEzPskSaF3VLflfSUCBwas/pn1/sV3pEhtMF6H8VUCd2rsedwoukeeCSEONqX7LLnQwRMA==} + cpu: [arm64] + os: [linux] + + '@libsql/linux-arm64-musl@0.5.22': + resolution: {integrity: sha512-mCHSMAsDTLK5YH//lcV3eFEgiR23Ym0U9oEvgZA0667gqRZg/2px+7LshDvErEKv2XZ8ixzw3p1IrBzLQHGSsw==} + cpu: [arm64] + os: [linux] + + '@libsql/linux-x64-gnu@0.5.22': + resolution: {integrity: sha512-kNBHaIkSg78Y4BqAdgjcR2mBilZXs4HYkAmi58J+4GRwDQZh5fIUWbnQvB9f95DkWUIGVeenqLRFY2pcTmlsew==} + cpu: [x64] + os: [linux] + + '@libsql/linux-x64-musl@0.5.22': + resolution: {integrity: sha512-UZ4Xdxm4pu3pQXjvfJiyCzZop/9j/eA2JjmhMaAhe3EVLH2g11Fy4fwyUp9sT1QJYR1kpc2JLuybPM0kuXv/Tg==} + cpu: [x64] + os: [linux] + + '@libsql/win32-x64-msvc@0.5.22': + resolution: {integrity: sha512-Fj0j8RnBpo43tVZUVoNK6BV/9AtDUM5S7DF3LB4qTYg1LMSZqi3yeCneUTLJD6XomQJlZzbI4mst89yspVSAnA==} + cpu: [x64] + os: [win32] + '@napi-rs/wasm-runtime@0.2.11': resolution: {integrity: sha512-9DPkXtvHydrcOsopiYpUgPHpmj0HWZKMUnL2dZqpvC42lsratuBG06V5ipyno0fUek5VlFsNQ+AcFATSrJXgMA==} + '@neon-rs/load@0.0.4': + resolution: {integrity: sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==} + '@next/env@15.3.4': resolution: {integrity: sha512-ZkdYzBseS6UjYzz6ylVKPOK+//zLWvD6Ta+vpoye8cW11AjiQjGYVibF0xuvT4L0iJfAPfZLFidaEzAOywyOAQ==} @@ -362,6 +429,9 @@ packages: '@types/react@19.1.8': resolution: {integrity: sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g==} + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@typescript-eslint/eslint-plugin@8.35.1': resolution: {integrity: sha512-9XNTlo7P7RJxbVeICaIIIEipqxLKguyh+3UbXuT2XQuFp6d8VOeDEGuz5IiX0dgZo8CiI6aOFLg4e8cF71SFVg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -718,6 +788,10 @@ packages: damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + data-view-buffer@1.0.2: resolution: {integrity: sha512-EmKO5V3OLXh1rtK2wgXRansaK1/mtVdTUEiEI0W8RkvgT05kfxaH29PliLnpLP73yYO6142Q72QNa8Wx/A5CqQ==} engines: {node: '>= 0.4'} @@ -758,6 +832,10 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} + detect-libc@2.0.2: + resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==} + engines: {node: '>=8'} + detect-libc@2.0.4: resolution: {integrity: sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==} engines: {node: '>=8'} @@ -962,6 +1040,10 @@ packages: picomatch: optional: true + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + fflate@0.7.4: resolution: {integrity: sha512-5u2V/CDW15QM1XbbgS+0DfPxVB+jUKhWEKuuFuHncbk3tEEqzmoXL+2KyOFuKGqOnmdIy0/davWF1CkuwtibCw==} @@ -988,6 +1070,10 @@ packages: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} @@ -1206,6 +1292,9 @@ packages: resolution: {integrity: sha512-H0dkQoCa3b2VEeKQBOxFph+JAbcrQdE7KC0UkqwpLmv2EC4P41QXP+rqo9wYodACiG5/WM5s9oDApTU8utwj9g==} engines: {node: '>= 0.4'} + js-base64@3.7.8: + resolution: {integrity: sha512-hNngCeKxIUQiEUN3GPJOkz4wF/YvdUdbNL9hsBcMQTkKzboD7T/q3OYOuuPZLUE6dBxSGpwhk5mwuDud7JVAow==} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -1244,6 +1333,11 @@ packages: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} + libsql@0.5.22: + resolution: {integrity: sha512-NscWthMQt7fpU8lqd7LXMvT9pi+KhhmTHAJWUB/Lj6MWa0MKFv0F2V4C6WKKpjCVZl0VwcDz4nOI3CyaT1DDiA==} + cpu: [x64, arm64, wasm32, arm] + os: [darwin, linux, win32] + linebreak@1.1.0: resolution: {integrity: sha512-MHp03UImeVhB7XZtjd0E4n6+3xr5Dq/9xI/5FptGk5FrbDR3zagPa2DS6U8ks/3HjbKWG9Q1M2ufOzxV2qLYSQ==} @@ -1320,6 +1414,15 @@ packages: sass: optional: true + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -1419,6 +1522,9 @@ packages: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} engines: {node: '>= 0.6.0'} + promise-limit@2.7.0: + resolution: {integrity: sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==} + prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} @@ -1680,6 +1786,10 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + which-boxed-primitive@1.1.1: resolution: {integrity: sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==} engines: {node: '>= 0.4'} @@ -1705,6 +1815,18 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + ws@8.18.3: + resolution: {integrity: sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -1872,6 +1994,68 @@ snapshots: '@img/sharp-win32-x64@0.34.2': optional: true + '@libsql/client@0.15.15': + dependencies: + '@libsql/core': 0.15.15 + '@libsql/hrana-client': 0.7.0 + js-base64: 3.7.8 + libsql: 0.5.22 + promise-limit: 2.7.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@libsql/core@0.15.15': + dependencies: + js-base64: 3.7.8 + + '@libsql/darwin-arm64@0.5.22': + optional: true + + '@libsql/darwin-x64@0.5.22': + optional: true + + '@libsql/hrana-client@0.7.0': + dependencies: + '@libsql/isomorphic-fetch': 0.3.1 + '@libsql/isomorphic-ws': 0.1.5 + js-base64: 3.7.8 + node-fetch: 3.3.2 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@libsql/isomorphic-fetch@0.3.1': {} + + '@libsql/isomorphic-ws@0.1.5': + dependencies: + '@types/ws': 8.18.1 + ws: 8.18.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@libsql/linux-arm-gnueabihf@0.5.22': + optional: true + + '@libsql/linux-arm-musleabihf@0.5.22': + optional: true + + '@libsql/linux-arm64-gnu@0.5.22': + optional: true + + '@libsql/linux-arm64-musl@0.5.22': + optional: true + + '@libsql/linux-x64-gnu@0.5.22': + optional: true + + '@libsql/linux-x64-musl@0.5.22': + optional: true + + '@libsql/win32-x64-msvc@0.5.22': + optional: true + '@napi-rs/wasm-runtime@0.2.11': dependencies: '@emnapi/core': 1.4.3 @@ -1879,6 +2063,8 @@ snapshots: '@tybys/wasm-util': 0.9.0 optional: true + '@neon-rs/load@0.0.4': {} + '@next/env@15.3.4': {} '@next/eslint-plugin-next@15.3.4': @@ -1967,6 +2153,10 @@ snapshots: dependencies: csstype: 3.1.3 + '@types/ws@8.18.1': + dependencies: + '@types/node': 24.0.8 + '@typescript-eslint/eslint-plugin@8.35.1(@typescript-eslint/parser@8.35.1(eslint@9.30.0)(typescript@5.8.3))(eslint@9.30.0)(typescript@5.8.3)': dependencies: '@eslint-community/regexpp': 4.12.1 @@ -2328,6 +2518,8 @@ snapshots: damerau-levenshtein@1.0.8: {} + data-uri-to-buffer@4.0.1: {} + data-view-buffer@1.0.2: dependencies: call-bound: 1.0.4 @@ -2368,6 +2560,8 @@ snapshots: has-property-descriptors: 1.0.2 object-keys: 1.1.1 + detect-libc@2.0.2: {} + detect-libc@2.0.4: optional: true @@ -2717,6 +2911,11 @@ snapshots: optionalDependencies: picomatch: 4.0.2 + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + fflate@0.7.4: {} file-entry-cache@8.0.0: @@ -2743,6 +2942,10 @@ snapshots: dependencies: is-callable: 1.2.7 + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + function-bind@1.1.2: {} function.prototype.name@1.1.8: @@ -2976,6 +3179,8 @@ snapshots: has-symbols: 1.1.0 set-function-name: 2.0.2 + js-base64@3.7.8: {} + js-tokens@4.0.0: {} js-yaml@4.1.0: @@ -3014,6 +3219,21 @@ snapshots: prelude-ls: 1.2.1 type-check: 0.4.0 + libsql@0.5.22: + dependencies: + '@neon-rs/load': 0.0.4 + detect-libc: 2.0.2 + optionalDependencies: + '@libsql/darwin-arm64': 0.5.22 + '@libsql/darwin-x64': 0.5.22 + '@libsql/linux-arm-gnueabihf': 0.5.22 + '@libsql/linux-arm-musleabihf': 0.5.22 + '@libsql/linux-arm64-gnu': 0.5.22 + '@libsql/linux-arm64-musl': 0.5.22 + '@libsql/linux-x64-gnu': 0.5.22 + '@libsql/linux-x64-musl': 0.5.22 + '@libsql/win32-x64-msvc': 0.5.22 + linebreak@1.1.0: dependencies: base64-js: 0.0.8 @@ -3085,6 +3305,14 @@ snapshots: - '@babel/core' - babel-plugin-macros + node-domexception@1.0.0: {} + + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + object-assign@4.1.1: {} object-inspect@1.13.4: {} @@ -3187,6 +3415,8 @@ snapshots: process@0.11.10: {} + promise-limit@2.7.0: {} + prop-types@15.8.1: dependencies: loose-envify: 1.4.0 @@ -3562,6 +3792,8 @@ snapshots: dependencies: punycode: 2.3.1 + web-streams-polyfill@3.3.3: {} + which-boxed-primitive@1.1.1: dependencies: is-bigint: 1.1.0 @@ -3609,6 +3841,8 @@ snapshots: word-wrap@1.2.5: {} + ws@8.18.3: {} + yocto-queue@0.1.0: {} yoga-wasm-web@0.3.3: {} diff --git a/requirements.txt b/requirements.txt index 2b2c392..dbfa718 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ beautifulsoup4==4.14.2 firebase_admin==7.1.0 -openai==2.1.0 -python-dotenv==1.1.1 +openai==2.8.1 +python-dotenv==1.2.1 pytz==2025.2 -selenium==4.36.0 +selenium==4.38.0 diff --git a/summarizer.py b/summarizer.py index 25d1f8f..0c30347 100644 --- a/summarizer.py +++ b/summarizer.py @@ -1,5 +1,5 @@ from dotenv import load_dotenv -from database import Database +from common import create_database_connection import openai import json import hashlib @@ -26,7 +26,7 @@ def generate_headline_and_content_from_images(img_urls: list[str], game: str, me # Limit message content to 500 characters if len(message_content) > MAX_CHAR_CONTENT_CONSIDERATION_LENGTH: message_content = message_content[:MAX_CHAR_CONTENT_CONSIDERATION_LENGTH] - database = Database() + database = create_database_connection() cache_key = _make_cache_key(game, img_urls) cache_entry = database.get_summary(cache_key) if cache_entry: diff --git a/translate.py b/translate.py index 31206a4..dd15cff 100644 --- a/translate.py +++ b/translate.py @@ -1,5 +1,5 @@ from dotenv import load_dotenv -from database import Database +from common import create_database_connection import requests import constants import re @@ -41,7 +41,7 @@ def request_google_translate(text: str, source: str="ja", target="en") -> tuple: Translates input text and returns the translated text using Google Cloud Translation API. """ key = hashlib.sha256((source + target + text).encode('utf-8')).hexdigest() - database = Database() + database = create_database_connection() tl_result = database.get_translation(key) if tl_result: return tl_result -- cgit v1.2.3