pgw00g/artemis
1
0
Fork 0
forked from Dniel97/artemis
Code Pull Requests Activity

move to alembic

Browse Source
This commit is contained in:
Hay1tsme 2024-01-09 13:57:59 -05:00
parent 07cbbcc377
commit edd3ce8ead
19 changed files with 152 additions and 368 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
core/data/alembic/alembic.ini
View File

@ -1,22 +1,11 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location=.
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory.
prepend_sys_path = .
# timezone to use when rendering the date
# within the migration file as well as the filename.
# string value is passed to dateutil.tz.gettz()
# leave blank for localtime
# timezone =
# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40
@ -31,28 +20,14 @@ prepend_sys_path = .
# sourceless = false
# version location specification; this defaults
# to ./versions. When using multiple version
# to migrations//versions. When using multiple version
# directories, initial revisions must be specified with --version-path
# version_locations = %(here)s/bar %(here)s/bat ./versions
# version_locations = %(here)s/bar %(here)s/bat migrations//versions
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
sqlalchemy.url = driver://user:pass@localhost/dbname
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks=black
# black.type=console_scripts
# black.entrypoint=black
# black.options=-l 79
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic

36
core/data/alembic/env.py
View File

@ -1,9 +1,9 @@
from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
from core.data.schema.base import metadata
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
@ -17,7 +17,7 @@ fileConfig(config.config_file_name)
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
target_metadata = metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
@ -37,13 +37,11 @@ def run_migrations_offline():
script output.
"""
raise Exception('Not implemented or configured!')
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
url=url, target_metadata=target_metadata, literal_binds=True)
with context.begin_transaction():
context.run_migrations()
@ -56,21 +54,27 @@ def run_migrations_online():
and associate a connection with the context.
"""
ini_section = config.get_section(config.config_ini_section)
overrides = context.get_x_argument(as_dictionary=True)
for override in overrides:
ini_section[override] = overrides[override]
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
ini_section,
prefix='sqlalchemy.',
poolclass=pool.NullPool)
with connectable.connect() as connection:
context.configure(
connection=connection, target_metadata=target_metadata
connection=connection,
target_metadata=target_metadata,
compare_type=True,
compare_server_default=True,
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:

24
core/data/alembic/versions/835b862f9bf0_initial_migration.py Normal file
View File

@ -0,0 +1,24 @@
"""Initial Migration
Revision ID: 835b862f9bf0
Revises:
Create Date: 2024-01-09 13:06:10.787432
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '835b862f9bf0'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
pass
def downgrade():
pass

29
core/data/alembic/versions/d8950c7ce2fc_remove_old_db_mgmt_system.py Normal file
View File

@ -0,0 +1,29 @@
"""Remove old db mgmt system
Revision ID: d8950c7ce2fc
Revises: 835b862f9bf0
Create Date: 2024-01-09 13:43:51.381175
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'd8950c7ce2fc'
down_revision = '835b862f9bf0'
branch_labels = None
depends_on = None
def upgrade():
op.drop_table("schema_versions")
def downgrade():
op.create_table(
"schema_versions",
sa.Column("game", sa.String(4), primary_key=True, nullable=False),
sa.Column("version", sa.Integer, nullable=False, server_default="1"),
mysql_charset="utf8mb4",
)

299
core/data/database.py
View File

@ -1,13 +1,13 @@
import logging, coloredlogs
from typing import Optional, Dict, List
from typing import Optional
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy import create_engine
from logging.handlers import TimedRotatingFileHandler
import importlib, os
import os
import secrets, string
import bcrypt
from hashlib import sha256
import alembic.config
from core.config import CoreConfig
from core.data.schema import *
@ -15,7 +15,6 @@ from core.utils import Utils
class Data:
current_schema_version = 6
engine = None
session = None
user = None
@ -77,281 +76,85 @@ class Data:
)
self.logger.handler_set = True # type: ignore
def __alembic_cmd(self, command: str, *args: str) -> None:
old_dir = os.path.abspath(os.path.curdir)
base_dir = os.path.join(os.path.abspath(os.path.curdir), 'core', 'data', 'alembic')
alembicArgs = [
"-c",
os.path.join(base_dir, "alembic.ini"),
"-x",
f"script_location={base_dir}",
"-x",
f"sqlalchemy.url={self.__url}",
command,
]
alembicArgs.extend(args)
os.chdir(base_dir)
alembic.config.main(argv=alembicArgs)
os.chdir(old_dir)
def create_database(self):
self.logger.info("Creating databases...")
try:
metadata.create_all(self.__engine.connect())
except SQLAlchemyError as e:
self.logger.error(f"Failed to create databases! {e}")
return
games = Utils.get_all_titles()
for game_dir, game_mod in games.items():
try:
if hasattr(game_mod, "database") and hasattr(
game_mod, "current_schema_version"
):
game_mod.database(self.config)
metadata.create_all(self.__engine.connect())
self.base.touch_schema_ver(
game_mod.current_schema_version, game_mod.game_codes[0]
metadata.create_all(
self.engine,
checkfirst=True,
)
except Exception as e:
self.logger.warning(
f"Could not load database schema from {game_dir} - {e}"
)
self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
self.base.set_schema_ver(self.current_schema_version)
self.logger.info(
f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
)
self.user.reset_autoincrement(
self.config.database.user_table_autoincrement_start
)
def recreate_database(self):
self.logger.info("Dropping all databases...")
self.base.execute("SET FOREIGN_KEY_CHECKS=0")
try:
metadata.drop_all(self.__engine.connect())
except SQLAlchemyError as e:
self.logger.error(f"Failed to drop databases! {e}")
return
for root, dirs, files in os.walk("./titles"):
for dir in dirs:
if not dir.startswith("__"):
try:
mod = importlib.import_module(f"titles.{dir}")
try:
for _, mod in Utils.get_all_titles().items():
if hasattr(mod, "database"):
mod.database(self.config)
metadata.drop_all(self.__engine.connect())
except Exception as e:
self.logger.warning(
f"Could not load database schema from {dir} - {e}"
metadata.create_all(
self.engine,
checkfirst=True,
)
except ImportError as e:
self.logger.warning(
f"Failed to load database schema dir {dir} - {e}"
)
break
self.base.execute("SET FOREIGN_KEY_CHECKS=1")
self.create_database()
def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
old_ver = self.base.get_schema_ver(game)
sql = ""
if version is None:
if not game == "CORE":
titles = Utils.get_all_titles()
for folder, mod in titles.items():
if not mod.game_codes[0] == game:
continue
if hasattr(mod, "current_schema_version"):
version = mod.current_schema_version
else:
self.logger.warning(
f"current_schema_version not found for {folder}"
# Stamp the end revision as if alembic had created it, so it can take off after this.
self.__alembic_cmd(
"stamp",
"head",
)
else:
version = self.current_schema_version
if version is None:
self.logger.warning(
f"Could not determine latest version for {game}, please specify --version"
def schema_upgrade(self, ver: str = None):
self.__alembic_cmd(
"upgrade",
"head",
)
if old_ver is None:
self.logger.error(
f"Schema for game {game} does not exist, did you run the creation script?"
)
return
if old_ver == version:
self.logger.info(
f"Schema for game {game} is already version {old_ver}, nothing to do"
)
return
if action == "upgrade":
for x in range(old_ver, version):
if not os.path.exists(
f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql"
):
self.logger.error(
f"Could not find {action} script {game.upper()}_{x + 1}_{action}.sql in core/data/schema/versions folder"
)
return
with open(
f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql",
"r",
encoding="utf-8",
) as f:
sql = f.read()
result = self.base.execute(sql)
if result is None:
self.logger.error("Error execuing sql script!")
return None
else:
for x in range(old_ver, version, -1):
if not os.path.exists(
f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql"
):
self.logger.error(
f"Could not find {action} script {game.upper()}_{x - 1}_{action}.sql in core/data/schema/versions folder"
)
return
with open(
f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql",
"r",
encoding="utf-8",
) as f:
sql = f.read()
result = self.base.execute(sql)
if result is None:
self.logger.error("Error execuing sql script!")
return None
result = self.base.set_schema_ver(version, game)
if result is None:
self.logger.error("Error setting version in schema_version table!")
return None
self.logger.info(f"Successfully migrated {game} to schema version {version}")
def create_owner(self, email: Optional[str] = None) -> None:
async def create_owner(self, email: Optional[str] = None, code: Optional[str] = "00000000000000000000") -> None:
pw = "".join(
secrets.choice(string.ascii_letters + string.digits) for i in range(20)
)
hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt())
user_id = self.user.create_user(email=email, permission=255, password=hash)
user_id = await self.user.create_user("sysowner", email, hash.decode(), 255)
if user_id is None:
self.logger.error(f"Failed to create owner with email {email}")
return
card_id = self.card.create_card(user_id, "00000000000000000000")
card_id = await self.card.create_card(user_id, code)
if card_id is None:
self.logger.error(f"Failed to create card for owner with id {user_id}")
return
self.logger.warning(
f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!"
f"Successfully created owner with email {email}, access code {code}, and password {pw} Make sure to change this password and assign a real card ASAP!"
)
def migrate_card(self, old_ac: str, new_ac: str, should_force: bool) -> None:
if old_ac == new_ac:
self.logger.error("Both access codes are the same!")
async def migrate(self) -> None:
exist = await self.base.execute("SELECT * FROM alembic_version")
if exist is not None:
self.logger.warn("No need to migrate as you have already migrated to alembic. If you are trying to upgrade the schema, use `upgrade` instead!")
return
new_card = self.card.get_card_by_access_code(new_ac)
if new_card is None:
self.card.update_access_code(old_ac, new_ac)
return
if not should_force:
self.logger.warning(
f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
self.logger.info("Stamp with initial revision")
self.__alembic_cmd(
"stamp",
"835b862f9bf0",
)
return
self.logger.info(
f"All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
self.logger.info("Upgrade")
self.__alembic_cmd(
"upgrade",
"head",
)
self.card.delete_card(new_card["id"])
self.card.update_access_code(old_ac, new_ac)
hanging_user = self.user.get_user(new_card["user"])
if hanging_user["password"] is None:
self.logger.info(f"Delete hanging user {hanging_user['id']}")
self.user.delete_user(hanging_user["id"])
def delete_hanging_users(self) -> None:
"""
Finds and deletes users that have not registered for the webui that have no cards assocated with them.
"""
unreg_users = self.user.get_unregistered_users()
if unreg_users is None:
self.logger.error("Error occoured finding unregistered users")
for user in unreg_users:
cards = self.card.get_user_cards(user["id"])
if cards is None:
self.logger.error(f"Error getting cards for user {user['id']}")
continue
if not cards:
self.logger.info(f"Delete hanging user {user['id']}")
self.user.delete_user(user["id"])
def autoupgrade(self) -> None:
all_game_versions = self.base.get_all_schema_vers()
if all_game_versions is None:
self.logger.warning("Failed to get schema versions")
return
all_games = Utils.get_all_titles()
all_games_list: Dict[str, int] = {}
for _, mod in all_games.items():
if hasattr(mod, "current_schema_version"):
all_games_list[mod.game_codes[0]] = mod.current_schema_version
for x in all_game_versions:
failed = False
game = x["game"].upper()
update_ver = int(x["version"])
latest_ver = all_games_list.get(game, 1)
if game == "CORE":
latest_ver = self.current_schema_version
if update_ver == latest_ver:
self.logger.info(f"{game} is already latest version")
continue
for y in range(update_ver + 1, latest_ver + 1):
if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
with open(
f"core/data/schema/versions/{game}_{y}_upgrade.sql",
"r",
encoding="utf-8",
) as f:
sql = f.read()
result = self.base.execute(sql)
if result is None:
self.logger.error(
f"Error execuing sql script for game {game} v{y}!"
)
failed = True
break
else:
self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
failed = True
if not failed:
self.base.set_schema_ver(latest_ver, game)
def show_versions(self) -> None:
all_game_versions = self.base.get_all_schema_vers()
for ver in all_game_versions:
self.logger.info(f"{ver['game']} -> v{ver['version']}")

53
core/data/schema/base.py
View File

@ -43,11 +43,11 @@ class BaseData:
self.conn = conn
self.logger = logging.getLogger("database")
def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
async def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
res = None
try:
self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}")
self.logger.debug(f"SQL Execute: {''.join(str(sql).splitlines())}")
res = self.conn.execute(text(sql), opts)
except SQLAlchemyError as e:
@ -82,52 +82,7 @@ class BaseData:
"""
return randrange(10000, 9999999)
def get_all_schema_vers(self) -> Optional[List[Row]]:
sql = select(schema_ver)
result = self.execute(sql)
if result is None:
return None
return result.fetchall()
def get_schema_ver(self, game: str) -> Optional[int]:
sql = select(schema_ver).where(schema_ver.c.game == game)
result = self.execute(sql)
if result is None:
return None
row = result.fetchone()
if row is None:
return None
return row["version"]
def touch_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
sql = insert(schema_ver).values(game=game, version=ver)
conflict = sql.on_duplicate_key_update(version=schema_ver.c.version)
result = self.execute(conflict)
if result is None:
self.logger.error(
f"Failed to update schema version for game {game} (v{ver})"
)
return None
return result.lastrowid
def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
sql = insert(schema_ver).values(game=game, version=ver)
conflict = sql.on_duplicate_key_update(version=ver)
result = self.execute(conflict)
if result is None:
self.logger.error(
f"Failed to update schema version for game {game} (v{ver})"
)
return None
return result.lastrowid
def log_event(
async def log_event(
self, system: str, type: str, severity: int, message: str, details: Dict = {}
) -> Optional[int]:
sql = event_log.insert().values(
@ -147,7 +102,7 @@ class BaseData:
return result.lastrowid
def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
async def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
sql = event_log.select().limit(entries).all()
result = self.execute(sql)

8
dbutils.py
View File

@ -5,7 +5,8 @@ from os import mkdir, path, access, W_OK
import yaml
import asyncio
from core import Data, CoreConfig
from core.data import Data
from core.config import CoreConfig
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Database utilities")
@ -49,6 +50,11 @@ if __name__ == "__main__":
elif args.action == "create-owner":
loop = asyncio.get_event_loop()
loop.run_until_complete(data.create_owner(args.email, args.access_code))
data.schema_upgrade(args.version)
elif args.action == "migrate":
loop = asyncio.get_event_loop()
loop.run_until_complete(data.migrate())
else:
logging.getLogger("database").info(f"Unknown action {args.action}")

3
example_config/core.yaml
View File

@ -19,6 +19,7 @@ title:
reboot_start_time: "04:00"
reboot_end_time: "05:00"
ssl_key: "cert/title.key"
database:
host: "localhost"
username: "aime"
@ -27,7 +28,7 @@ database:
port: 3306
protocol: "mysql"
sha2_password: False
loglevel: "warn"
loglevel: "info"
enable_memcached: True
memcached_host: "localhost"

1
titles/chuni/__init__.py
View File

@ -7,4 +7,3 @@ index = ChuniServlet
database = ChuniData
reader = ChuniReader
game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW, ChuniConstants.GAME_CODE_INT]
current_schema_version = 5

3
titles/cm/__init__.py
View File

@ -6,7 +6,4 @@ from titles.cm.database import CardMakerData
index = CardMakerServlet
reader = CardMakerReader
database = CardMakerData
game_codes = [CardMakerConstants.GAME_CODE]
current_schema_version = 1

1
titles/cxb/__init__.py
View File

@ -7,4 +7,3 @@ index = CxbServlet
database = CxbData
reader = CxbReader
game_codes = [CxbConstants.GAME_CODE]
current_schema_version = 1

1
titles/diva/__init__.py
View File

@ -7,4 +7,3 @@ index = DivaServlet
database = DivaData
reader = DivaReader
game_codes = [DivaConstants.GAME_CODE]
current_schema_version = 6

1
titles/idac/__init__.py
View File

@ -9,4 +9,3 @@ database = IDACData
reader = IDACReader
frontend = IDACFrontend
game_codes = [IDACConstants.GAME_CODE]
current_schema_version = 1

1
titles/idz/__init__.py
View File

@ -5,4 +5,3 @@ from titles.idz.database import IDZData
index = IDZServlet
database = IDZData
game_codes = [IDZConstants.GAME_CODE]
current_schema_version = 1

1
titles/mai2/__init__.py
View File

@ -16,4 +16,3 @@ game_codes = [
Mai2Constants.GAME_CODE_GREEN,
Mai2Constants.GAME_CODE,
]
current_schema_version = 8

1
titles/ongeki/__init__.py
View File

@ -9,4 +9,3 @@ database = OngekiData
reader = OngekiReader
frontend = OngekiFrontend
game_codes = [OngekiConstants.GAME_CODE]
current_schema_version = 6

1
titles/pokken/__init__.py
View File

@ -6,5 +6,4 @@ from .frontend import PokkenFrontend
index = PokkenServlet
database = PokkenData
game_codes = [PokkenConstants.GAME_CODE]
current_schema_version = 1
frontend = PokkenFrontend

1
titles/sao/__init__.py
View File

@ -7,4 +7,3 @@ index = SaoServlet
database = SaoData
reader = SaoReader
game_codes = [SaoConstants.GAME_CODE]
current_schema_version = 1

1
titles/wacca/__init__.py
View File

@ -9,4 +9,3 @@ database = WaccaData
reader = WaccaReader
frontend = WaccaFrontend
game_codes = [WaccaConstants.GAME_CODE]
current_schema_version = 5