move to alembic

This commit is contained in:
Hay1tsme 2024-01-09 13:57:59 -05:00
parent 07cbbcc377
commit edd3ce8ead
19 changed files with 152 additions and 368 deletions

View File

@ -1,25 +1,14 @@
# A generic, single database configuration. # A generic, single database configuration.
[alembic] [alembic]
# path to migration scripts script_location=.
script_location = .
# template used to generate migration files # template used to generate migration files
# file_template = %%(rev)s_%%(slug)s # file_template = %%(rev)s_%%(slug)s
# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory.
prepend_sys_path = .
# timezone to use when rendering the date
# within the migration file as well as the filename.
# string value is passed to dateutil.tz.gettz()
# leave blank for localtime
# timezone =
# max length of characters to apply to the # max length of characters to apply to the
# "slug" field # "slug" field
# truncate_slug_length = 40 #truncate_slug_length = 40
# set to 'true' to run the environment during # set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate # the 'revision' command, regardless of autogenerate
@ -31,28 +20,14 @@ prepend_sys_path = .
# sourceless = false # sourceless = false
# version location specification; this defaults # version location specification; this defaults
# to ./versions. When using multiple version # to migrations//versions. When using multiple version
# directories, initial revisions must be specified with --version-path # directories, initial revisions must be specified with --version-path
# version_locations = %(here)s/bar %(here)s/bat ./versions # version_locations = %(here)s/bar %(here)s/bat migrations//versions
# the output encoding used when revision files # the output encoding used when revision files
# are written from script.py.mako # are written from script.py.mako
# output_encoding = utf-8 # output_encoding = utf-8
sqlalchemy.url = driver://user:pass@localhost/dbname
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks=black
# black.type=console_scripts
# black.entrypoint=black
# black.options=-l 79
# Logging configuration # Logging configuration
[loggers] [loggers]
keys = root,sqlalchemy,alembic keys = root,sqlalchemy,alembic

View File

@ -1,9 +1,9 @@
from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig from logging.config import fileConfig
from sqlalchemy import engine_from_config from core.data.schema.base import metadata
from sqlalchemy import pool
from alembic import context
# this is the Alembic Config object, which provides # this is the Alembic Config object, which provides
# access to the values within the .ini file in use. # access to the values within the .ini file in use.
@ -17,7 +17,7 @@ fileConfig(config.config_file_name)
# for 'autogenerate' support # for 'autogenerate' support
# from myapp import mymodel # from myapp import mymodel
# target_metadata = mymodel.Base.metadata # target_metadata = mymodel.Base.metadata
target_metadata = None target_metadata = metadata
# other values from the config, defined by the needs of env.py, # other values from the config, defined by the needs of env.py,
# can be acquired: # can be acquired:
@ -37,13 +37,11 @@ def run_migrations_offline():
script output. script output.
""" """
raise Exception('Not implemented or configured!')
url = config.get_main_option("sqlalchemy.url") url = config.get_main_option("sqlalchemy.url")
context.configure( context.configure(
url=url, url=url, target_metadata=target_metadata, literal_binds=True)
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction(): with context.begin_transaction():
context.run_migrations() context.run_migrations()
@ -56,21 +54,27 @@ def run_migrations_online():
and associate a connection with the context. and associate a connection with the context.
""" """
ini_section = config.get_section(config.config_ini_section)
overrides = context.get_x_argument(as_dictionary=True)
for override in overrides:
ini_section[override] = overrides[override]
connectable = engine_from_config( connectable = engine_from_config(
config.get_section(config.config_ini_section), ini_section,
prefix="sqlalchemy.", prefix='sqlalchemy.',
poolclass=pool.NullPool, poolclass=pool.NullPool)
)
with connectable.connect() as connection: with connectable.connect() as connection:
context.configure( context.configure(
connection=connection, target_metadata=target_metadata connection=connection,
target_metadata=target_metadata,
compare_type=True,
compare_server_default=True,
) )
with context.begin_transaction(): with context.begin_transaction():
context.run_migrations() context.run_migrations()
if context.is_offline_mode(): if context.is_offline_mode():
run_migrations_offline() run_migrations_offline()
else: else:

View File

@ -0,0 +1,24 @@
"""Initial Migration
Revision ID: 835b862f9bf0
Revises:
Create Date: 2024-01-09 13:06:10.787432
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '835b862f9bf0'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
pass
def downgrade():
pass

View File

@ -0,0 +1,29 @@
"""Remove old db mgmt system
Revision ID: d8950c7ce2fc
Revises: 835b862f9bf0
Create Date: 2024-01-09 13:43:51.381175
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'd8950c7ce2fc'
down_revision = '835b862f9bf0'
branch_labels = None
depends_on = None
def upgrade():
op.drop_table("schema_versions")
def downgrade():
op.create_table(
"schema_versions",
sa.Column("game", sa.String(4), primary_key=True, nullable=False),
sa.Column("version", sa.Integer, nullable=False, server_default="1"),
mysql_charset="utf8mb4",
)

View File

@ -1,13 +1,13 @@
import logging, coloredlogs import logging, coloredlogs
from typing import Optional, Dict, List from typing import Optional
from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy import create_engine from sqlalchemy import create_engine
from logging.handlers import TimedRotatingFileHandler from logging.handlers import TimedRotatingFileHandler
import importlib, os import os
import secrets, string import secrets, string
import bcrypt import bcrypt
from hashlib import sha256 from hashlib import sha256
import alembic.config
from core.config import CoreConfig from core.config import CoreConfig
from core.data.schema import * from core.data.schema import *
@ -15,7 +15,6 @@ from core.utils import Utils
class Data: class Data:
current_schema_version = 6
engine = None engine = None
session = None session = None
user = None user = None
@ -77,281 +76,85 @@ class Data:
) )
self.logger.handler_set = True # type: ignore self.logger.handler_set = True # type: ignore
def __alembic_cmd(self, command: str, *args: str) -> None:
old_dir = os.path.abspath(os.path.curdir)
base_dir = os.path.join(os.path.abspath(os.path.curdir), 'core', 'data', 'alembic')
alembicArgs = [
"-c",
os.path.join(base_dir, "alembic.ini"),
"-x",
f"script_location={base_dir}",
"-x",
f"sqlalchemy.url={self.__url}",
command,
]
alembicArgs.extend(args)
os.chdir(base_dir)
alembic.config.main(argv=alembicArgs)
os.chdir(old_dir)
def create_database(self): def create_database(self):
self.logger.info("Creating databases...") self.logger.info("Creating databases...")
try: metadata.create_all(
metadata.create_all(self.__engine.connect()) self.engine,
except SQLAlchemyError as e: checkfirst=True,
self.logger.error(f"Failed to create databases! {e}")
return
games = Utils.get_all_titles()
for game_dir, game_mod in games.items():
try:
if hasattr(game_mod, "database") and hasattr(
game_mod, "current_schema_version"
):
game_mod.database(self.config)
metadata.create_all(self.__engine.connect())
self.base.touch_schema_ver(
game_mod.current_schema_version, game_mod.game_codes[0]
) )
except Exception as e: for _, mod in Utils.get_all_titles().items():
self.logger.warning(
f"Could not load database schema from {game_dir} - {e}"
)
self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
self.base.set_schema_ver(self.current_schema_version)
self.logger.info(
f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
)
self.user.reset_autoincrement(
self.config.database.user_table_autoincrement_start
)
def recreate_database(self):
self.logger.info("Dropping all databases...")
self.base.execute("SET FOREIGN_KEY_CHECKS=0")
try:
metadata.drop_all(self.__engine.connect())
except SQLAlchemyError as e:
self.logger.error(f"Failed to drop databases! {e}")
return
for root, dirs, files in os.walk("./titles"):
for dir in dirs:
if not dir.startswith("__"):
try:
mod = importlib.import_module(f"titles.{dir}")
try:
if hasattr(mod, "database"): if hasattr(mod, "database"):
mod.database(self.config) mod.database(self.config)
metadata.drop_all(self.__engine.connect()) metadata.create_all(
self.engine,
except Exception as e: checkfirst=True,
self.logger.warning(
f"Could not load database schema from {dir} - {e}"
) )
except ImportError as e: # Stamp the end revision as if alembic had created it, so it can take off after this.
self.logger.warning( self.__alembic_cmd(
f"Failed to load database schema dir {dir} - {e}" "stamp",
) "head",
break
self.base.execute("SET FOREIGN_KEY_CHECKS=1")
self.create_database()
def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
old_ver = self.base.get_schema_ver(game)
sql = ""
if version is None:
if not game == "CORE":
titles = Utils.get_all_titles()
for folder, mod in titles.items():
if not mod.game_codes[0] == game:
continue
if hasattr(mod, "current_schema_version"):
version = mod.current_schema_version
else:
self.logger.warning(
f"current_schema_version not found for {folder}"
) )
else: def schema_upgrade(self, ver: str = None):
version = self.current_schema_version self.__alembic_cmd(
"upgrade",
if version is None: "head",
self.logger.warning(
f"Could not determine latest version for {game}, please specify --version"
) )
if old_ver is None: async def create_owner(self, email: Optional[str] = None, code: Optional[str] = "00000000000000000000") -> None:
self.logger.error(
f"Schema for game {game} does not exist, did you run the creation script?"
)
return
if old_ver == version:
self.logger.info(
f"Schema for game {game} is already version {old_ver}, nothing to do"
)
return
if action == "upgrade":
for x in range(old_ver, version):
if not os.path.exists(
f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql"
):
self.logger.error(
f"Could not find {action} script {game.upper()}_{x + 1}_{action}.sql in core/data/schema/versions folder"
)
return
with open(
f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql",
"r",
encoding="utf-8",
) as f:
sql = f.read()
result = self.base.execute(sql)
if result is None:
self.logger.error("Error execuing sql script!")
return None
else:
for x in range(old_ver, version, -1):
if not os.path.exists(
f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql"
):
self.logger.error(
f"Could not find {action} script {game.upper()}_{x - 1}_{action}.sql in core/data/schema/versions folder"
)
return
with open(
f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql",
"r",
encoding="utf-8",
) as f:
sql = f.read()
result = self.base.execute(sql)
if result is None:
self.logger.error("Error execuing sql script!")
return None
result = self.base.set_schema_ver(version, game)
if result is None:
self.logger.error("Error setting version in schema_version table!")
return None
self.logger.info(f"Successfully migrated {game} to schema version {version}")
def create_owner(self, email: Optional[str] = None) -> None:
pw = "".join( pw = "".join(
secrets.choice(string.ascii_letters + string.digits) for i in range(20) secrets.choice(string.ascii_letters + string.digits) for i in range(20)
) )
hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt()) hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt())
user_id = self.user.create_user(email=email, permission=255, password=hash) user_id = await self.user.create_user("sysowner", email, hash.decode(), 255)
if user_id is None: if user_id is None:
self.logger.error(f"Failed to create owner with email {email}") self.logger.error(f"Failed to create owner with email {email}")
return return
card_id = self.card.create_card(user_id, "00000000000000000000") card_id = await self.card.create_card(user_id, code)
if card_id is None: if card_id is None:
self.logger.error(f"Failed to create card for owner with id {user_id}") self.logger.error(f"Failed to create card for owner with id {user_id}")
return return
self.logger.warning( self.logger.warning(
f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!" f"Successfully created owner with email {email}, access code {code}, and password {pw} Make sure to change this password and assign a real card ASAP!"
) )
def migrate_card(self, old_ac: str, new_ac: str, should_force: bool) -> None: async def migrate(self) -> None:
if old_ac == new_ac: exist = await self.base.execute("SELECT * FROM alembic_version")
self.logger.error("Both access codes are the same!") if exist is not None:
self.logger.warn("No need to migrate as you have already migrated to alembic. If you are trying to upgrade the schema, use `upgrade` instead!")
return return
new_card = self.card.get_card_by_access_code(new_ac) self.logger.info("Stamp with initial revision")
if new_card is None: self.__alembic_cmd(
self.card.update_access_code(old_ac, new_ac) "stamp",
return "835b862f9bf0",
if not should_force:
self.logger.warning(
f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
) )
return
self.logger.info( self.logger.info("Upgrade")
f"All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}." self.__alembic_cmd(
"upgrade",
"head",
) )
self.card.delete_card(new_card["id"])
self.card.update_access_code(old_ac, new_ac)
hanging_user = self.user.get_user(new_card["user"])
if hanging_user["password"] is None:
self.logger.info(f"Delete hanging user {hanging_user['id']}")
self.user.delete_user(hanging_user["id"])
def delete_hanging_users(self) -> None:
"""
Finds and deletes users that have not registered for the webui that have no cards assocated with them.
"""
unreg_users = self.user.get_unregistered_users()
if unreg_users is None:
self.logger.error("Error occoured finding unregistered users")
for user in unreg_users:
cards = self.card.get_user_cards(user["id"])
if cards is None:
self.logger.error(f"Error getting cards for user {user['id']}")
continue
if not cards:
self.logger.info(f"Delete hanging user {user['id']}")
self.user.delete_user(user["id"])
def autoupgrade(self) -> None:
all_game_versions = self.base.get_all_schema_vers()
if all_game_versions is None:
self.logger.warning("Failed to get schema versions")
return
all_games = Utils.get_all_titles()
all_games_list: Dict[str, int] = {}
for _, mod in all_games.items():
if hasattr(mod, "current_schema_version"):
all_games_list[mod.game_codes[0]] = mod.current_schema_version
for x in all_game_versions:
failed = False
game = x["game"].upper()
update_ver = int(x["version"])
latest_ver = all_games_list.get(game, 1)
if game == "CORE":
latest_ver = self.current_schema_version
if update_ver == latest_ver:
self.logger.info(f"{game} is already latest version")
continue
for y in range(update_ver + 1, latest_ver + 1):
if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
with open(
f"core/data/schema/versions/{game}_{y}_upgrade.sql",
"r",
encoding="utf-8",
) as f:
sql = f.read()
result = self.base.execute(sql)
if result is None:
self.logger.error(
f"Error execuing sql script for game {game} v{y}!"
)
failed = True
break
else:
self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
failed = True
if not failed:
self.base.set_schema_ver(latest_ver, game)
def show_versions(self) -> None:
all_game_versions = self.base.get_all_schema_vers()
for ver in all_game_versions:
self.logger.info(f"{ver['game']} -> v{ver['version']}")

View File

@ -43,11 +43,11 @@ class BaseData:
self.conn = conn self.conn = conn
self.logger = logging.getLogger("database") self.logger = logging.getLogger("database")
def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]: async def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
res = None res = None
try: try:
self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}") self.logger.debug(f"SQL Execute: {''.join(str(sql).splitlines())}")
res = self.conn.execute(text(sql), opts) res = self.conn.execute(text(sql), opts)
except SQLAlchemyError as e: except SQLAlchemyError as e:
@ -82,52 +82,7 @@ class BaseData:
""" """
return randrange(10000, 9999999) return randrange(10000, 9999999)
def get_all_schema_vers(self) -> Optional[List[Row]]: async def log_event(
sql = select(schema_ver)
result = self.execute(sql)
if result is None:
return None
return result.fetchall()
def get_schema_ver(self, game: str) -> Optional[int]:
sql = select(schema_ver).where(schema_ver.c.game == game)
result = self.execute(sql)
if result is None:
return None
row = result.fetchone()
if row is None:
return None
return row["version"]
def touch_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
sql = insert(schema_ver).values(game=game, version=ver)
conflict = sql.on_duplicate_key_update(version=schema_ver.c.version)
result = self.execute(conflict)
if result is None:
self.logger.error(
f"Failed to update schema version for game {game} (v{ver})"
)
return None
return result.lastrowid
def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
sql = insert(schema_ver).values(game=game, version=ver)
conflict = sql.on_duplicate_key_update(version=ver)
result = self.execute(conflict)
if result is None:
self.logger.error(
f"Failed to update schema version for game {game} (v{ver})"
)
return None
return result.lastrowid
def log_event(
self, system: str, type: str, severity: int, message: str, details: Dict = {} self, system: str, type: str, severity: int, message: str, details: Dict = {}
) -> Optional[int]: ) -> Optional[int]:
sql = event_log.insert().values( sql = event_log.insert().values(
@ -147,7 +102,7 @@ class BaseData:
return result.lastrowid return result.lastrowid
def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]: async def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
sql = event_log.select().limit(entries).all() sql = event_log.select().limit(entries).all()
result = self.execute(sql) result = self.execute(sql)

View File

@ -5,7 +5,8 @@ from os import mkdir, path, access, W_OK
import yaml import yaml
import asyncio import asyncio
from core import Data, CoreConfig from core.data import Data
from core.config import CoreConfig
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Database utilities") parser = argparse.ArgumentParser(description="Database utilities")
@ -49,6 +50,11 @@ if __name__ == "__main__":
elif args.action == "create-owner": elif args.action == "create-owner":
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.run_until_complete(data.create_owner(args.email, args.access_code)) loop.run_until_complete(data.create_owner(args.email, args.access_code))
data.schema_upgrade(args.version)
elif args.action == "migrate":
loop = asyncio.get_event_loop()
loop.run_until_complete(data.migrate())
else: else:
logging.getLogger("database").info(f"Unknown action {args.action}") logging.getLogger("database").info(f"Unknown action {args.action}")

View File

@ -19,6 +19,7 @@ title:
reboot_start_time: "04:00" reboot_start_time: "04:00"
reboot_end_time: "05:00" reboot_end_time: "05:00"
ssl_key: "cert/title.key" ssl_key: "cert/title.key"
database: database:
host: "localhost" host: "localhost"
username: "aime" username: "aime"
@ -27,7 +28,7 @@ database:
port: 3306 port: 3306
protocol: "mysql" protocol: "mysql"
sha2_password: False sha2_password: False
loglevel: "warn" loglevel: "info"
enable_memcached: True enable_memcached: True
memcached_host: "localhost" memcached_host: "localhost"

View File

@ -7,4 +7,3 @@ index = ChuniServlet
database = ChuniData database = ChuniData
reader = ChuniReader reader = ChuniReader
game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW, ChuniConstants.GAME_CODE_INT] game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW, ChuniConstants.GAME_CODE_INT]
current_schema_version = 5

View File

@ -6,7 +6,4 @@ from titles.cm.database import CardMakerData
index = CardMakerServlet index = CardMakerServlet
reader = CardMakerReader reader = CardMakerReader
database = CardMakerData database = CardMakerData
game_codes = [CardMakerConstants.GAME_CODE] game_codes = [CardMakerConstants.GAME_CODE]
current_schema_version = 1

View File

@ -7,4 +7,3 @@ index = CxbServlet
database = CxbData database = CxbData
reader = CxbReader reader = CxbReader
game_codes = [CxbConstants.GAME_CODE] game_codes = [CxbConstants.GAME_CODE]
current_schema_version = 1

View File

@ -7,4 +7,3 @@ index = DivaServlet
database = DivaData database = DivaData
reader = DivaReader reader = DivaReader
game_codes = [DivaConstants.GAME_CODE] game_codes = [DivaConstants.GAME_CODE]
current_schema_version = 6

View File

@ -9,4 +9,3 @@ database = IDACData
reader = IDACReader reader = IDACReader
frontend = IDACFrontend frontend = IDACFrontend
game_codes = [IDACConstants.GAME_CODE] game_codes = [IDACConstants.GAME_CODE]
current_schema_version = 1

View File

@ -5,4 +5,3 @@ from titles.idz.database import IDZData
index = IDZServlet index = IDZServlet
database = IDZData database = IDZData
game_codes = [IDZConstants.GAME_CODE] game_codes = [IDZConstants.GAME_CODE]
current_schema_version = 1

View File

@ -16,4 +16,3 @@ game_codes = [
Mai2Constants.GAME_CODE_GREEN, Mai2Constants.GAME_CODE_GREEN,
Mai2Constants.GAME_CODE, Mai2Constants.GAME_CODE,
] ]
current_schema_version = 8

View File

@ -9,4 +9,3 @@ database = OngekiData
reader = OngekiReader reader = OngekiReader
frontend = OngekiFrontend frontend = OngekiFrontend
game_codes = [OngekiConstants.GAME_CODE] game_codes = [OngekiConstants.GAME_CODE]
current_schema_version = 6

View File

@ -6,5 +6,4 @@ from .frontend import PokkenFrontend
index = PokkenServlet index = PokkenServlet
database = PokkenData database = PokkenData
game_codes = [PokkenConstants.GAME_CODE] game_codes = [PokkenConstants.GAME_CODE]
current_schema_version = 1
frontend = PokkenFrontend frontend = PokkenFrontend

View File

@ -7,4 +7,3 @@ index = SaoServlet
database = SaoData database = SaoData
reader = SaoReader reader = SaoReader
game_codes = [SaoConstants.GAME_CODE] game_codes = [SaoConstants.GAME_CODE]
current_schema_version = 1

View File

@ -9,4 +9,3 @@ database = WaccaData
reader = WaccaReader reader = WaccaReader
frontend = WaccaFrontend frontend = WaccaFrontend
game_codes = [WaccaConstants.GAME_CODE] game_codes = [WaccaConstants.GAME_CODE]
current_schema_version = 5