Begin the process of transitioning from megaime

.gitignore

@@ -0,0 +1,162 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log*
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+.vscode/*
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+cert/*
+!cert/billing.key
+!cert/server.key
+!cert/server.pem
+config/*
+deliver/*
+
+dbdump-*.json

cert/billing.key

@@ -0,0 +1,16 @@
+-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAML2GPUuzv2N4bYC
+xtc5bZSzolHFWdCUbP+whjr3K98FOLnYeoi7mtUSUUYOW8wIqy6WM3c4c0Bp7FcQ
+LnZ0zWMm1TfLGHZzZmk5n7Iv6HDPr3ehDgbWLnOpRrVqZDxpAGD2vQb4p2DW4I2x
+GUqnqDa++C8dH/0lXqE6cqwGXNGtAgMBAAECgYEAizgPhG4Dk55QkpeTBDfXH3vT
+Ko9B3qdO2ptkjxDX/C8PXe7POXq2SvcEoIE6Xg3Gp8LMR5NBAbth8J32f9JSov3P
+SiGCGno4k2i2s3jRuVg76FGLDsZH/N1dt4h78VnW0VlInwaM6bQv3zp0u8rXVk/P
+wpYh9AGmquBJS3VYUcECQQD0PDRe28SrhollygGZSO321rYbYhoTIstDXZWyQ/y/
+PWKNwNHcYTHIVGmTrJx2AJUyr1tJhwjiOwlsI5Y1Q4/9AkEAzFpFPcs1r/xgSFxB
+eYrcNseWYbVajtVxG9t57sayaEQbH2UMNA2vqSYK/nU6oJhj5eLRIsPHlA5ZbIiZ
+rvc/cQJAKS0RQ0DX+ncXKQMSm+4wuGHgl+NFNB60mCnp+AEAVpmZyP5OI1J7myOo
+HQ6H3lkgzkfEIzRR6ho773BcfaRjXQJAfS4nEE11G9ML4AezjBLGB0CIHF6NlMWn
+PhtaPCy3iSt/OeIacaCYpJNLVMjXGx1+xIoG9rbbgRSxLs0W55lJ4QJBALOUVcNw
+GKEJdxhIkA8iuUlEyGpKluAgHUNOOKvC3ogRoB0OyH+If/9o8wWDfxgexgM0zGBc
+u178W9XDW+IijDA=
+-----END PRIVATE KEY-----

cert/server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDg1xvj8JLt4nDq
+rRQOGzCys0bD2VdGRnQ/TYC3ezg0whYbQRD7bUdkNQm1KeNOlj8yr8kjz30swIY6
+6Ufw82qGH0QjsmbSOTLr13B9K1ogd11aMxErvVnxtNBOkuwtzKL+Zb1D99jvxX0u
+e0dIsBcuojemyoz6iNbU7WSNtbIFeQFr+MlZujHIv0Hj76MYyZAqNq2F3Ou1SduV
+kw+Acol8KzTfiII38ibZcP+INgK6YvnrJrxbDZ+NXwsXOeg0y/vZ7hc2h1k5wT3E
+xWoyMD/bvnVrKTneQfwOI9s4s3pB6o1hGeUGRhPvpJ20+FGOE8LIRzBEixDKl82T
+G63DVoMJAgMBAAECggEBANKVBf0+BA8jZ4iUpFT16G1mdZ/W/uPF9viXGThAAwt+
+wH+0ODiUSCo1dqsj2U5wcC6D74pHukBg7RdeCFBHW3zU6dfZLm40vlmfRS8mnFoO
+EfP6IlnqFcTJCdSdzPC0WfCUz2hKSPeA61bOhZwxuPSnYCIqUVIROczhrqz/AQYX
+ZuuY7ut0h6X11xoddeeTOdfd+rktxPVpHeJDcz9F6Gk/0pg1ezs66TZ5sWt0/ZhP
+ZWTB40KPec2lomRGwY36AMOT3uFucnbtJgxhTGdqsRv7Kl0xCTo7xW/85Za7rhXK
++xRdHrYr9w6xYTKHi9Ap4HEmUcx/fGAtddxr0fzdpIECgYEA8nRyYYHMdSr4WI/E
+fD/cpIjKLxw9BCdoXhruVSKp1GmhyQTH7Y0zJQ0uvG7GlsEXJ2V9yqTofzWev8iQ
+vVzX+14yTKaNUAwc7HCEXMN+xqnq61Bjldx6U54CfdInOhTFuoq8CXu80cHobYMs
+8SjnYuhp5NqABGR+9YTYaHzjBhkCgYEA7Wa9orxkWruXUdROUZrdhlU44ilxvA66
+r1vVVLQFNVle2hbie3b93ZQCZrZL8iFsGKOIhXSncQ6pl9GQqDLJvMUiffzUci6G
+A4GgGlzXpkj+RmAkIkGYafmQqTZhMHNws35LMYoIabn0l8cWwG0XL1pO2YSjaoYg
+5Kj0TjoNonECgYAcAk3Qa+FFy+ACwyEMxYfkzhSlWprF5xOMg4ny9d0ut8FD6rR6
+Aezdo+c5R4bTlZzqJTRh+6kMQRKEz1PBPH+K/3fKGReMHsocmmcAHGmB49FKu++1
+OVI8ZK2fAW8cq5eoFCzi35ORm9gRBq1jcrlAWN8a3A8b8swj6uPhNkQ3yQKBgCH+
+HBk5MIVtZvVomO5GZoHdog+AL7DlywVg+OLwA+7npRVFQZi8KQ2ZK97ZK3a4ImpE
+wD+bvH4Lw2zhrPzoiMpmz9GKakEPOFE4NlyP/rDossAQ9BuTmOdTvMr95lyxqumI
+o+usABhjcAprj25uMGuvWqr6uwt9uSgEqTaqSVmBAoGAMs6/59L8EABC/CkBLp0t
+y3sYzjyjZr6gnMCkIi95b2FWyIVjlK53dwjKYg67BUI8qLNbUW1NPhyotc/9GklP
+qiNTkUeKcHOlZqOzP0zPquFk2lSfJuuGrMgElZPcEFLsAWRqnjRGp5iP05nk5OQh
+foY7svzPedFk/t3eFcOyrPo=
+-----END PRIVATE KEY-----

cert/server.pem

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICtDCCAZwCAQAwDQYJKoZIhvcNAQELBQAwJTEQMA4GA1UEAwwHRHVtbXlDQTER
+MA8GA1UECgwIRHVtbXlQS0kwIBcNMTgxMDE1MTkzMDUxWhgPMjExODEwMTUxOTMw
+NTFaMBkxFzAVBgNVBAMMDmliLm5hb21pbmV0LmpwMIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEA4Ncb4/CS7eJw6q0UDhswsrNGw9lXRkZ0P02At3s4NMIW
+G0EQ+21HZDUJtSnjTpY/Mq/JI899LMCGOulH8PNqhh9EI7Jm0jky69dwfStaIHdd
+WjMRK71Z8bTQTpLsLcyi/mW9Q/fY78V9LntHSLAXLqI3psqM+ojW1O1kjbWyBXkB
+a/jJWboxyL9B4++jGMmQKjathdzrtUnblZMPgHKJfCs034iCN/Im2XD/iDYCumL5
+6ya8Ww2fjV8LFznoNMv72e4XNodZOcE9xMVqMjA/2751ayk53kH8DiPbOLN6QeqN
+YRnlBkYT76SdtPhRjhPCyEcwRIsQypfNkxutw1aDCQIDAQABMA0GCSqGSIb3DQEB
+CwUAA4IBAQANOqFqjnGf80vvwYEkUsfOWp8rNVat+8rdXl0BShGBXiDItzMOU79K
+YkPObniQ5RLBMhrvqlCsSk0Np+ZgvV12J4Wtmf/znLa5ZKyeI4N1FCefU9cl4xpB
+08Fv8YWbYV7SMNr54ZkURdho4FVR1pAnpuittpAEjMT4R4ubbOH8UEbMTbVgxXdn
+086IAlfsYn0gnOlf76RkJFLe4UlWaZB75SaaXnNavBPN9iFnqXLckg6tsFUJnNMC
+esq5aHQ9sXWs4oKpJi8SXxt/zNRmgTnQK2ONM38NeZpLmlWPkyNxzsRNlYWo+kWP
+C98jVFe1+3K88ISk0DSN4XOQQnrIvn68
+-----END CERTIFICATE-----

core/__init__.py

@@ -0,0 +1,4 @@
+from core.config import CoreConfig
+from core.allnet import AllnetServlet, BillingServlet
+from core.aimedb import AimedbFactory
+from core.title import TitleServlet

core/aimedb.py

@@ -0,0 +1,234 @@
+from twisted.internet.protocol import Factory, Protocol
+import logging, coloredlogs
+from Crypto.Cipher import AES
+import struct
+from typing import Dict, Any
+from logging.handlers import TimedRotatingFileHandler
+
+from core.config import CoreConfig
+from core.data import Data
+
+class AimedbProtocol(Protocol):
+    AIMEDB_RESPONSE_CODES = {
+        "felica_lookup": 0x03,
+        "lookup": 0x06,
+        "log": 0x0a,
+        "campaign": 0x0c,
+        "touch": 0x0e,
+        "lookup2": 0x10,
+        "felica_lookup2": 0x12,
+        "log2": 0x14,
+        "hello": 0x65
+    }
+
+    request_list: Dict[int, Any] = {}
+
+    def __init__(self, core_cfg: CoreConfig) -> None:
+        self.logger = logging.getLogger("aimedb")
+        self.config = core_cfg
+        self.data = Data(core_cfg)
+        if core_cfg.aimedb.key == "":
+            self.logger.error("!!!KEY NOT SET!!!")
+            exit(1)
+        
+        self.request_list[0x01] = self.handle_felica_lookup
+        self.request_list[0x04] = self.handle_lookup
+        self.request_list[0x05] = self.handle_register
+        self.request_list[0x09] = self.handle_log        
+        self.request_list[0x0b] = self.handle_campaign
+        self.request_list[0x0d] = self.handle_touch
+        self.request_list[0x0f] = self.handle_lookup2
+        self.request_list[0x11] = self.handle_felica_lookup2
+        self.request_list[0x13] = self.handle_log2
+        self.request_list[0x64] = self.handle_hello
+
+    def append_padding(self, data: bytes):
+        """Appends 0s to the end of the data until it's at the correct size"""
+        length = struct.unpack_from("<H", data, 6)
+        padding_size = length[0] - len(data)
+        data += bytes(padding_size)
+        return data
+
+    def connectionMade(self) -> None:
+        self.logger.debug(f"{self.transport.getPeer().host} Connected")
+
+    def connectionLost(self, reason) -> None:
+        self.logger.debug(f"{self.transport.getPeer().host} Disconnected - {reason.value}")
+    
+    def dataReceived(self, data: bytes) -> None:
+        cipher = AES.new(self.config.aimedb.key.encode(), AES.MODE_ECB)
+
+        try:
+            decrypted = cipher.decrypt(data)
+        except:
+            self.logger.error(f"Failed to decrypt {data.hex()}")
+            return None
+
+        self.logger.debug(f"{self.transport.getPeer().host} wrote {decrypted.hex()}")
+
+        if not decrypted[1] == 0xa1 and not decrypted[0] == 0x3e:
+            self.logger.error(f"Bad magic")
+            return None
+
+        req_code = decrypted[4]
+
+        if req_code == 0x66:
+            self.logger.info(f"goodbye from {self.transport.getPeer().host}")
+            self.transport.loseConnection()
+            return
+
+        try:
+            resp = self.request_list[req_code](decrypted)
+            encrypted = cipher.encrypt(resp)
+            self.logger.debug(f"Response {resp.hex()}")
+            self.transport.write(encrypted)
+
+        except KeyError:
+            self.logger.error(f"Unknown command code {hex(req_code)}")
+            return None
+
+        except ValueError as e:
+            self.logger.error(f"Failed to encrypt {resp.hex()} because {e}")
+            return None
+    
+    def handle_campaign(self, data: bytes) -> bytes:
+        self.logger.info(f"campaign from {self.transport.getPeer().host}")
+        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["campaign"], 0x0200, 0x0001)
+        return self.append_padding(ret)
+    
+    def handle_hello(self, data: bytes) -> bytes:
+        self.logger.info(f"hello from {self.transport.getPeer().host}")
+        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["hello"], 0x0020, 0x0001)
+        return self.append_padding(ret)
+
+    def handle_lookup(self, data: bytes) -> bytes:
+        luid = data[0x20: 0x2a].hex()
+        user_id = self.data.card.get_user_id_from_card(access_code=luid)
+
+        if user_id is None: user_id = -1
+
+        self.logger.info(f"lookup from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}")
+
+        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["lookup"], 0x0130, 0x0001)
+        ret += bytes(0x20 - len(ret))
+
+        if user_id is None: ret += struct.pack("<iH", -1, 0)
+        else: ret += struct.pack("<l", user_id)
+        return self.append_padding(ret)
+
+    def handle_lookup2(self, data: bytes) -> bytes:
+        self.logger.info(f"lookup2")
+
+        ret = bytearray(self.handle_lookup(data))
+        ret[4] = self.AIMEDB_RESPONSE_CODES["lookup2"]
+
+        return bytes(ret)
+
+    def handle_felica_lookup(self, data: bytes) -> bytes:
+        idm = data[0x20: 0x28].hex()
+        pmm = data[0x28: 0x30].hex()
+        access_code = self.data.card.to_access_code(idm)
+        self.logger.info(f"felica_lookup from {self.transport.getPeer().host}: idm {idm} pmm {pmm} -> access_code {access_code}")
+
+        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["felica_lookup"], 0x0030, 0x0001)
+        ret += bytes(26)
+        ret += bytes.fromhex(access_code)
+
+        return self.append_padding(ret)
+
+    def handle_felica_lookup2(self, data: bytes) -> bytes:
+        idm = data[0x30: 0x38].hex()
+        pmm = data[0x38: 0x40].hex()
+        access_code = self.data.card.to_access_code(idm)
+        user_id = self.data.card.get_user_id_from_card(access_code=access_code)
+
+        if user_id is None: user_id = -1
+
+        self.logger.info(f"felica_lookup2 from {self.transport.getPeer().host}: idm {idm} ipm {pmm} -> access_code {access_code} user_id {user_id}")
+
+        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["felica_lookup2"], 0x0140, 0x0001)
+        ret += bytes(22)
+        ret += struct.pack("<lq", user_id, -1) # first -1 is ext_id, 3rd is access code
+        ret += bytes.fromhex(access_code)
+        ret += struct.pack("<l", 1)
+        
+        return self.append_padding(ret)
+    
+    def handle_touch(self, data: bytes) -> bytes:
+        self.logger.info(f"touch from {self.transport.getPeer().host}")
+        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["touch"], 0x0050, 0x0001)
+        ret += bytes(5)
+        ret += struct.pack("<3H", 0x6f, 0, 1)
+
+        return self.append_padding(ret)
+
+    def handle_register(self, data: bytes) -> bytes:        
+        luid = data[0x20: 0x2a].hex()
+        if self.config.server.allow_registration:
+            user_id = self.data.user.create_user()
+
+            if user_id is None: 
+                user_id = -1
+                self.logger.error("Failed to register user!")
+
+            else:
+                card_id = self.data.card.create_card(user_id, luid)
+
+                if card_id is None: 
+                    user_id = -1
+                    self.logger.error("Failed to register card!")
+
+            self.logger.info(f"register from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}")
+        
+        else:
+            self.logger.info(f"register from {self.transport.getPeer().host} blocked!: luid {luid}")
+            user_id = -1
+
+        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["lookup"], 0x0030, 0x0001 if user_id > -1 else 0)
+        ret += bytes(0x20 - len(ret))
+        ret += struct.pack("<l", user_id)
+
+        return self.append_padding(ret)
+
+    def handle_log(self, data: bytes) -> bytes:
+        # TODO: Save aimedb logs
+        self.logger.info(f"log from {self.transport.getPeer().host}")
+        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["log"], 0x0020, 0x0001)
+        return self.append_padding(ret)
+
+    def handle_log2(self, data: bytes) -> bytes:
+        self.logger.info(f"log2 from {self.transport.getPeer().host}")
+        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["log2"], 0x0040, 0x0001)
+        ret += bytes(22)
+        ret += struct.pack("H", 1)
+
+        return self.append_padding(ret)
+
+class AimedbFactory(Factory):
+    protocol = AimedbProtocol
+    def __init__(self, cfg: CoreConfig) -> None:
+        self.config = cfg
+        log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
+        log_fmt = logging.Formatter(log_fmt_str)
+        self.logger = logging.getLogger("aimedb")
+
+        fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.logs, "aimedb"), when="d", backupCount=10)
+        fileHandler.setFormatter(log_fmt)
+        
+        consoleHandler = logging.StreamHandler()
+        consoleHandler.setFormatter(log_fmt)
+
+        self.logger.addHandler(fileHandler)
+        self.logger.addHandler(consoleHandler)
+        
+        self.logger.setLevel(self.config.aimedb.loglevel)
+        coloredlogs.install(level=cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str)
+        
+        if self.config.aimedb.key == "":
+            self.logger.error("Please set 'key' field in your config file.")
+            exit(1)
+
+        self.logger.info(f"Ready on port {self.config.aimedb.port}")
+    
+    def buildProtocol(self, addr):
+        return AimedbProtocol(self.config)

core/allnet.py

@@ -0,0 +1,39 @@
+from twisted.web import resource
+import logging, coloredlogs
+from logging.handlers import TimedRotatingFileHandler
+
+from core.config import CoreConfig
+from core.data import Data
+
+class AllnetServlet(resource.Resource):
+    isLeaf = True
+    def __init__(self, core_cfg: CoreConfig, cfg_folder: str):        
+        super().__init__()
+        self.config = core_cfg
+        self.config_folder = cfg_folder
+        self.data = Data(core_cfg)
+
+        self.logger = logging.getLogger("allnet")
+        log_fmt_str = "[%(asctime)s] Allnet | %(levelname)s | %(message)s"
+        log_fmt = logging.Formatter(log_fmt_str)        
+
+        fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.log_dir, "allnet"), when="d", backupCount=10)
+        fileHandler.setFormatter(log_fmt)
+        
+        consoleHandler = logging.StreamHandler()
+        consoleHandler.setFormatter(log_fmt)
+
+        self.logger.addHandler(fileHandler)
+        self.logger.addHandler(consoleHandler)
+        
+        self.logger.setLevel(core_cfg.allnet.loglevel)
+        coloredlogs.install(level=core_cfg.allnet.loglevel, logger=self.logger, fmt=log_fmt_str)
+
+class BillingServlet(resource.Resource):
+    isLeaf = True
+    def __init__(self, core_cfg: CoreConfig, cfg_folder: str):        
+        super().__init__()
+        self.config = core_cfg
+        self.config_folder = cfg_folder
+        self.data = Data(core_cfg)
+        self.logger = logging.getLogger('allnet')

core/config.py

@@ -0,0 +1,198 @@
+import logging, os
+from typing import Any
+
+class ServerConfig:
+    def __init__(self, parent_config: "CoreConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def listen_address(self) -> str:
+        return CoreConfig.get_config_field(self.__config, '127.0.0.1', 'core', 'server', 'listen_address')
+
+    @property
+    def allow_user_registration(self) -> bool:
+        return CoreConfig.get_config_field(self.__config, True, 'core', 'server', 'allow_user_registration')
+
+    @property
+    def allow_unregistered_games(self) -> bool:
+        return CoreConfig.get_config_field(self.__config, True, 'core', 'server', 'allow_unregistered_games')
+
+    @property
+    def name(self) -> str:
+        return CoreConfig.get_config_field(self.__config, "ARTEMiS", 'core', 'server', 'name')
+
+    @property
+    def is_develop(self) -> bool:
+        return CoreConfig.get_config_field(self.__config, True, 'core', 'server', 'is_develop')
+
+    @property
+    def log_dir(self) -> str:
+        return CoreConfig.get_config_field(self.__config, 'logs', 'core', 'server', 'log_dir')
+
+class TitleConfig:
+    def __init__(self, parent_config: "CoreConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, "info", 'core', 'title', 'loglevel'))
+
+    @property
+    def hostname(self) -> str:
+        return CoreConfig.get_config_field(self.__config, "localhost", 'core', 'title', 'hostname')
+
+    @property
+    def port(self) -> int:
+        return CoreConfig.get_config_field(self.__config, 8080, 'core', 'title', 'port')
+
+class DatabaseConfig:
+    def __init__(self, parent_config: "CoreConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def host(self) -> str:
+        return CoreConfig.get_config_field(self.__config, "localhost", 'core', 'database', 'host')
+
+    @property
+    def username(self) -> str:
+        return CoreConfig.get_config_field(self.__config, 'aime', 'core', 'database', 'username')
+
+    @property
+    def password(self) -> str:
+        return CoreConfig.get_config_field(self.__config, 'aime', 'core', 'database', 'password')
+
+    @property
+    def name(self) -> str:
+        return CoreConfig.get_config_field(self.__config, 'aime', 'core', 'database', 'name')
+
+    @property
+    def port(self) -> int:
+        return CoreConfig.get_config_field(self.__config, 3306, 'core', 'database', 'port')
+
+    @property
+    def protocol(self) -> str:
+        return CoreConfig.get_config_field(self.__config, "mysql", 'core', 'database', 'type')
+    
+    @property
+    def sha2_password(self) -> bool:
+        return CoreConfig.get_config_field(self.__config, False, 'core', 'database', 'sha2_password')
+    
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, "info", 'core', 'database', 'loglevel'))
+
+    @property
+    def user_table_autoincrement_start(self) -> int:
+        return CoreConfig.get_config_field(self.__config, 10000, 'core', 'database', 'user_table_autoincrement_start')
+    
+    @property
+    def memcached_host(self) -> str:
+        return CoreConfig.get_config_field(self.__config, "localhost", 'core', 'database', 'memcached_host')
+
+class FrontendConfig:
+    def __init__(self, parent_config: "CoreConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def enable(self) -> int:
+        return CoreConfig.get_config_field(self.__config, False, 'core', 'frontend', 'enable')
+
+    @property
+    def port(self) -> int:
+        return CoreConfig.get_config_field(self.__config, 8090, 'core', 'frontend', 'port')
+    
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'frontend', 'loglevel', "info"))
+
+class AllnetConfig:
+    def __init__(self, parent_config: "CoreConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, "info", 'core', 'allnet', 'loglevel'))
+
+    @property
+    def port(self) -> int:
+        return CoreConfig.get_config_field(self.__config, 80, 'core', 'allnet', 'port')
+    
+    @property
+    def allow_online_updates(self) -> int:
+        return CoreConfig.get_config_field(self.__config, False, 'core', 'allnet', 'allow_online_updates')
+
+class BillingConfig:
+    def __init__(self, parent_config: "CoreConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def port(self) -> int:
+        return CoreConfig.get_config_field(self.__config, 8443, 'core', 'billing', 'port')
+
+    @property
+    def ssl_key(self) -> str:
+        return CoreConfig.get_config_field(self.__config, "cert/server.key", 'core', 'billing', 'ssl_key')
+
+    @property
+    def ssl_cert(self) -> str:
+        return CoreConfig.get_config_field(self.__config, "cert/server.pem", 'core', 'billing', 'ssl_cert')
+    
+    @property
+    def signing_key(self) -> str:
+        return CoreConfig.get_config_field(self.__config, "cert/billing.key", 'core', 'billing', 'signing_key')
+
+class AimedbConfig:
+    def __init__(self, parent_config: "CoreConfig") -> None:
+        self.__config = parent_config
+    
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, "info", 'core', 'aimedb', 'loglevel'))
+
+    @property
+    def port(self) -> int:
+        return CoreConfig.get_config_field(self.__config, 22345, 'core', 'aimedb', 'port')
+
+    @property
+    def key(self) -> str:
+        return CoreConfig.get_config_field(self.__config, "", 'core', 'aimedb', 'key')
+
+class CoreConfig(dict):
+    def __init__(self) -> None:
+        self.server = ServerConfig(self)
+        self.title = TitleConfig(self)
+        self.database = DatabaseConfig(self)
+        self.frontend = FrontendConfig(self)
+        self.allnet = AllnetConfig(self)
+        self.billing = BillingConfig(self)
+        self.aimedb = AimedbConfig(self)
+
+    @classmethod
+    def str_to_loglevel(cls, level_str: str):
+        if level_str.lower() == "error":
+            return logging.ERROR
+        elif level_str.lower().startswith("warn"): # Fits warn or warning
+            return logging.WARN
+        elif level_str.lower() == "debug":
+            return logging.DEBUG
+        else:
+             return logging.INFO
+
+    @classmethod
+    def get_config_field(cls, __config: dict, default: Any, *path: str) -> Any:
+        envKey = 'CFG_'
+        for arg in path:
+            envKey += arg + '_'
+        
+        if envKey.endswith('_'):
+            envKey = envKey[:-1]
+
+        if envKey in os.environ:
+            return os.environ.get(envKey)
+
+        read = __config
+
+        for x in range(len(path) - 1):
+            read = read.get(path[x], {})
+
+        return read.get(path[len(path) - 1], default)

core/const.py

@@ -0,0 +1,36 @@
+from enum import Enum
+
+class MainboardPlatformCodes():
+    RINGEDGE = "AALE"
+    RINGWIDE = "AAML"
+    NU = "AAVE"
+    NUSX = "AAWE"
+    ALLS_UX = "ACAE"
+    ALLS_HX = "ACAX"
+
+class MainboardRevisions():
+    RINGEDGE = 1
+    RINGEDGE2 = 2
+
+    RINGWIDE = 1
+
+    NU1 = 1
+    NU11 = 11
+    NU2 = 12
+
+    NUSX = 1
+    NUSX11 = 11
+
+    ALLS_UX = 1
+    ALLS_HX = 11
+    ALLS_UX2 = 2
+    ALLS_HX2 = 12
+
+class KeychipPlatformsCodes():
+    RING = "A72E"
+    NU = ("A60E", "A60E", "A60E")
+    NUSX = ("A61X", "A69X")
+    ALLS = "A63E"
+    
+class RegionIDs(Enum):
+    pass

core/data/__init__.py

@@ -0,0 +1,2 @@
+from core.data.database import Data
+from core.data.cache import cached

core/data/cache.py

@@ -0,0 +1,65 @@
+
+from typing import Any, Callable
+from functools import wraps
+import hashlib
+import pickle
+import logging
+from core.config import CoreConfig
+
+cfg:CoreConfig = None # type: ignore
+# Make memcache optional
+try:
+    import pylibmc  # type: ignore
+    has_mc = True
+except ModuleNotFoundError:
+    has_mc = False
+
+def cached(lifetime: int=10, extra_key: Any=None) -> Callable:
+    def _cached(func: Callable) -> Callable:
+        if has_mc:
+            hostname = "127.0.0.1"
+            if cfg:
+                hostname = cfg.database.memcached_host
+            memcache = pylibmc.Client([hostname], binary=True)
+            memcache.behaviors = {"tcp_nodelay": True, "ketama": True}
+            
+            @wraps(func)
+            def wrapper(*args: Any, **kwargs: Any) -> Any:
+                if lifetime is not None:
+
+                    # Hash function args
+                    items = kwargs.items()
+                    hashable_args = (args[1:], sorted(list(items)))
+                    args_key = hashlib.md5(pickle.dumps(hashable_args)).hexdigest()
+
+                    # Generate unique cache key
+                    cache_key = f'{func.__module__}-{func.__name__}-{args_key}-{extra_key() if hasattr(extra_key, "__call__") else extra_key}'
+
+                    # Return cached version if allowed and available
+                    try:
+                        result = memcache.get(cache_key)
+                    except pylibmc.Error as e:
+                        logging.getLogger("database").error(f"Memcache failed: {e}")
+                        result = None
+                        
+                    if result is not None:
+                        logging.getLogger("database").debug(f"Cache hit: {result}")
+                        return result
+
+                # Generate output
+                result = func(*args, **kwargs)
+
+                # Cache output if allowed
+                if lifetime is not None and result is not None:
+                    logging.getLogger("database").debug(f"Setting cache: {result}")
+                    memcache.set(cache_key, result, lifetime)
+
+                return result
+        else:
+            @wraps(func)
+            def wrapper(*args: Any, **kwargs: Any) -> Any:
+                return func(*args, **kwargs)
+
+        return wrapper
+
+    return _cached

core/data/database.py

@@ -0,0 +1,53 @@
+import logging, coloredlogs
+from typing import Any, Dict, List
+from sqlalchemy.orm import scoped_session, sessionmaker
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy import create_engine
+from logging.handlers import TimedRotatingFileHandler
+
+from hashlib import sha256
+
+from core.config import CoreConfig
+from core.data.schema import *
+
+class Data:
+    def __init__(self, cfg: CoreConfig) -> None:
+        self.config = cfg
+
+        if self.config.database.sha2_password:
+            passwd = sha256(self.config.database.password.encode()).digest()
+            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
+        else:
+            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
+        
+        self.__engine = create_engine(self.__url, pool_recycle=3600)
+        session = sessionmaker(bind=self.__engine, autoflush=True, autocommit=True)
+        self.session = scoped_session(session)
+
+        self.user = UserData(self.config, self.session)
+        self.arcade = ArcadeData(self.config, self.session)
+        self.card = CardData(self.config, self.session)
+        self.base = BaseData(self.config, self.session)
+        self.schema_ver_latest = 1
+
+        log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
+        log_fmt = logging.Formatter(log_fmt_str)
+        self.logger = logging.getLogger("database")
+
+        # Prevent the logger from adding handlers multiple times
+        if not getattr(self.logger, 'handler_set', None):
+            fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.logs, "db"), encoding="utf-8",
+                when="d", backupCount=10)
+            fileHandler.setFormatter(log_fmt)
+            
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
+
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(self.config.database.loglevel)
+            coloredlogs.install(cfg.database.loglevel, logger=self.logger, fmt=log_fmt_str)
+            self.logger.handler_set = True # type: ignore
+
+    

core/data/schema/__init__.py

@@ -0,0 +1,6 @@
+from core.data.schema.user import UserData
+from core.data.schema.card import CardData
+from core.data.schema.base import BaseData, metadata
+from core.data.schema.arcade import ArcadeData
+
+__all__ = ["UserData", "CardData", "BaseData", "metadata", "ArcadeData"]

core/data/schema/arcade.py

@@ -0,0 +1,113 @@
+from typing import Optional, Dict
+from sqlalchemy import Table, Column
+from sqlalchemy.sql.schema import ForeignKey, PrimaryKeyConstraint
+from sqlalchemy.types import Integer, String, Boolean
+from sqlalchemy.sql import func, select
+from sqlalchemy.dialects.mysql import insert
+
+from core.data.schema.base import BaseData, metadata
+
+arcade = Table(
+    "arcade",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("name", String(255)),
+    Column("nickname", String(255)),    
+    Column("country", String(3)),
+    Column("country_id", Integer),
+    Column("state", String(255)),
+    Column("city", String(255)),
+    Column("region_id", Integer),
+    Column("timezone", String(255)),
+    mysql_charset='utf8mb4'
+)
+
+machine = Table(
+    "machine",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("arcade", ForeignKey("arcade.id", ondelete="cascade", onupdate="cascade"), nullable=False),    
+    Column("serial", String(15), nullable=False),
+    Column("board", String(15)),
+    Column("game", String(4)),
+    Column("country", String(3)), # overwrites if not null
+    Column("timezone", String(255)),
+    Column("ota_enable", Boolean),
+    Column("is_cab", Boolean),
+    mysql_charset='utf8mb4'
+)
+
+arcade_owner = Table(
+    'arcade_owner',
+    metadata,
+    Column('user', Integer, ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column('arcade', Integer, ForeignKey("arcade.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column('permissions', Integer, nullable=False),
+    PrimaryKeyConstraint('user', 'arcade', name='arcade_owner_pk'),
+    mysql_charset='utf8mb4'
+)
+
+class ArcadeData(BaseData):
+    def get_machine(self, serial: str = None, id: int = None) -> Optional[Dict]:
+        if serial is not None:
+            sql = machine.select(machine.c.serial == serial)
+        elif id is not None:
+            sql = machine.select(machine.c.id == id)
+        else: 
+            self.logger.error(f"{__name__ }: Need either serial or ID to look up!")
+            return None
+        
+        result = self.execute(sql)
+        if result is None: return None
+        return result.fetchone()
+    
+    def put_machine(self, arcade_id: int, serial: str = None, board: str = None, game: str = None, is_cab: bool = False) -> Optional[int]:
+        if arcade_id:
+            self.logger.error(f"{__name__ }: Need arcade id!")
+            return None
+
+        if serial is None:
+            pass
+
+        sql = machine.insert().values(arcade = arcade_id, keychip = serial, board = board, game = game, is_cab = is_cab)
+        
+        result = self.execute(sql)
+        if result is None: return None
+        return result.lastrowid
+    
+    def get_arcade(self, id: int) -> Optional[Dict]:
+        sql = arcade.select(arcade.c.id == id)
+        result = self.execute(sql)
+        if result is None: return None
+        return result.fetchone()
+    
+    def put_arcade(self, name: str, nickname: str = None, country: str = "JPN", country_id: int = 1, 
+        state: str = "", city: str = "", regional_id: int = 1) -> Optional[int]:
+        if nickname is None: nickname = name
+
+        sql = arcade.insert().values(name = name, nickname = nickname, country = country, country_id = country_id,
+        state = state, city = city, regional_id = regional_id)
+
+        result = self.execute(sql)
+        if result is None: return None
+        return result.lastrowid
+
+    def get_arcade_owners(self, arcade_id: int) -> Optional[Dict]:
+        sql = select(arcade_owner).where(arcade_owner.c.arcade==arcade_id)
+
+        result = self.execute(sql)
+        if result is None: return None
+        return result.fetchall()
+
+    def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
+        sql = insert(arcade_owner).values(
+            arcade=arcade_id,
+            user=user_id
+        )
+
+        result = self.execute(sql)
+        if result is None: return None
+        return result.lastrowid
+
+    def generate_keychip_serial(self, platform_id: int) -> str:
+        pass

core/data/schema/base.py

@@ -0,0 +1,124 @@
+import json
+import logging
+from random import randrange
+from typing import Any, Optional, Dict, List
+from sqlalchemy.engine.cursor import CursorResult
+from sqlalchemy.engine.base import Connection
+from sqlalchemy.sql import text, func, select
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy import MetaData, Table, Column
+from sqlalchemy.types import Integer, String, TIMESTAMP, JSON
+from sqlalchemy.dialects.mysql import insert
+
+from core.config import CoreConfig
+
+metadata = MetaData()
+
+schema_ver = Table(
+    "schema_versions",
+    metadata,
+    Column("game", String(4), primary_key=True, nullable=False),
+    Column("version", Integer, nullable=False, server_default="1"),
+    mysql_charset='utf8mb4'
+)
+
+event_log = Table(
+    "event_log",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("system", String(255), nullable=False),
+    Column("type", String(255), nullable=False),
+    Column("severity", Integer, nullable=False),
+    Column("details", JSON, nullable=False),
+    Column("when_logged", TIMESTAMP, nullable=False, server_default=func.now()),
+    mysql_charset='utf8mb4'
+)
+
+class BaseData():
+    def __init__(self, cfg: CoreConfig, conn: Connection) -> None:
+        self.config = cfg
+        self.conn = conn
+        self.logger = logging.getLogger("database")
+    
+    def execute(self, sql: str, opts: Dict[str, Any]={}) -> Optional[CursorResult]:
+        res = None
+
+        try:
+            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())} || {opts}")
+            res = self.conn.execute(text(sql), opts)
+
+        except SQLAlchemyError as e:
+            self.logger.error(f"SQLAlchemy error {e}")
+            return None
+        
+        except UnicodeEncodeError as e:
+            self.logger.error(f"UnicodeEncodeError error {e}")
+            return None
+
+        except:
+            try:
+                res = self.conn.execute(sql, opts)
+
+            except SQLAlchemyError as e:
+                self.logger.error(f"SQLAlchemy error {e}")
+                return None
+            
+            except UnicodeEncodeError as e:
+                self.logger.error(f"UnicodeEncodeError error {e}")
+                return None
+
+            except:
+                self.logger.error(f"Unknown error")
+                raise
+
+        return res
+    
+    def generate_id(self) -> int:
+        """
+        Generate a random 5-7 digit id
+        """
+        return randrange(10000, 9999999)
+    
+    def get_schema_ver(self, game: str) -> Optional[int]:
+        sql = select(schema_ver).where(schema_ver.c.game == game)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()["version"]
+    
+    def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
+        sql = insert(schema_ver).values(game = game, version = ver)
+        conflict = sql.on_duplicate_key_update(version = ver)
+        
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(f"Failed to update schema version for game {game} (v{ver})")
+            return None
+        return result.lastrowid
+
+    def log_event(self, system: str, type: str, severity: int, details: Dict) -> Optional[int]:
+        sql = event_log.insert().values(system = system, type = type, severity = severity, details = json.dumps(details))
+        result = self.execute(sql)
+
+        if result is None:
+            self.logger.error(f"{__name__}: Failed to insert event into event log! system = {system}, type = {type}, severity = {severity}, details = {details}")
+            return None
+
+        return result.lastrowid
+    
+    def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
+        sql = event_log.select().limit(entries).all()
+        result = self.execute(sql)
+
+        if result is None: return None
+        return result.fetchall()
+    
+    def fix_bools(self, data: Dict) -> Dict:
+        for k,v in data.items():
+            if type(v) == str and v.lower() == "true":
+                data[k] = True
+            elif type(v) == str and v.lower() == "false":
+                data[k] = False
+        
+        return data

core/data/schema/card.py

@@ -0,0 +1,67 @@
+from typing import Dict, List, Optional
+from sqlalchemy import Table, Column, UniqueConstraint
+from sqlalchemy.types import Integer, String, Boolean, TIMESTAMP
+from sqlalchemy.sql.schema import ForeignKey
+from sqlalchemy.sql import func
+
+from core.data.schema.base import BaseData, metadata
+
+aime_card = Table(
+    'aime_card',
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("access_code", String(20)),
+    Column("created_date", TIMESTAMP, server_default=func.now()),
+    Column("last_login_date", TIMESTAMP, onupdate=func.now()),
+    Column("is_locked", Boolean, server_default="0"),
+    Column("is_banned", Boolean, server_default="0"),
+    UniqueConstraint("user", "access_code", name="aime_card_uk"),
+    mysql_charset='utf8mb4'
+)
+
+class CardData(BaseData):
+    def get_user_id_from_card(self, access_code: str) -> Optional[int]:
+        """
+        Given a 20 digit access code as a string, get the user id associated with that card
+        """
+        sql = aime_card.select(aime_card.c.access_code == access_code)
+
+        result = self.execute(sql)
+        if result is None: return None 
+
+        card = result.fetchone()
+        if card is None: return None
+
+        return int(card["user"])
+
+    def get_user_cards(self, aime_id: int) -> Optional[List[Dict]]:
+        """
+        Returns all cards owned by a user
+        """
+        sql = aime_card.select(aime_card.c.user == aime_id)
+        result = self.execute(sql)
+        if result is None: return None        
+        return result.fetchall()
+
+    
+    def create_card(self, user_id: int, access_code: str) -> Optional[int]:
+        """
+        Given a aime_user id and a 20 digit access code as a string, create a card and return the ID if successful
+        """
+        sql = aime_card.insert().values(user=user_id, access_code=access_code)
+        result = self.execute(sql)
+        if result is None: return None
+        return result.lastrowid
+
+    def to_access_code(self, luid: str) -> str:
+        """
+        Given a felica cards internal 16 hex character luid, convert it to a 0-padded 20 digit access code as a string
+        """
+        return f"{int(luid, base=16):0{20}}"
+
+    def to_idm(self, access_code: str) -> str:
+        """
+        Given a 20 digit access code as a string, return the 16 hex character luid
+        """
+        return f'{int(access_code):0{16}x}'

core/data/schema/user.py

@@ -0,0 +1,57 @@
+from enum import Enum
+from typing import Dict, Optional
+from sqlalchemy import Table, Column
+from sqlalchemy.types import Integer, String, TIMESTAMP
+from sqlalchemy.sql.schema import ForeignKey
+from sqlalchemy.sql import func
+
+from core.data.schema.base import BaseData, metadata
+
+aime_user = Table(
+    "aime_user",
+    metadata,
+    Column("id", Integer, nullable=False, primary_key=True, autoincrement=True),
+    Column("username", String(25), unique=True),
+    Column("email", String(255), unique=True),
+    Column("password", String(255)),
+    Column("permissions", Integer),    
+    Column("created_date", TIMESTAMP, server_default=func.now()),
+    Column("last_login_date", TIMESTAMP, onupdate=func.now()),
+    Column("suspend_expire_time", TIMESTAMP),
+    mysql_charset='utf8mb4'
+)
+
+frontend_session = Table(
+    "frontend_session",
+    metadata,
+    Column("id", Integer, primary_key=True, unique=True),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column('session_cookie', String(32), nullable=False, unique=True),
+    Column("expires", TIMESTAMP, nullable=False),
+    mysql_charset='utf8mb4'
+)
+
+class PermissionBits(Enum):
+    PermUser = 1
+    PermMod = 2
+    PermSysAdmin = 4
+
+class UserData(BaseData):
+    def create_user(self, username: str = None, email: str = None, password: str = None) -> Optional[int]:
+
+        if email is None:
+            permission = None
+        else:
+            permission = 0
+
+        sql = aime_user.insert().values(username=username, email=email, password=password, permissions=permission)
+        
+        result = self.execute(sql)
+        if result is None: return None
+        return result.lastrowid
+    
+    def reset_autoincrement(self, ai_value: int) -> None:
+        # Didn't feel like learning how to do this the right way
+        # if somebody wants a free PR go nuts I guess
+        sql = f"ALTER TABLE aime_user AUTO_INCREMENT={ai_value}"
+        self.execute(sql)

core/data/schema/versions/SDDT_1_rollback.sql

@@ -0,0 +1,14 @@
+SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechBasicHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechAdvancedHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechExpertHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechMasterHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechLunaticHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleBasicHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleAdvancedHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleExpertHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleMasterHighScore int;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleLunaticHighScore int;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDDT_2_upgrade.sql

@@ -0,0 +1,14 @@
+SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechBasicHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechAdvancedHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechExpertHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechMasterHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumTechLunaticHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleBasicHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleAdvancedHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleExpertHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleMasterHighScore bigint;
+ALTER TABLE ongeki_profile_data MODIFY COLUMN sumBattleLunaticHighScore bigint;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDFE_1_rollback.sql

@@ -0,0 +1,3 @@
+UPDATE wacca_score_stageup SET version = 2 WHERE version = 3;
+UPDATE wacca_score_stageup SET version = 3 WHERE version = 4;
+ALTER TABLE wacca_score_stageup CHANGE version season int(11) DEFAULT NULL NULL;

core/data/schema/versions/SDFE_2_rollback.sql

@@ -0,0 +1,16 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+ALTER TABLE wacca_profile ADD season int(11) NOT NULL;
+ALTER TABLE wacca_profile ADD playcount_stageup_season int(11) NULL;
+ALTER TABLE wacca_profile ADD playcount_multi_coop_season int(11) NULL;
+ALTER TABLE wacca_profile ADD playcount_multi_vs_season int(11) NULL;
+ALTER TABLE wacca_profile ADD playcount_single_season int(11) NULL;
+ALTER TABLE wacca_profile ADD xp_season int(11) NULL;
+ALTER TABLE wacca_profile ADD wp_season int(11) NULL;
+ALTER TABLE wacca_profile ADD wp_spent_season int(11) NULL;
+ALTER TABLE wacca_item ADD use_count_season int(11) NULL;
+
+ALTER TABLE wacca_profile DROP COLUMN gate_tutorial_flags;
+
+
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDFE_2_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE wacca_score_stageup CHANGE season version int(11) DEFAULT NULL NULL;
+UPDATE wacca_score_stageup SET version = 4 WHERE version = 3;
+UPDATE wacca_score_stageup SET version = 3 WHERE version = 2;

core/data/schema/versions/SDFE_3_upgrade.sql

@@ -0,0 +1,15 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+ALTER TABLE wacca_profile DROP COLUMN season;
+ALTER TABLE wacca_profile DROP COLUMN playcount_stageup_season;
+ALTER TABLE wacca_profile DROP COLUMN playcount_multi_coop_season;
+ALTER TABLE wacca_profile DROP COLUMN playcount_multi_vs_season;
+ALTER TABLE wacca_profile DROP COLUMN playcount_single_season;
+ALTER TABLE wacca_profile DROP COLUMN xp_season;
+ALTER TABLE wacca_profile DROP COLUMN wp_season;
+ALTER TABLE wacca_profile DROP COLUMN wp_spent_season;
+ALTER TABLE wacca_item DROP COLUMN use_count_season;
+
+ALTER TABLE wacca_profile ADD gate_tutorial_flags JSON NULL;
+
+SET FOREIGN_KEY_CHECKS=1;

core/title.py

@@ -0,0 +1,14 @@
+from twisted.web import resource
+import logging, coloredlogs
+from logging.handlers import TimedRotatingFileHandler
+
+from core.config import CoreConfig
+from core.data import Data
+
+class TitleServlet(resource.Resource):
+    isLeaf = True
+    def __init__(self, core_cfg: CoreConfig, cfg_folder: str):        
+        super().__init__()
+        self.config = core_cfg
+        self.config_folder = cfg_folder
+        self.data = Data(core_cfg)

dbutils.py

@@ -0,0 +1,6 @@
+import argparse
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="ARTEMiS main entry point")
+    parser.add_argument("--config", "-c", type=str, default="config", help="Configuration folder")
+    args = parser.parse_args()

docs/config.md

@@ -0,0 +1,40 @@
+# ARTEMiS Configuration
+## Server
+- `listen_address`: IP Address or hostname that the server will listen for connections on. Set to 127.0.0.1 for local only, or 0.0.0.0 for all interfaces. Default `127.0.0.1`
+- `allow_user_registration`: Allows users to register in-game via the AimeDB `register` function. Disable to be able to control who can use cards on your server. Default `True`
+- `allow_unregistered_games`: Allows games that do not have registered keychips to connect and authenticate. Disable to restrict who can connect to your server. Recomended to disable for production setups. Default `True`
+- `name`: Name for the server, used by some games in their default MOTDs. Default `ARTEMiS`
+- `is_develop`: Flags that the server is a development instance without a proxy standing in front of it. Setting to `False` tells the server not to listen for SSL, because the proxy should be handling all SSL-related things, among other things. Default `True`
+- `log_dir`: Directory to store logs. Server MUST have read and write permissions to this directory or you will have issues. Default `logs`
+## Title
+- `loglevel`: Logging level for the title server. Default `info`
+- `hostname`: Hostname that gets sent to clients to tell them where to connect. Games must be able to connect to your server via the hostname or IP you spcify here. Note that most games will reject `localhost` or `127.0.0.1`. Default `localhost`
+- `port`: Port that the title server will listen for connections on. Set to 0 to use the Allnet handler to reduce the port footprint. Default `8080`
+## Database
+- `host`: Host of the database. Default `localhost`
+- `username`: Username of the account the server should connect to the database with. Default `aime`
+- `password`: Password of the account the server should connect to the database with. Default `aime`
+- `name`: Name of the database the server should expect. Default `aime`
+- `port`: Port the database server is listening on. Default `3306`
+- `protocol`: Protocol used in the connection string, e.i `mysql` would result in `mysql://...`. Default `mysql`
+- `sha2_password`: Weather or not the password in the connection string should be hashed via SHA2. Default `False`
+- `loglevel`: Logging level for the database. Default `warn`
+- `user_table_autoincrement_start`: What the `aime_user` table ID autoincrememnt should start with. Default `10000`
+- `memcached_host`: Host of the memcached server. Default `localhost`
+## Frontend
+- `enable`: Weather or not the frontend should be enabled. Default `False`
+- `port`: Port the frontend should listen for connections on. Default `8090`
+- `loglevel`: Logging level for the frontend server. Default `info`
+## Allnet
+- `loglevel`: Logging level for the allnet server. Default `info`
+- `port`: Port the allnet server should listen for connections on. Games are hardcoded to ask for port `80` so only change if you have a proxy redirecting properly. Default `80`
+- `allow_online_updates`: Allow allnet to distribute online updates via DownloadOrders. This system is currently non-functional, so leave it disabled. Default `False`
+## Billing
+- `port`: Port the billing server should listen for connections on. Games are hardcoded to ask for port `8443` so only change if you have a proxy redirecting properly. Set to 0 to use the allnet handler to reduce the number of ports the server eats up. Default `8443`
+- `ssl_key`: Location of the ssl server key for the billing server. Ignored if `port` is set to `0` or `is_develop` set to `False`. Default `cert/server.key`
+- `ssl_cert`: Location of the ssl server certificate for the billing server. Must match the CA distributed to users or the billing server will not connect. Ignored if `port` is set to `0` or `is_develop` is set to `False`. Default `cert/server.pem`
+- `signing_key`: Location of the RSA Private key used to sign billing requests. Must match the public key distributed to users or the billing server will not connect. Default `cert/billing.key`
+## Aimedb
+- `loglevel`: Logging level for the aimedb server. Default `info`
+- `port`: Port the aimedb server should listen for connections on. Games are hardcoded to ask for port `22345` so only change if you have a proxy redirecting properly. Default `22345`
+- `key`: Key to encrypt/decrypt aimedb requests and responses. MUST be set or the server will not start. If set incorrectly, your server will not properly handle aimedb requests. Default `""`

example_config/core.yaml

@@ -0,0 +1,45 @@
+server:
+  listen_address: "127.0.0.1"
+  allow_user_registration: True
+  allow_unregistered_games: True
+  name: "ARTEMiS"
+  is_develop: True
+  log_dir: False
+
+title:
+  loglevel: "info"
+  hostname: "localhost"
+  port: 8080
+
+database:
+  host: "localhost"
+  username: "aime"
+  password: "aime"
+  name: "aime"
+  port: 3306
+  protocol: "mysql"
+  sha2_password: False
+  loglevel: "warn"
+  user_table_autoincrement_start: 10000
+  memcached_host: "localhost"
+
+frontend:
+  enable: False
+  port: 8090
+  loglevel: "info"
+
+allnet:
+  loglevel: "info"
+  port: 80
+  allow_online_updates: False
+
+billing:
+  port: 8443
+  ssl_key: "cert/server.key"
+  ssl_cert: "cert/server.pem"
+  signing_key: "cert/billing.key"
+
+aimedb:
+  loglevel: "info"
+  port: 22345
+  key: ""

index.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import argparse
+import yaml
+from os import path, mkdir, access, W_OK
+from core import *
+
+from twisted.web import server
+from twisted.internet import reactor, endpoints
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="ARTEMiS main entry point")
+    parser.add_argument("--config", "-c", type=str, default="config", help="Configuration folder")
+    args = parser.parse_args()
+
+    if not path.exists(f"{args.config}/core.yaml"):
+        print(f"The config folder you specified ({args.config}) does not exist or does not contain core.yaml.\nDid you copy the example folder?")
+        exit(1)
+
+    cfg: CoreConfig = CoreConfig()
+    cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
+
+    if not path.exists(cfg.server.log_dir):
+        mkdir(cfg.server.log_dir)
+    
+    if not access(cfg.server.log_dir, W_OK):
+        print(f"Log directory {cfg.server.log_dir} NOT writable, please check permissions")
+        exit(1)
+
+    if cfg.aimedb.key == "":
+        print("!!AIMEDB KEY BLANK, SET KEY IN CORE.YAML!!")
+        exit(1)
+    
+    print(f"ARTEMiS starting in {'develop' if cfg.server.is_develop else 'production'} mode")
+
+    allnet_server_str = f"tcp:{cfg.allnet.port}:interface={cfg.server.listen_address}"    
+    title_server_str = f"tcp:{cfg.billing.port}:interface={cfg.server.listen_address}"
+    adb_server_str = f"tcp:{cfg.aimedb.port}:interface={cfg.server.listen_address}"
+
+    billing_server_str = f"tcp:{cfg.billing.port}:interface={cfg.server.listen_address}"
+    if cfg.server.is_develop:
+        billing_server_str = f"ssl:{cfg.billing.port}:interface={cfg.server.listen_address}"\
+            f":privateKey={cfg.billing.ssl_key}:certKey={cfg.billing.ssl_cert}"
+    
+    endpoints.serverFromString(reactor, allnet_server_str).listen(server.Site(AllnetServlet(cfg, args.config)))
+    endpoints.serverFromString(reactor, adb_server_str).listen(AimedbFactory(cfg))
+
+    if cfg.billing.port > 0:
+        endpoints.serverFromString(reactor, billing_server_str).listen(server.Site(BillingServlet(cfg)))
+    
+    if cfg.title.port > 0:        
+        endpoints.serverFromString(reactor, title_server_str).listen(server.Site(TitleServlet(cfg, args.config)))
+    
+    reactor.run() # type: ignore

readme.md

@@ -0,0 +1,38 @@
+# ARTEMiS
+A network service emulator for games running SEGA'S ALL.NET service, and similar.
+
+# Supported games
+Games listed below have been tested and confirmed working. Only game versions older then the current one in active use in arcades (n-0) or current game versions older then a year (y-1) are supported.
++ Chunithm
+    + All versions up to New!! Plus
+
++ Crossbeats Rev
+    + All versions + omnimix
+
++ Maimai
+    + All versions up to Universe Plus
+
++ Hatsune Miku Arcade
+    + All versions
+
++ Ongeki
+    + All versions up to Bright
+
++ Wacca
+    + Lily R
+    + Reverse
+
+
+## Requirements
+- python 3 (tested working with 3.9 and 3.10, other versions YMMV)
+- pip
+- memcached (for non-windows platforms)
+- mysql/mariadb server
+
+## Quick start guide
+1) Clone this repository
+2) Install requirements (see the platform-specific guides for instructions)
+3) Install python libraries via `pip`
+4) Copy the example configuration files into another folder (by default the server looks for the `config` directory)
+5) Edit the newly copied configuration files to your liking, using [this](docs/config.md) doc as a guide.
+6) Run the server by invoking `index.py` ex. `python3 index.py`

requirements.txt

@@ -0,0 +1,14 @@
+mypy
+wheel
+twisted
+pytz
+pyyaml
+sqlalchemy
+mysqlclient
+pyopenssl
+service_identity
+PyCryptodome
+inflection
+coloredlogs
+pylibmc
+wacky

requirements_win.txt

@@ -0,0 +1,13 @@
+mypy
+wheel
+twisted
+pytz
+pyyaml
+sqlalchemy
+mysqlclient
+pyopenssl
+service_identity
+PyCryptodome
+inflection
+coloredlogs
+wacky