Error063/artemis
1
0
Fork 0
forked from Hay1tsme/artemis
Code Pull Requests Activity

use SQL's limit/offset pagination for nextIndex/maxCount requests (#185)

Browse Source 
Instead of retrieving the entire list of items/characters/scores/etc. at once (and even store them in memory), use SQL's `LIMIT ... OFFSET ...` pagination so we only take what we need.

Currently only CHUNITHM uses this, but this will also affect maimai DX and O.N.G.E.K.I. once the PR is ready.

Also snuck in a fix for CHUNITHM/maimai DX's `GetUserRivalMusicApi` to respect the `userRivalMusicLevelList` sent by the client.

### How this works

Say we have a `GetUserCharacterApi` request:

```json
{
    "userId": 10000,
    "maxCount": 700,
    "nextIndex": 0
}
```

Instead of getting the entire character list from the database (which can be very large if the user force unlocked everything), add limit/offset to the query:

```python
select(character)
.where(character.c.user == user_id)
.order_by(character.c.id.asc())
.limit(max_count + 1)
.offset(next_index)
```

The query takes `maxCount + 1` items from the database to determine if there is more items than can be returned:

```python
rows = ...

if len(rows) > max_count:
    # return only max_count rows
    next_index += max_count
else:
    # return everything left
    next_index = -1
```

This has the benefit of not needing to load everything into memory (and also having to store server state, as seen in the [`SCORE_BUFFER` list](2274b42358/titles/chuni/base.py (L13)).)

Reviewed-on: Hay1tsme/artemis#185
Co-authored-by: beerpsi <beerpsi@duck.com>
Co-committed-by: beerpsi <beerpsi@duck.com>
This commit is contained in:
beerpsi
2024-11-16 19:10:29 +00:00
committed by Hay1tsme
parent cb009f6e23
commit 58a5177a30
18 changed files with 1410 additions and 713 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
core/config.py
View File

@ -1,5 +1,9 @@
import logging, os
from typing import Any
import logging
import os
import ssl
from typing import Any, Union
from typing_extensions import Optional
class ServerConfig:
def __init__(self, parent_config: "CoreConfig") -> None:
@ -175,12 +179,60 @@ class DatabaseConfig:
return CoreConfig.get_config_field(
self.__config, "core", "database", "protocol", default="mysql"
)
@property
def ssl_enabled(self) -> str:
def ssl_enabled(self) -> bool:
return CoreConfig.get_config_field(
self.__config, "core", "database", "ssl_enabled", default=False
)
@property
def ssl_cafile(self) -> Optional[str]:
return CoreConfig.get_config_field(
self.__config, "core", "database", "ssl_cafile", default=None
)
@property
def ssl_capath(self) -> Optional[str]:
return CoreConfig.get_config_field(
self.__config, "core", "database", "ssl_capath", default=None
)
@property
def ssl_cert(self) -> Optional[str]:
return CoreConfig.get_config_field(
self.__config, "core", "database", "ssl_cert", default=None
)
@property
def ssl_key(self) -> Optional[str]:
return CoreConfig.get_config_field(
self.__config, "core", "database", "ssl_key", default=None
)
@property
def ssl_key_password(self) -> Optional[str]:
return CoreConfig.get_config_field(
self.__config, "core", "database", "ssl_key_password", default=None
)
@property
def ssl_verify_identity(self) -> bool:
return CoreConfig.get_config_field(
self.__config, "core", "database", "ssl_verify_identity", default=True
)
@property
def ssl_verify_cert(self) -> Optional[Union[str, bool]]:
return CoreConfig.get_config_field(
self.__config, "core", "database", "ssl_verify_cert", default=None
)
@property
def ssl_ciphers(self) -> Optional[str]:
return CoreConfig.get_config_field(
self.__config, "core", "database", "ssl_ciphers", default=None
)
@property
def sha2_password(self) -> bool:
@ -208,6 +260,53 @@ class DatabaseConfig:
self.__config, "core", "database", "memcached_host", default="localhost"
)
def create_ssl_context_if_enabled(self):
if not self.ssl_enabled:
return
no_ca = (
self.ssl_cafile is None
and self.ssl_capath is None
)
ctx = ssl.create_default_context(
cafile=self.ssl_cafile,
capath=self.ssl_capath,
)
ctx.check_hostname = not no_ca and self.ssl_verify_identity
if self.ssl_verify_cert is None:
ctx.verify_mode = ssl.CERT_NONE if no_ca else ssl.CERT_REQUIRED
elif isinstance(self.ssl_verify_cert, bool):
ctx.verify_mode = (
ssl.CERT_REQUIRED
if self.ssl_verify_cert
else ssl.CERT_NONE
)
elif isinstance(self.ssl_verify_cert, str):
value = self.ssl_verify_cert.lower()
if value in ("none", "0", "false", "no"):
ctx.verify_mode = ssl.CERT_NONE
elif value == "optional":
ctx.verify_mode = ssl.CERT_OPTIONAL
elif value in ("required", "1", "true", "yes"):
ctx.verify_mode = ssl.CERT_REQUIRED
else:
ctx.verify_mode = ssl.CERT_NONE if no_ca else ssl.CERT_REQUIRED
if self.ssl_cert:
ctx.load_cert_chain(
self.ssl_cert,
self.ssl_key,
self.ssl_key_password,
)
if self.ssl_ciphers:
ctx.set_ciphers(self.ssl_ciphers)
return ctx
class FrontendConfig:
def __init__(self, parent_config: "CoreConfig") -> None:
self.__config = parent_config

16
core/data/alembic/env.py
View File

@ -1,14 +1,18 @@
from __future__ import with_statement
import asyncio
import os
from pathlib import Path
import threading
from logging.config import fileConfig
import yaml
from alembic import context
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from core.config import CoreConfig
from core.data.schema.base import metadata
# this is the Alembic Config object, which provides
@ -74,8 +78,18 @@ async def run_async_migrations() -> None:
for override in overrides:
ini_section[override] = overrides[override]
core_config = CoreConfig()
with (Path("../../..") / os.environ["ARTEMIS_CFG_DIR"] / "core.yaml").open(encoding="utf-8") as f:
core_config.update(yaml.safe_load(f))
connectable = async_engine_from_config(
ini_section, prefix="sqlalchemy.", poolclass=pool.NullPool
ini_section,
poolclass=pool.NullPool,
connect_args={
"charset": "utf8mb4",
"ssl": core_config.database.create_ssl_context_if_enabled(),
}
)
async with connectable.connect() as connection:

17
core/data/database.py
View File

@ -1,11 +1,12 @@
import logging
import os
import secrets
import ssl
import string
import warnings
from hashlib import sha256
from logging.handlers import TimedRotatingFileHandler
from typing import ClassVar, Optional
from typing import Any, ClassVar, Optional
import alembic.config
import bcrypt
@ -35,12 +36,20 @@ class Data:
if self.config.database.sha2_password:
passwd = sha256(self.config.database.password.encode()).digest()
self.__url = f"{self.config.database.protocol}+aiomysql://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4&ssl={str(self.config.database.ssl_enabled).lower()}"
self.__url = f"{self.config.database.protocol}+aiomysql://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}"
else:
self.__url = f"{self.config.database.protocol}+aiomysql://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4&ssl={str(self.config.database.ssl_enabled).lower()}"
self.__url = f"{self.config.database.protocol}+aiomysql://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}"
if Data.engine is MISSING:
Data.engine = create_async_engine(self.__url, pool_recycle=3600, isolation_level="AUTOCOMMIT")
Data.engine = create_async_engine(
self.__url,
pool_recycle=3600,
isolation_level="AUTOCOMMIT",
connect_args={
"charset": "utf8mb4",
"ssl": self.config.database.create_ssl_context_if_enabled(),
},
)
self.__engine = Data.engine
if Data.session is MISSING: