beerpsi/artemis
1
0
Fork 0
forked from Hay1tsme/artemis
Code Pull Requests Activity

wip: use SQL's limit/offset pagination for nextIndex/maxCount requests

Browse Source
This commit is contained in:
beerpsi
2024-11-14 16:29:24 +07:00
parent cb009f6e23
commit 37d07e6035
5 changed files with 311 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

279
titles/chuni/base.py
View File

@ -1,16 +1,16 @@
import logging
import itertools
import json
import logging
from datetime import datetime, timedelta
from time import strftime
from typing import Any, Dict, List
import pytz
from typing import Dict, Any, List
from core.config import CoreConfig
from titles.chuni.config import ChuniConfig
from titles.chuni.const import ChuniConstants, ItemKind
from titles.chuni.database import ChuniData
from titles.chuni.config import ChuniConfig
SCORE_BUFFER = {}
class ChuniBase:
def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
@ -277,35 +277,39 @@ class ChuniBase:
}
async def handle_get_user_character_api_request(self, data: Dict) -> Dict:
characters = await self.data.item.get_characters(data["userId"])
user_id = int(data["userId"])
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
# add one to the limit so we know if there's a next page of items
characters = await self.data.item.get_characters(
user_id, limit=max_ct + 1, offset=next_idx
)
if characters is None:
return {
"userId": data["userId"],
"userId": user_id,
"length": 0,
"nextIndex": -1,
"userCharacterList": [],
}
character_count = len(characters)
character_list = []
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
for x in range(next_idx, len(characters)):
for x in range(min(character_count, max_ct)):
tmp = characters[x]._asdict()
tmp.pop("user")
tmp.pop("id")
character_list.append(tmp)
if len(character_list) >= max_ct:
break
if len(characters) >= next_idx + max_ct:
if character_count > max_ct:
next_idx += max_ct
else:
next_idx = -1
return {
"userId": data["userId"],
"userId": user_id,
"length": len(character_list),
"nextIndex": next_idx,
"userCharacterList": character_list,
@ -335,29 +339,31 @@ class ChuniBase:
}
async def handle_get_user_course_api_request(self, data: Dict) -> Dict:
user_course_list = await self.data.score.get_courses(data["userId"])
if user_course_list is None:
user_id = int(data["userId"])
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
rows = await self.data.score.get_courses(
user_id, limit=max_ct + 1, offset=next_idx
)
if rows is None or len(rows) == 0:
return {
"userId": data["userId"],
"userId": user_id,
"length": 0,
"nextIndex": -1,
"userCourseList": [],
}
course_list = []
next_idx = int(data.get("nextIndex", 0))
max_ct = int(data.get("maxCount", 300))
for x in range(next_idx, len(user_course_list)):
tmp = user_course_list[x]._asdict()
for x in range(min(len(rows), max_ct)):
tmp = rows[x]._asdict()
tmp.pop("user")
tmp.pop("id")
course_list.append(tmp)
if len(user_course_list) >= max_ct:
break
if len(user_course_list) >= next_idx + max_ct:
if len(rows) > max_ct:
next_idx += max_ct
else:
next_idx = -1
@ -425,75 +431,94 @@ class ChuniBase:
}
async def handle_get_user_rival_music_api_request(self, data: Dict) -> Dict:
rival_id = data["rivalId"]
next_index = int(data["nextIndex"])
max_count = int(data["maxCount"])
user_rival_music_list = []
user_id = int(data["userId"])
rival_id = int(data["rivalId"])
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
rival_levels = [int(x["level"]) for x in data["userRivalMusicLevelList"]]
# Fetch all the rival music entries for the user
all_entries = await self.data.score.get_rival_music(rival_id)
music_detail_rows = await self.data.score.get_scores(
rival_id,
levels=rival_levels,
limit=max_ct + 1,
offset=next_idx,
)
# Process the entries based on max_count and nextIndex
for music in all_entries:
music_id = music["musicId"]
level = music["level"]
score = music["scoreMax"]
rank = music["scoreRank"]
if music_detail_rows is None or len(music_detail_rows) == 0:
return {
"userId": user_id,
"rivalId": rival_id,
"nextIndex": -1,
"userRivalMusicList": [],
}
# Create a music entry for the current music_id if it's unique
music_entry = next((entry for entry in user_rival_music_list if entry["musicId"] == music_id), None)
if music_entry is None:
music_entry = {
"musicId": music_id,
"length": 0,
"userRivalMusicDetailList": []
}
user_rival_music_list.append(music_entry)
music_details = [x._asdict() for x in music_detail_rows]
returned_music_details_count = 0
music_list = []
# Create a level entry for the current level if it's unique or has a higher score
level_entry = next((entry for entry in music_entry["userRivalMusicDetailList"] if entry["level"] == level), None)
if level_entry is None:
level_entry = {
"level": level,
"scoreMax": score,
"scoreRank": rank
}
music_entry["userRivalMusicDetailList"].append(level_entry)
elif score > level_entry["scoreMax"]:
level_entry["scoreMax"] = score
level_entry["scoreRank"] = rank
# note that itertools.groupby will only work on sorted keys, which is already sorted by
# the query in get_scores
for music_id, details_iter in itertools.groupby(music_details, key=lambda x: x["musicId"]):
details: list[dict[Any, Any]] = [
{"level": d["level"], "scoreMax": d["scoreMax"]}
for d in details_iter
]
# Calculate the length for each "musicId" by counting the unique levels
for music_entry in user_rival_music_list:
music_entry["length"] = len(music_entry["userRivalMusicDetailList"])
music_list.append({"musicId": music_id, "length": len(details), "userMusicDetailList": details})
returned_music_details_count += len(details)
# Prepare the result dictionary with user rival music data
result = {
"userId": data["userId"],
"rivalId": data["rivalId"],
"nextIndex": str(next_index + len(user_rival_music_list[next_index: next_index + max_count]) if max_count <= len(user_rival_music_list[next_index: next_index + max_count]) else -1),
"userRivalMusicList": user_rival_music_list[next_index: next_index + max_count]
if len(music_list) >= max_ct:
break
# if we returned fewer PBs than we originally asked for from the database, that means
# we queried for the PBs of max_ct + 1 songs.
if returned_music_details_count < len(music_detail_rows):
next_idx += max_ct
else:
next_idx = -1
return {
"userId": user_id,
"rivalId": rival_id,
"length": len(music_list),
"nextIndex": next_idx,
"userRivalMusicList": music_list,
}
return result
async def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
user_id = int(data["userId"])
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
kind = int(data["kind"])
is_all_favorite_item = str(data["isAllFavoriteItem"]) == "true"
user_fav_item_list = []
# still needs to be implemented on WebUI
# 1: Music, 2: User, 3: Character
fav_list = await self.data.item.get_all_favorites(
data["userId"], self.version, fav_kind=int(data["kind"])
user_id,
self.version,
fav_kind=kind,
limit=max_ct + 1,
offset=next_idx,
)
if fav_list is not None:
for fav in fav_list:
user_fav_item_list.append({"id": fav["favId"]})
if fav_list is None or len(fav_list) <= max_ct:
next_idx = -1
else:
next_idx += max_ct
return {
"userId": data["userId"],
"userId": user_id,
"length": len(user_fav_item_list),
"kind": data["kind"],
"nextIndex": -1,
"kind": kind,
"nextIndex": next_idx,
"userFavoriteItemList": user_fav_item_list,
}
@ -505,36 +530,39 @@ class ChuniBase:
return {"userId": data["userId"], "length": 0, "userFavoriteMusicList": []}
async def handle_get_user_item_api_request(self, data: Dict) -> Dict:
kind = int(int(data["nextIndex"]) / 10000000000)
next_idx = int(int(data["nextIndex"]) % 10000000000)
user_item_list = await self.data.item.get_items(data["userId"], kind)
user_id = int(data["userId"])
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
if user_item_list is None or len(user_item_list) == 0:
kind = next_idx // 10000000000
next_idx = next_idx % 10000000000
rows = await self.data.item.get_items(
user_id, kind, limit=max_ct + 1, offset=next_idx
)
if rows is None or len(rows) == 0:
return {
"userId": data["userId"],
"userId": user_id,
"nextIndex": -1,
"itemKind": kind,
"userItemList": [],
}
items: List[Dict[str, Any]] = []
for i in range(next_idx, len(user_item_list)):
tmp = user_item_list[i]._asdict()
for i in range(min(len(rows), max_ct)):
tmp = rows[i]._asdict()
tmp.pop("user")
tmp.pop("id")
items.append(tmp)
if len(items) >= int(data["maxCount"]):
break
xout = kind * 10000000000 + next_idx + len(items)
if len(items) < int(data["maxCount"]):
next_idx = 0
if len(rows) > max_ct:
next_idx = kind * 10000000000 + next_idx + max_ct
else:
next_idx = xout
next_idx = -1
return {
"userId": data["userId"],
"userId": user_id,
"nextIndex": next_idx,
"itemKind": kind,
"length": len(items),
@ -586,62 +614,55 @@ class ChuniBase:
}
async def handle_get_user_music_api_request(self, data: Dict) -> Dict:
music_detail = await self.data.score.get_scores(data["userId"])
if music_detail is None:
user_id = int(data["userId"])
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
rows = await self.data.score.get_scores(
user_id, limit=max_ct + 1, offset=next_idx
)
if rows is None or len(rows) == 0:
return {
"userId": data["userId"],
"userId": user_id,
"length": 0,
"nextIndex": -1,
"userMusicList": [], # 240
}
song_list = []
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
music_details = [x._asdict() for x in rows]
returned_music_details_count = 0
music_list = []
for x in range(next_idx, len(music_detail)):
found = False
tmp = music_detail[x]._asdict()
tmp.pop("user")
tmp.pop("id")
# note that itertools.groupby will only work on sorted keys, which is already sorted by
# the query in get_scores
for _music_id, details_iter in itertools.groupby(music_details, key=lambda x: x["musicId"]):
details: list[dict[Any, Any]] = []
for song in song_list:
score_buf = SCORE_BUFFER.get(str(data["userId"])) or []
if song["userMusicDetailList"][0]["musicId"] == tmp["musicId"]:
found = True
song["userMusicDetailList"].append(tmp)
song["length"] = len(song["userMusicDetailList"])
score_buf.append(tmp["musicId"])
SCORE_BUFFER[str(data["userId"])] = score_buf
for d in details_iter:
d.pop("id")
d.pop("user")
score_buf = SCORE_BUFFER.get(str(data["userId"])) or []
if not found and tmp["musicId"] not in score_buf:
song_list.append({"length": 1, "userMusicDetailList": [tmp]})
score_buf.append(tmp["musicId"])
SCORE_BUFFER[str(data["userId"])] = score_buf
details.append(d)
if len(song_list) >= max_ct:
music_list.append({"length": len(details), "userMusicDetailList": details})
returned_music_details_count += len(details)
if len(music_list) >= max_ct:
break
for songIdx in range(len(song_list)):
for recordIdx in range(x+1, len(music_detail)):
if song_list[songIdx]["userMusicDetailList"][0]["musicId"] == music_detail[recordIdx]["musicId"]:
music = music_detail[recordIdx]._asdict()
music.pop("user")
music.pop("id")
song_list[songIdx]["userMusicDetailList"].append(music)
song_list[songIdx]["length"] += 1
if len(song_list) >= max_ct:
next_idx += len(song_list)
# if we returned fewer PBs than we originally asked for from the database, that means
# we queried for the PBs of max_ct + 1 songs.
if returned_music_details_count < len(rows):
next_idx += max_ct
else:
next_idx = -1
SCORE_BUFFER[str(data["userId"])] = []
return {
"userId": data["userId"],
"length": len(song_list),
"userId": user_id,
"length": len(music_list),
"nextIndex": next_idx,
"userMusicList": song_list, # 240
"userMusicList": music_list,
}
async def handle_get_user_option_api_request(self, data: Dict) -> Dict:

54
titles/chuni/const.py
View File

@ -1,4 +1,4 @@
from enum import Enum
from enum import Enum, IntEnum
class ChuniConstants:
@ -81,12 +81,31 @@ class ChuniConstants:
return cls.VERSION_NAMES[ver]
class MapAreaConditionType(Enum):
UNLOCKED = 0
class MapAreaConditionType(IntEnum):
"""Condition types for the GetGameMapAreaConditionApi endpoint. Incomplete.
For the MAP_CLEARED/MAP_AREA_CLEARED/TROPHY_OBTAINED conditions, the conditionId
is the map/map area/trophy.
For the RANK_*/ALL_JUSTICE conditions, the conditionId is songId * 100 + difficultyId.
For example, Halcyon [ULTIMA] would be 173 * 100 + 4 = 17304.
"""
ALWAYS_UNLOCKED = 0
MAP_CLEARED = 1
MAP_AREA_CLEARED = 2
TROPHY_OBTAINED = 3
RANK_SSS = 19
RANK_SSP = 20
RANK_SS = 21
RANK_SP = 22
RANK_S = 23
ALL_JUSTICE = 28
class MapAreaConditionLogicalOperator(Enum):
AND = 1
@ -102,11 +121,36 @@ class AvatarCategory(Enum):
FRONT = 6
BACK = 7
class ItemKind(Enum):
class ItemKind(IntEnum):
NAMEPLATE = 1
FRAME = 2
"""
"Frame" is the background for the gauge/score/max combo display
shown during gameplay. This item cannot be equipped (as of LUMINOUS)
and is hardcoded to the current game's version.
"""
TROPHY = 3
SKILL = 4
TICKET = 5
"""A statue is also a ticket."""
PRESENT = 6
MUSIC_UNLOCK = 7
MAP_ICON = 8
SYSTEM_VOICE = 9
AVATAR_ACCESSORY = 11
SYMBOL_CHAT = 10
AVATAR_ACCESSORY = 11
ULTIMA_UNLOCK = 12
"""This only applies to ULTIMA difficulties that are *not* unlocked by
SS-ing EXPERT+MASTER.
"""
class FavoriteItemKind(IntEnum):
MUSIC = 1
RIVAL = 2
CHARACTER = 3

34
titles/chuni/new.py
View File

@ -4,12 +4,14 @@ from random import randint
from typing import Dict
import pytz
from core.config import CoreConfig
from core.utils import Utils
from titles.chuni.const import ChuniConstants
from titles.chuni.database import ChuniData
from titles.chuni.base import ChuniBase
from titles.chuni.config import ChuniConfig
from titles.chuni.const import ChuniConstants
from titles.chuni.database import ChuniData
class ChuniNew(ChuniBase):
ITEM_TYPE = {"character": 20, "story": 21, "card": 22}
@ -285,30 +287,32 @@ class ChuniNew(ChuniBase):
}
async def handle_get_user_printed_card_api_request(self, data: Dict) -> Dict:
user_print_list = await self.data.item.get_user_print_states(
data["userId"], has_completed=True
user_id = int(data["userId"])
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
rows = await self.data.item.get_user_print_states(
user_id,
has_completed=True,
limit=max_ct + 1,
offset=next_idx,
)
if user_print_list is None:
if rows is None or len(rows) == 0:
return {
"userId": data["userId"],
"userId": user_id,
"length": 0,
"nextIndex": -1,
"userPrintedCardList": [],
}
print_list = []
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
for x in range(next_idx, len(user_print_list)):
tmp = user_print_list[x]._asdict()
for i in range(min(max_ct, len(rows))):
tmp = rows[i]._asdict()
print_list.append(tmp["cardId"])
if len(print_list) >= max_ct:
break
if len(print_list) >= max_ct:
next_idx = next_idx + max_ct
if len(rows) > max_ct:
next_idx += max_ct
else:
next_idx = -1

61
titles/chuni/schema/item.py
View File

@ -1,22 +1,22 @@
from typing import Dict, List, Optional
from sqlalchemy import (
Table,
Column,
UniqueConstraint,
PrimaryKeyConstraint,
Table,
UniqueConstraint,
and_,
delete,
)
from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON
from sqlalchemy.engine.base import Connection
from sqlalchemy.schema import ForeignKey
from sqlalchemy.sql import func, select
from sqlalchemy.dialects.mysql import insert
from sqlalchemy.engine import Row
from sqlalchemy.schema import ForeignKey
from sqlalchemy.sql import func, select
from sqlalchemy.types import JSON, TIMESTAMP, Boolean, Integer, String
from core.data.schema import BaseData, metadata
character = Table(
character: Table = Table(
"chuni_item_character",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@ -40,7 +40,7 @@ character = Table(
mysql_charset="utf8mb4",
)
item = Table(
item: Table = Table(
"chuni_item_item",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@ -141,7 +141,7 @@ gacha = Table(
mysql_charset="utf8mb4",
)
print_state = Table(
print_state: Table = Table(
"chuni_item_print_state",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@ -210,7 +210,7 @@ login_bonus = Table(
mysql_charset="utf8mb4",
)
favorite = Table(
favorite: Table = Table(
"chuni_item_favorite",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@ -379,9 +379,14 @@ class ChuniItemData(BaseData):
return True if len(result.all()) else False
async def get_all_favorites(
self, user_id: int, version: int, fav_kind: int = 1
self,
user_id: int,
version: int,
fav_kind: int = 1,
limit: Optional[int] = None,
offset: int = 0,
) -> Optional[List[Row]]:
sql = favorite.select(
sql = select(favorite).where(
and_(
favorite.c.version == version,
favorite.c.user == user_id,
@ -389,6 +394,9 @@ class ChuniItemData(BaseData):
)
)
if limit is not None:
sql = sql.order_by(favorite.c.id.asc()).limit(limit).offset(offset)
result = await self.execute(sql)
if result is None:
return None
@ -488,9 +496,14 @@ class ChuniItemData(BaseData):
return None
return result.fetchone()
async def get_characters(self, user_id: int) -> Optional[List[Row]]:
async def get_characters(
self, user_id: int, limit: Optional[int] = None, offset: int = 0
) -> Optional[List[Row]]:
sql = select(character).where(character.c.user == user_id)
if limit is not None:
sql = sql.limit(limit).offset(offset).order_by(character.c.id.asc())
result = await self.execute(sql)
if result is None:
return None
@ -509,7 +522,13 @@ class ChuniItemData(BaseData):
return None
return result.lastrowid
async def get_items(self, user_id: int, kind: int = None) -> Optional[List[Row]]:
async def get_items(
self,
user_id: int,
kind: Optional[int] = None,
limit: Optional[int] = None,
offset: int = 0
) -> Optional[List[Row]]:
if kind is None:
sql = select(item).where(item.c.user == user_id)
else:
@ -517,6 +536,9 @@ class ChuniItemData(BaseData):
and_(item.c.user == user_id, item.c.itemKind == kind)
)
if limit is not None:
sql = sql.order_by(item.c.id.asc()).limit(limit).offset(offset)
result = await self.execute(sql)
if result is None:
return None
@ -609,15 +631,22 @@ class ChuniItemData(BaseData):
return result.lastrowid
async def get_user_print_states(
self, aime_id: int, has_completed: bool = False
self,
aime_id: int,
has_completed: bool = False,
limit: Optional[int] = None,
offset: int = 0,
) -> Optional[List[Row]]:
sql = print_state.select(
sql = select(print_state).where(
and_(
print_state.c.user == aime_id,
print_state.c.hasCompleted == has_completed,
)
)
if limit is not None:
sql = sql.order_by(print_state.c.id.asc()).limit(limit).offset(offset)
result = await self.execute(sql)
if result is None:
return None

66
titles/chuni/schema/score.py
View File

@ -1,16 +1,19 @@
from typing import Dict, List, Optional
from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger
from typing import Dict, List, Optional, Union
from sqlalchemy import Column, PrimaryKeyConstraint, Table, UniqueConstraint, and_
from sqlalchemy.dialects.mysql import insert
from sqlalchemy.engine import Row
from sqlalchemy.engine.base import Connection
from sqlalchemy.schema import ForeignKey
from sqlalchemy.engine import Row
from sqlalchemy.sql import func, select
from sqlalchemy.dialects.mysql import insert
from sqlalchemy.sql.expression import exists
from sqlalchemy.types import JSON, TIMESTAMP, BigInteger, Boolean, Integer, String
from core.data.schema import BaseData, metadata
from ..config import ChuniConfig
course = Table(
course: Table = Table(
"chuni_score_course",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@ -41,7 +44,7 @@ course = Table(
mysql_charset="utf8mb4",
)
best_score = Table(
best_score: Table = Table(
"chuni_score_best",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@ -229,9 +232,12 @@ class ChuniRomVersion():
return -1
class ChuniScoreData(BaseData):
async def get_courses(self, aime_id: int) -> Optional[Row]:
async def get_courses(self, aime_id: int, limit: Optional[int] = None, offset: int = 0) -> Optional[List[Row]]:
sql = select(course).where(course.c.user == aime_id)
if limit is not None:
sql = sql.order_by(course.c.id.asc()).limit(limit).offset(offset)
result = await self.execute(sql)
if result is None:
return None
@ -249,8 +255,40 @@ class ChuniScoreData(BaseData):
return None
return result.lastrowid
async def get_scores(self, aime_id: int) -> Optional[Row]:
sql = select(best_score).where(best_score.c.user == aime_id)
async def get_scores(
self,
aime_id: int,
levels: Optional[list[int]] = None,
limit: Optional[int] = None,
offset: int = 0,
) -> Optional[List[Row]]:
condition = best_score.c.user == aime_id
if levels is not None:
condition &= best_score.c.level.in_(levels)
if limit is None:
sql = (
select(best_score)
.where(condition)
.order_by(best_score.c.musicId.asc(), best_score.c.level.asc())
)
else:
subq = (
select(best_score.c.musicId)
.distinct()
.where(condition)
.order_by(best_score.c.musicId.asc())
.limit(limit)
.offset(offset)
.subquery()
)
sql = (
select(best_score)
.join(subq, best_score.c.musicId == subq.c.musicId)
.where(condition)
.order_by(best_score.c.musicId.asc(), best_score.c.level.asc())
)
result = await self.execute(sql)
if result is None:
@ -360,11 +398,3 @@ class ChuniScoreData(BaseData):
rows = result.fetchall()
return [dict(row) for row in rows]
async def get_rival_music(self, rival_id: int) -> Optional[List[Dict]]:
sql = select(best_score).where(best_score.c.user == rival_id)
result = await self.execute(sql)
if result is None:
return None
return result.fetchall()