forked from Hay1tsme/artemis
use SQL's limit/offset pagination for nextIndex/maxCount requests (#185)
Instead of retrieving the entire list of items/characters/scores/etc. at once (and even store them in memory), use SQL's `LIMIT ... OFFSET ...` pagination so we only take what we need.
Currently only CHUNITHM uses this, but this will also affect maimai DX and O.N.G.E.K.I. once the PR is ready.
Also snuck in a fix for CHUNITHM/maimai DX's `GetUserRivalMusicApi` to respect the `userRivalMusicLevelList` sent by the client.
### How this works
Say we have a `GetUserCharacterApi` request:
```json
{
"userId": 10000,
"maxCount": 700,
"nextIndex": 0
}
```
Instead of getting the entire character list from the database (which can be very large if the user force unlocked everything), add limit/offset to the query:
```python
select(character)
.where(character.c.user == user_id)
.order_by(character.c.id.asc())
.limit(max_count + 1)
.offset(next_index)
```
The query takes `maxCount + 1` items from the database to determine if there is more items than can be returned:
```python
rows = ...
if len(rows) > max_count:
# return only max_count rows
next_index += max_count
else:
# return everything left
next_index = -1
```
This has the benefit of not needing to load everything into memory (and also having to store server state, as seen in the [`SCORE_BUFFER` list](2274b42358/titles/chuni/base.py (L13)).)
Reviewed-on: Hay1tsme/artemis#185
Co-authored-by: beerpsi <beerpsi@duck.com>
Co-committed-by: beerpsi <beerpsi@duck.com>
This commit is contained in:
@ -1,16 +1,16 @@
|
||||
from datetime import date, datetime, timedelta
|
||||
from typing import Any, Dict, List
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import pytz
|
||||
|
||||
from core.config import CoreConfig
|
||||
from core.data.cache import cached
|
||||
from titles.ongeki.config import OngekiConfig
|
||||
from titles.ongeki.const import OngekiConstants
|
||||
from titles.ongeki.config import OngekiConfig
|
||||
from titles.ongeki.database import OngekiData
|
||||
from titles.ongeki.config import OngekiConfig
|
||||
|
||||
|
||||
class OngekiBattleGrade(Enum):
|
||||
@ -500,57 +500,93 @@ class OngekiBase:
|
||||
}
|
||||
|
||||
async def handle_get_user_music_api_request(self, data: Dict) -> Dict:
|
||||
song_list = await self.util_generate_music_list(data["userId"])
|
||||
max_ct = data["maxCount"]
|
||||
next_idx = data["nextIndex"]
|
||||
start_idx = next_idx
|
||||
end_idx = max_ct + start_idx
|
||||
user_id: int = data["userId"]
|
||||
next_idx: int = data["nextIndex"]
|
||||
max_ct: int = data["maxCount"]
|
||||
|
||||
if len(song_list[start_idx:]) > max_ct:
|
||||
rows = await self.data.score.get_best_scores(
|
||||
user_id, limit=max_ct + 1, offset=next_idx
|
||||
)
|
||||
|
||||
if rows is None:
|
||||
return {
|
||||
"userId": user_id,
|
||||
"length": 0,
|
||||
"nextIndex": 0,
|
||||
"userMusicList": [],
|
||||
}
|
||||
|
||||
music_details = [row._asdict() for row in rows]
|
||||
returned_count = 0
|
||||
music_list = []
|
||||
|
||||
for _music_id, details_iter in itertools.groupby(music_details, key=lambda d: d["musicId"]):
|
||||
details: list[dict[Any, Any]] = []
|
||||
|
||||
for d in details_iter:
|
||||
d.pop("id")
|
||||
d.pop("user")
|
||||
|
||||
details.append(d)
|
||||
|
||||
music_list.append({"length": len(details), "userMusicDetailList": details})
|
||||
returned_count += len(details)
|
||||
|
||||
if len(music_list) >= max_ct:
|
||||
break
|
||||
|
||||
if returned_count < len(rows):
|
||||
next_idx += max_ct
|
||||
|
||||
else:
|
||||
next_idx = -1
|
||||
next_idx = 0
|
||||
|
||||
return {
|
||||
"userId": data["userId"],
|
||||
"length": len(song_list[start_idx:end_idx]),
|
||||
"userId": user_id,
|
||||
"length": len(music_list),
|
||||
"nextIndex": next_idx,
|
||||
"userMusicList": song_list[start_idx:end_idx],
|
||||
"userMusicList": music_list,
|
||||
}
|
||||
|
||||
async def handle_get_user_item_api_request(self, data: Dict) -> Dict:
|
||||
kind = data["nextIndex"] / 10000000000
|
||||
p = await self.data.item.get_items(data["userId"], kind)
|
||||
user_id: int = data["userId"]
|
||||
next_idx: int = data["nextIndex"]
|
||||
max_ct: int = data["maxCount"]
|
||||
|
||||
if p is None:
|
||||
kind = next_idx // 10000000000
|
||||
next_idx = next_idx % 10000000000
|
||||
|
||||
rows = await self.data.item.get_items(
|
||||
user_id, kind, limit=max_ct + 1, offset=next_idx
|
||||
)
|
||||
|
||||
if rows is None:
|
||||
return {
|
||||
"userId": data["userId"],
|
||||
"nextIndex": -1,
|
||||
"userId": user_id,
|
||||
"nextIndex": 0,
|
||||
"itemKind": kind,
|
||||
"length": 0,
|
||||
"userItemList": [],
|
||||
}
|
||||
|
||||
items: List[Dict[str, Any]] = []
|
||||
for i in range(data["nextIndex"] % 10000000000, len(p)):
|
||||
if len(items) > data["maxCount"]:
|
||||
break
|
||||
tmp = p[i]._asdict()
|
||||
tmp.pop("user")
|
||||
tmp.pop("id")
|
||||
items.append(tmp)
|
||||
|
||||
xout = kind * 10000000000 + (data["nextIndex"] % 10000000000) + len(items)
|
||||
for row in rows[:max_ct]:
|
||||
item = row._asdict()
|
||||
|
||||
item.pop("id")
|
||||
item.pop("user")
|
||||
|
||||
items.append(item)
|
||||
|
||||
if len(items) < data["maxCount"] or data["maxCount"] == 0:
|
||||
nextIndex = 0
|
||||
if len(rows) > max_ct:
|
||||
next_idx = kind * 10000000000 + next_idx + max_ct
|
||||
else:
|
||||
nextIndex = xout
|
||||
next_idx = 0
|
||||
|
||||
return {
|
||||
"userId": data["userId"],
|
||||
"nextIndex": int(nextIndex),
|
||||
"itemKind": int(kind),
|
||||
"userId": user_id,
|
||||
"nextIndex": next_idx,
|
||||
"itemKind": kind,
|
||||
"length": len(items),
|
||||
"userItemList": items,
|
||||
}
|
||||
@ -1143,43 +1179,56 @@ class OngekiBase:
|
||||
"""
|
||||
Added in Bright
|
||||
"""
|
||||
rival_id = data["rivalUserId"]
|
||||
next_idx = data["nextIndex"]
|
||||
max_ct = data["maxCount"]
|
||||
music = self.handle_get_user_music_api_request(
|
||||
{"userId": rival_id, "nextIndex": next_idx, "maxCount": max_ct}
|
||||
user_id: int = data["userId"]
|
||||
rival_id: int = data["rivalUserId"]
|
||||
next_idx: int = data["nextIndex"]
|
||||
max_ct: int = data["maxCount"]
|
||||
|
||||
rows = await self.data.score.get_best_scores(
|
||||
rival_id, limit=max_ct + 1, offset=next_idx
|
||||
)
|
||||
|
||||
for song in music["userMusicList"]:
|
||||
song["userRivalMusicDetailList"] = song["userMusicDetailList"]
|
||||
song.pop("userMusicDetailList")
|
||||
if rows is None:
|
||||
return {
|
||||
"userId": user_id,
|
||||
"rivalUserId": rival_id,
|
||||
"nextIndex": 0,
|
||||
"length": 0,
|
||||
"userRivalMusicList": [],
|
||||
}
|
||||
|
||||
music_details = [row._asdict() for row in rows]
|
||||
returned_count = 0
|
||||
music_list = []
|
||||
|
||||
for _music_id, details_iter in itertools.groupby(music_details, key=lambda d: d["musicId"]):
|
||||
details: list[dict[Any, Any]] = []
|
||||
|
||||
for d in details_iter:
|
||||
d.pop("id")
|
||||
d.pop("user")
|
||||
d.pop("playCount")
|
||||
d.pop("isLock")
|
||||
d.pop("clearStatus")
|
||||
d.pop("isStoryWatched")
|
||||
|
||||
details.append(d)
|
||||
|
||||
music_list.append({"length": len(details), "userRivalMusicDetailList": details})
|
||||
returned_count += len(details)
|
||||
|
||||
if len(music_list) >= max_ct:
|
||||
break
|
||||
|
||||
if returned_count < len(rows):
|
||||
next_idx += max_ct
|
||||
else:
|
||||
next_idx = 0
|
||||
|
||||
return {
|
||||
"userId": data["userId"],
|
||||
"userId": user_id,
|
||||
"rivalUserId": rival_id,
|
||||
"length": music["length"],
|
||||
"nextIndex": music["nextIndex"],
|
||||
"userRivalMusicList": music["userMusicList"],
|
||||
"nextIndex": next_idx,
|
||||
"length": len(music_list),
|
||||
"userRivalMusicList": music_list,
|
||||
}
|
||||
|
||||
@cached(2)
|
||||
async def util_generate_music_list(self, user_id: int) -> List:
|
||||
music_detail = await self.data.score.get_best_scores(user_id)
|
||||
song_list = []
|
||||
|
||||
for md in music_detail:
|
||||
found = False
|
||||
tmp = md._asdict()
|
||||
tmp.pop("user")
|
||||
tmp.pop("id")
|
||||
|
||||
for song in song_list:
|
||||
if song["userMusicDetailList"][0]["musicId"] == tmp["musicId"]:
|
||||
found = True
|
||||
song["userMusicDetailList"].append(tmp)
|
||||
song["length"] = len(song["userMusicDetailList"])
|
||||
break
|
||||
|
||||
if not found:
|
||||
song_list.append({"length": 1, "userMusicDetailList": [tmp]})
|
||||
|
||||
return song_list
|
||||
|
||||
Reference in New Issue
Block a user