use SQL's limit/offset pagination for nextIndex/maxCount requests (#185)

Instead of retrieving the entire list of items/characters/scores/etc. at once (and even store them in memory), use SQL's `LIMIT ... OFFSET ...` pagination so we only take what we need.

Currently only CHUNITHM uses this, but this will also affect maimai DX and O.N.G.E.K.I. once the PR is ready.

Also snuck in a fix for CHUNITHM/maimai DX's `GetUserRivalMusicApi` to respect the `userRivalMusicLevelList` sent by the client.

### How this works

Say we have a `GetUserCharacterApi` request:

```json
{
    "userId": 10000,
    "maxCount": 700,
    "nextIndex": 0
}
```

Instead of getting the entire character list from the database (which can be very large if the user force unlocked everything), add limit/offset to the query:

```python
select(character)
.where(character.c.user == user_id)
.order_by(character.c.id.asc())
.limit(max_count + 1)
.offset(next_index)
```

The query takes `maxCount + 1` items from the database to determine if there is more items than can be returned:

```python
rows = ...

if len(rows) > max_count:
    # return only max_count rows
    next_index += max_count
else:
    # return everything left
    next_index = -1
```

This has the benefit of not needing to load everything into memory (and also having to store server state, as seen in the [`SCORE_BUFFER` list](2274b42358/titles/chuni/base.py (L13)).)

Reviewed-on: Hay1tsme/artemis#185
Co-authored-by: beerpsi <beerpsi@duck.com>
Co-committed-by: beerpsi <beerpsi@duck.com>
This commit is contained in:
2024-11-16 19:10:29 +00:00
committed by Hay1tsme
parent cb009f6e23
commit 58a5177a30
18 changed files with 1410 additions and 713 deletions

View File

@ -1,13 +1,11 @@
from datetime import date, datetime, timedelta
from typing import Any, Dict
from datetime import datetime
from random import randint
import pytz
import json
from typing import Dict
from core.config import CoreConfig
from titles.ongeki.base import OngekiBase
from titles.ongeki.const import OngekiConstants
from titles.ongeki.config import OngekiConfig
from titles.ongeki.const import OngekiConstants
class OngekiBright(OngekiBase):
@ -62,66 +60,72 @@ class OngekiBright(OngekiBase):
return {"returnCode": 1}
async def handle_cm_get_user_card_api_request(self, data: Dict) -> Dict:
user_cards = await self.data.item.get_cards(data["userId"])
if user_cards is None:
user_id: int = data["userId"]
max_ct: int = data["maxCount"]
next_idx: int = data["nextIndex"]
rows = await self.data.item.get_cards(
user_id, limit=max_ct + 1, offset=next_idx
)
if rows is None:
return {}
card_list = []
max_ct = data["maxCount"]
next_idx = data["nextIndex"]
start_idx = next_idx
end_idx = max_ct + start_idx
if len(user_cards[start_idx:]) > max_ct:
for row in rows[:max_ct]:
card = row._asdict()
card.pop("id")
card.pop("user")
card_list.append(card)
if len(rows) > max_ct:
next_idx += max_ct
else:
next_idx = -1
card_list = []
for card in user_cards:
tmp = card._asdict()
tmp.pop("id")
tmp.pop("user")
card_list.append(tmp)
next_idx = 0
return {
"userId": data["userId"],
"length": len(card_list[start_idx:end_idx]),
"length": len(card_list),
"nextIndex": next_idx,
"userCardList": card_list[start_idx:end_idx],
"userCardList": card_list,
}
async def handle_cm_get_user_character_api_request(self, data: Dict) -> Dict:
user_characters = await self.data.item.get_characters(data["userId"])
if user_characters is None:
user_id: int = data["userId"]
max_ct: int = data["maxCount"]
next_idx: int = data["nextIndex"]
rows = await self.data.item.get_characters(
user_id, limit=max_ct + 1, offset=next_idx
)
if rows is None:
return {
"userId": data["userId"],
"userId": user_id,
"length": 0,
"nextIndex": 0,
"userCharacterList": [],
}
max_ct = data["maxCount"]
next_idx = data["nextIndex"]
start_idx = next_idx
end_idx = max_ct + start_idx
character_list = []
if len(user_characters[start_idx:]) > max_ct:
for row in rows[:max_ct]:
character = row._asdict()
character.pop("id")
character.pop("user")
character_list.append(character)
if len(rows) > max_ct:
next_idx += max_ct
else:
next_idx = -1
character_list = []
for character in user_characters:
tmp = character._asdict()
tmp.pop("id")
tmp.pop("user")
character_list.append(tmp)
next_idx = 0
return {
"userId": data["userId"],
"length": len(character_list[start_idx:end_idx]),
"length": len(character_list),
"nextIndex": next_idx,
"userCharacterList": character_list[start_idx:end_idx],
"userCharacterList": character_list,
}
async def handle_get_user_gacha_api_request(self, data: Dict) -> Dict: