[Benchmark] Auto-detect and correct client/server tokenizer mismatch for random dataset (#44708)
This commit is contained in:
@@ -31,7 +31,7 @@ import time
|
||||
import uuid
|
||||
import warnings
|
||||
from collections.abc import AsyncGenerator, Iterable
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
@@ -62,6 +62,80 @@ TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) a
|
||||
)
|
||||
|
||||
|
||||
async def _align_prompts_to_server_tokenizer(
|
||||
base_url: str,
|
||||
model_id: str,
|
||||
input_requests: list[SampleRequest],
|
||||
ssl_context: ssl.SSLContext | bool | None = None,
|
||||
) -> list[SampleRequest]:
|
||||
"""Re-align prompts if local/server tokenizers disagree."""
|
||||
if not input_requests or not isinstance(input_requests[0].prompt, str):
|
||||
return input_requests
|
||||
|
||||
tok_url = f"{base_url}/tokenize"
|
||||
detok_url = f"{base_url}/detokenize"
|
||||
connector = aiohttp.TCPConnector(ssl=ssl_context)
|
||||
|
||||
async with aiohttp.ClientSession(connector=connector) as session:
|
||||
sem = asyncio.Semaphore(64)
|
||||
|
||||
async def _tokenize(prompt: str) -> list[int]:
|
||||
async with (
|
||||
sem,
|
||||
session.post(
|
||||
tok_url,
|
||||
json={
|
||||
"model": model_id,
|
||||
"prompt": prompt,
|
||||
"add_special_tokens": False,
|
||||
},
|
||||
) as r,
|
||||
):
|
||||
r.raise_for_status()
|
||||
return (await r.json())["tokens"]
|
||||
|
||||
async def _detokenize(tokens: list[int]) -> str:
|
||||
async with (
|
||||
sem,
|
||||
session.post(
|
||||
detok_url, json={"model": model_id, "tokens": tokens}
|
||||
) as r,
|
||||
):
|
||||
r.raise_for_status()
|
||||
return (await r.json())["prompt"]
|
||||
|
||||
try:
|
||||
first_tokens = await _tokenize(input_requests[0].prompt)
|
||||
except Exception:
|
||||
print("WARNING: /tokenize unavailable, skipping alignment.")
|
||||
return input_requests
|
||||
|
||||
expected = input_requests[0].prompt_len
|
||||
if len(first_tokens) == expected:
|
||||
return input_requests
|
||||
|
||||
print(
|
||||
f"WARNING: tokenizer mismatch "
|
||||
f"(server={len(first_tokens)}, expected={expected}), "
|
||||
f"re-aligning prompts."
|
||||
)
|
||||
|
||||
async def _fix_one(req: SampleRequest) -> SampleRequest:
|
||||
tokens = await _tokenize(req.prompt)
|
||||
if len(tokens) <= req.prompt_len:
|
||||
return req
|
||||
corrected = await _detokenize(tokens[: req.prompt_len])
|
||||
return replace(req, prompt=corrected, prompt_len=req.prompt_len)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*[_fix_one(r) for r in input_requests], return_exceptions=True
|
||||
)
|
||||
return [
|
||||
res if not isinstance(res, BaseException) else orig
|
||||
for orig, res in zip(input_requests, results)
|
||||
]
|
||||
|
||||
|
||||
async def get_first_model_from_server(
|
||||
base_url: str,
|
||||
headers: dict | None = None,
|
||||
@@ -1821,6 +1895,12 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
||||
|
||||
# Load the dataset.
|
||||
input_requests = get_samples(args, tokenizer)
|
||||
|
||||
if args.dataset_name in ("random", "prefix_repetition"):
|
||||
input_requests = await _align_prompts_to_server_tokenizer(
|
||||
base_url, model_id, input_requests, ssl_context
|
||||
)
|
||||
|
||||
goodput_config_dict = check_goodput_args(args)
|
||||
|
||||
backend = args.backend
|
||||
|
||||
Reference in New Issue
Block a user