diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 5ebc297d50..6c55d287a3 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -31,7 +31,7 @@ import time import uuid import warnings from collections.abc import AsyncGenerator, Iterable -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime from enum import Enum from pathlib import Path @@ -62,6 +62,80 @@ TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) a ) +async def _align_prompts_to_server_tokenizer( + base_url: str, + model_id: str, + input_requests: list[SampleRequest], + ssl_context: ssl.SSLContext | bool | None = None, +) -> list[SampleRequest]: + """Re-align prompts if local/server tokenizers disagree.""" + if not input_requests or not isinstance(input_requests[0].prompt, str): + return input_requests + + tok_url = f"{base_url}/tokenize" + detok_url = f"{base_url}/detokenize" + connector = aiohttp.TCPConnector(ssl=ssl_context) + + async with aiohttp.ClientSession(connector=connector) as session: + sem = asyncio.Semaphore(64) + + async def _tokenize(prompt: str) -> list[int]: + async with ( + sem, + session.post( + tok_url, + json={ + "model": model_id, + "prompt": prompt, + "add_special_tokens": False, + }, + ) as r, + ): + r.raise_for_status() + return (await r.json())["tokens"] + + async def _detokenize(tokens: list[int]) -> str: + async with ( + sem, + session.post( + detok_url, json={"model": model_id, "tokens": tokens} + ) as r, + ): + r.raise_for_status() + return (await r.json())["prompt"] + + try: + first_tokens = await _tokenize(input_requests[0].prompt) + except Exception: + print("WARNING: /tokenize unavailable, skipping alignment.") + return input_requests + + expected = input_requests[0].prompt_len + if len(first_tokens) == expected: + return input_requests + + print( + f"WARNING: tokenizer mismatch " + f"(server={len(first_tokens)}, expected={expected}), " + f"re-aligning prompts." + ) + + async def _fix_one(req: SampleRequest) -> SampleRequest: + tokens = await _tokenize(req.prompt) + if len(tokens) <= req.prompt_len: + return req + corrected = await _detokenize(tokens[: req.prompt_len]) + return replace(req, prompt=corrected, prompt_len=req.prompt_len) + + results = await asyncio.gather( + *[_fix_one(r) for r in input_requests], return_exceptions=True + ) + return [ + res if not isinstance(res, BaseException) else orig + for orig, res in zip(input_requests, results) + ] + + async def get_first_model_from_server( base_url: str, headers: dict | None = None, @@ -1821,6 +1895,12 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]: # Load the dataset. input_requests = get_samples(args, tokenizer) + + if args.dataset_name in ("random", "prefix_repetition"): + input_requests = await _align_prompts_to_server_tokenizer( + base_url, model_id, input_requests, ssl_context + ) + goodput_config_dict = check_goodput_args(args) backend = args.backend