Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
138 lines
4.8 KiB
Python
138 lines
4.8 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Exact batch shapes for `vllm bench forward`."""
|
|
|
|
import json
|
|
import zlib
|
|
from dataclasses import dataclass, replace
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import regex as re
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BenchRequest:
|
|
query_len: int
|
|
# KV length after the step: the prefix already in the KV cache plus the
|
|
# query tokens.
|
|
kv_len: int
|
|
# Spec-decode verify: query_len == 1 + num_draft_tokens.
|
|
num_draft_tokens: int = 0
|
|
# The prefix was computed by this request's earlier chunks. Otherwise a
|
|
# prefill's prefix is a prefix-cache hit, of which models with SWA bounded
|
|
# replay (DeepSeek-V4.1) recompute the last window.
|
|
chunked: bool = False
|
|
|
|
def __post_init__(self) -> None:
|
|
if (
|
|
self.query_len < 1
|
|
or self.kv_len < self.query_len
|
|
or self.num_draft_tokens < 0
|
|
):
|
|
raise ValueError(f"Invalid request: {self}")
|
|
if self.num_draft_tokens and not self.is_decode:
|
|
raise ValueError(
|
|
f"Verify needs query_len == 1 + num_draft_tokens and a prefix: {self}"
|
|
)
|
|
if self.chunked and (self.is_decode or not self.prefix_len):
|
|
raise ValueError(f"A chunk needs a prefill with a prefix: {self}")
|
|
|
|
@property
|
|
def prefix_len(self) -> int:
|
|
return self.kv_len - self.query_len
|
|
|
|
@property
|
|
def is_decode(self) -> bool:
|
|
"""Decode or verify; otherwise a prefill."""
|
|
return self.prefix_len > 0 and self.query_len == 1 + self.num_draft_tokens
|
|
|
|
@property
|
|
def prefix_hit(self) -> bool:
|
|
return self.prefix_len > 0 and not self.is_decode and not self.chunked
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BenchCase:
|
|
name: str
|
|
requests: tuple[BenchRequest, ...]
|
|
|
|
@property
|
|
def num_tokens(self) -> int:
|
|
return sum(r.query_len for r in self.requests)
|
|
|
|
|
|
def jitter_kv_lens(
|
|
case: BenchCase, jitter: float, max_kv_len: int, seed: int
|
|
) -> BenchCase:
|
|
"""Draw each request's prefix uniformly within +-jitter of its value.
|
|
|
|
Query lengths are kept, so a fresh prefill stays one and a non-empty
|
|
prefix stays non-empty. KV lengths are capped at `max_kv_len`. The draw
|
|
depends only on the seed and the case's name, not on the other cases.
|
|
"""
|
|
rng = np.random.default_rng([seed, zlib.crc32(case.name.encode())])
|
|
requests = []
|
|
for req in case.requests:
|
|
if req.kv_len > max_kv_len:
|
|
raise ValueError(f"{case.name}: KV length {req.kv_len} > {max_kv_len}")
|
|
prefix = req.prefix_len
|
|
if prefix and jitter:
|
|
prefix = max(1, round(prefix * rng.uniform(1 - jitter, 1 + jitter)))
|
|
kv_len = min(req.query_len + prefix, max_kv_len)
|
|
requests.append(replace(req, kv_len=kv_len))
|
|
return replace(case, requests=tuple(requests))
|
|
|
|
|
|
_REQ_RE = re.compile(r"^(?:(\d+)\*)?(\d+)(?:@(\d+))?(c)?(?::(\d+))?$")
|
|
|
|
|
|
def parse_case(text: str) -> BenchCase:
|
|
"""Parse `NAME=REQS`, where REQS is a comma list of `[N*]Q[@K][c][:D]`.
|
|
|
|
Q is the query length, K the KV length including the query (default Q, a
|
|
fresh prefill), D the number of draft tokens (verify) and N the number of
|
|
such requests. A prefill's prefix is a prefix-cache hit unless `c` marks
|
|
it as a chunked prefill. For example, `decode=32*1@8192`,
|
|
`verify=16*4@4096:3`, `prefill=8192`, `hit=8192@40960`,
|
|
`chunk=8192@40960c` or `mix=8192,32*1@8192`.
|
|
"""
|
|
name, sep, body = text.partition("=")
|
|
if not sep:
|
|
name, body = text, text
|
|
requests: list[BenchRequest] = []
|
|
for item in body.split(","):
|
|
match = _REQ_RE.match(item.strip())
|
|
if match is None:
|
|
raise ValueError(f"Invalid request {item!r} in case {text!r}")
|
|
count, query, kv, chunked, drafts = match.groups()
|
|
request = BenchRequest(
|
|
int(query), int(kv or query), int(drafts or 0), chunked=bool(chunked)
|
|
)
|
|
requests.extend([request] * int(count or 1))
|
|
return BenchCase(name, tuple(requests))
|
|
|
|
|
|
def load_cases(path: str) -> list[BenchCase]:
|
|
"""Load cases from a JSON or YAML file.
|
|
|
|
Format: `{"cases": [{"name": ..., "requests": [{"query_len": ...,
|
|
"kv_len": ..., "num_draft_tokens": ..., "chunked": ..., "count": ...}]}]}`.
|
|
"""
|
|
text = Path(path).read_text()
|
|
if path.endswith((".yaml", ".yml")):
|
|
import yaml
|
|
|
|
data = yaml.safe_load(text)
|
|
else:
|
|
data = json.loads(text)
|
|
cases = []
|
|
for case in data["cases"]:
|
|
requests: list[BenchRequest] = []
|
|
for req in case["requests"]:
|
|
req = dict(req)
|
|
count = req.pop("count", 1)
|
|
requests.extend([BenchRequest(**req)] * count)
|
|
cases.append(BenchCase(case["name"], tuple(requests)))
|
|
return cases
|