Files
2026-10-14 10:52:26 -07:00

138 lines
4.8 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Exact batch shapes for `vllm bench forward`."""
import json
import zlib
from dataclasses import dataclass, replace
from pathlib import Path
import numpy as np
import regex as re
@dataclass(frozen=True)
class BenchRequest:
query_len: int
# KV length after the step: the prefix already in the KV cache plus the
# query tokens.
kv_len: int
# Spec-decode verify: query_len == 1 + num_draft_tokens.
num_draft_tokens: int = 0
# The prefix was computed by this request's earlier chunks. Otherwise a
# prefill's prefix is a prefix-cache hit, of which models with SWA bounded
# replay (DeepSeek-V4.1) recompute the last window.
chunked: bool = False
def __post_init__(self) -> None:
if (
self.query_len < 1
or self.kv_len < self.query_len
or self.num_draft_tokens < 0
):
raise ValueError(f"Invalid request: {self}")
if self.num_draft_tokens and not self.is_decode:
raise ValueError(
f"Verify needs query_len == 1 + num_draft_tokens and a prefix: {self}"
)
if self.chunked and (self.is_decode or not self.prefix_len):
raise ValueError(f"A chunk needs a prefill with a prefix: {self}")
@property
def prefix_len(self) -> int:
return self.kv_len - self.query_len
@property
def is_decode(self) -> bool:
"""Decode or verify; otherwise a prefill."""
return self.prefix_len > 0 and self.query_len == 1 + self.num_draft_tokens
@property
def prefix_hit(self) -> bool:
return self.prefix_len > 0 and not self.is_decode and not self.chunked
@dataclass(frozen=True)
class BenchCase:
name: str
requests: tuple[BenchRequest, ...]
@property
def num_tokens(self) -> int:
return sum(r.query_len for r in self.requests)
def jitter_kv_lens(
case: BenchCase, jitter: float, max_kv_len: int, seed: int
) -> BenchCase:
"""Draw each request's prefix uniformly within +-jitter of its value.
Query lengths are kept, so a fresh prefill stays one and a non-empty
prefix stays non-empty. KV lengths are capped at `max_kv_len`. The draw
depends only on the seed and the case's name, not on the other cases.
"""
rng = np.random.default_rng([seed, zlib.crc32(case.name.encode())])
requests = []
for req in case.requests:
if req.kv_len > max_kv_len:
raise ValueError(f"{case.name}: KV length {req.kv_len} > {max_kv_len}")
prefix = req.prefix_len
if prefix and jitter:
prefix = max(1, round(prefix * rng.uniform(1 - jitter, 1 + jitter)))
kv_len = min(req.query_len + prefix, max_kv_len)
requests.append(replace(req, kv_len=kv_len))
return replace(case, requests=tuple(requests))
_REQ_RE = re.compile(r"^(?:(\d+)\*)?(\d+)(?:@(\d+))?(c)?(?::(\d+))?$")
def parse_case(text: str) -> BenchCase:
"""Parse `NAME=REQS`, where REQS is a comma list of `[N*]Q[@K][c][:D]`.
Q is the query length, K the KV length including the query (default Q, a
fresh prefill), D the number of draft tokens (verify) and N the number of
such requests. A prefill's prefix is a prefix-cache hit unless `c` marks
it as a chunked prefill. For example, `decode=32*1@8192`,
`verify=16*4@4096:3`, `prefill=8192`, `hit=8192@40960`,
`chunk=8192@40960c` or `mix=8192,32*1@8192`.
"""
name, sep, body = text.partition("=")
if not sep:
name, body = text, text
requests: list[BenchRequest] = []
for item in body.split(","):
match = _REQ_RE.match(item.strip())
if match is None:
raise ValueError(f"Invalid request {item!r} in case {text!r}")
count, query, kv, chunked, drafts = match.groups()
request = BenchRequest(
int(query), int(kv or query), int(drafts or 0), chunked=bool(chunked)
)
requests.extend([request] * int(count or 1))
return BenchCase(name, tuple(requests))
def load_cases(path: str) -> list[BenchCase]:
"""Load cases from a JSON or YAML file.
Format: `{"cases": [{"name": ..., "requests": [{"query_len": ...,
"kv_len": ..., "num_draft_tokens": ..., "chunked": ..., "count": ...}]}]}`.
"""
text = Path(path).read_text()
if path.endswith((".yaml", ".yml")):
import yaml
data = yaml.safe_load(text)
else:
data = json.loads(text)
cases = []
for case in data["cases"]:
requests: list[BenchRequest] = []
for req in case["requests"]:
req = dict(req)
count = req.pop("count", 1)
requests.extend([BenchRequest(**req)] * count)
cases.append(BenchCase(case["name"], tuple(requests)))
return cases