[Bugfix] Validate draft-token capacity in forward benchmark (#61809)
Signed-off-by: znzryb <znzryb34@gmail.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -31,6 +31,9 @@ Requests: `[N*]Q[@K][c][:D]`, comma-separated, per DP rank.
|
||||
| `2048@10240` | prefill after an 8192-token prefix-cache hit |
|
||||
| `2048@10240c` | chunked prefill: the prefix came from earlier chunks |
|
||||
|
||||
Verify cases require `--speculative-config` with `num_speculative_tokens`
|
||||
at least as large as the case's draft count; insufficient capacity is rejected.
|
||||
|
||||
Hit vs chunk only differs on models with SWA bounded replay (DSV4.1): a hit
|
||||
also recomputes the last window (128 tokens) of the prefix.
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.forward.batch import build_batch
|
||||
from vllm.benchmarks.forward.main import run_case
|
||||
from vllm.benchmarks.forward.shapes import jitter_kv_lens, parse_case
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -84,6 +85,18 @@ def test_jitter_kv_lens():
|
||||
jitter_kv_lens(case, 0, 1000, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capacity", [0, 2])
|
||||
def test_bench_forward_rejects_oversized_verify(capacity):
|
||||
"""Reject drafts beyond the runner's buffer before launching CUDA kernels."""
|
||||
runner = SimpleNamespace(num_speculative_steps=capacity)
|
||||
case = parse_case("verify=2*4@64:3")
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=rf"Case 'verify' needs 3 draft tokens.*supports {capacity}",
|
||||
):
|
||||
run_case(runner, case, SimpleNamespace())
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
# TODO(woosuk): Support ROCm and re-enable this test there.
|
||||
@pytest.mark.skipif(
|
||||
|
||||
@@ -120,6 +120,14 @@ def _make_engine_args(args: argparse.Namespace, cases: list[BenchCase]):
|
||||
|
||||
|
||||
def run_case(runner, case: BenchCase, args: argparse.Namespace) -> dict[str, Any]:
|
||||
required_drafts = max((r.num_draft_tokens for r in case.requests), default=0)
|
||||
if required_drafts > runner.num_speculative_steps:
|
||||
raise ValueError(
|
||||
f"Case {case.name!r} needs {required_drafts} draft tokens per request, "
|
||||
f"but the runner supports {runner.num_speculative_steps}. "
|
||||
"Set --speculative-config with a sufficient num_speculative_tokens."
|
||||
)
|
||||
|
||||
from vllm.benchmarks.forward.batch import build_batch
|
||||
from vllm.benchmarks.forward.capture import ForwardBench
|
||||
from vllm.benchmarks.forward.kv_init import touched_kv_views, zero_kv_blocks
|
||||
|
||||
Reference in New Issue
Block a user