diff --git a/.agents/skills/model-microbenchmark/SKILL.md b/.agents/skills/model-microbenchmark/SKILL.md index 7e48325dcb..7f18c2bb87 100644 --- a/.agents/skills/model-microbenchmark/SKILL.md +++ b/.agents/skills/model-microbenchmark/SKILL.md @@ -31,6 +31,9 @@ Requests: `[N*]Q[@K][c][:D]`, comma-separated, per DP rank. | `2048@10240` | prefill after an 8192-token prefix-cache hit | | `2048@10240c` | chunked prefill: the prefix came from earlier chunks | +Verify cases require `--speculative-config` with `num_speculative_tokens` +at least as large as the case's draft count; insufficient capacity is rejected. + Hit vs chunk only differs on models with SWA bounded replay (DSV4.1): a hit also recomputes the last window (128 tokens) of the prefix. diff --git a/tests/benchmarks/test_forward_cli.py b/tests/benchmarks/test_forward_cli.py index 34f8e0c355..d44c6b9466 100644 --- a/tests/benchmarks/test_forward_cli.py +++ b/tests/benchmarks/test_forward_cli.py @@ -9,6 +9,7 @@ import numpy as np import pytest from vllm.benchmarks.forward.batch import build_batch +from vllm.benchmarks.forward.main import run_case from vllm.benchmarks.forward.shapes import jitter_kv_lens, parse_case from vllm.platforms import current_platform @@ -84,6 +85,18 @@ def test_jitter_kv_lens(): jitter_kv_lens(case, 0, 1000, 0) +@pytest.mark.parametrize("capacity", [0, 2]) +def test_bench_forward_rejects_oversized_verify(capacity): + """Reject drafts beyond the runner's buffer before launching CUDA kernels.""" + runner = SimpleNamespace(num_speculative_steps=capacity) + case = parse_case("verify=2*4@64:3") + with pytest.raises( + ValueError, + match=rf"Case 'verify' needs 3 draft tokens.*supports {capacity}", + ): + run_case(runner, case, SimpleNamespace()) + + @pytest.mark.benchmark # TODO(woosuk): Support ROCm and re-enable this test there. @pytest.mark.skipif( diff --git a/vllm/benchmarks/forward/main.py b/vllm/benchmarks/forward/main.py index ff709562a8..d7841c641e 100644 --- a/vllm/benchmarks/forward/main.py +++ b/vllm/benchmarks/forward/main.py @@ -120,6 +120,14 @@ def _make_engine_args(args: argparse.Namespace, cases: list[BenchCase]): def run_case(runner, case: BenchCase, args: argparse.Namespace) -> dict[str, Any]: + required_drafts = max((r.num_draft_tokens for r in case.requests), default=0) + if required_drafts > runner.num_speculative_steps: + raise ValueError( + f"Case {case.name!r} needs {required_drafts} draft tokens per request, " + f"but the runner supports {runner.num_speculative_steps}. " + "Set --speculative-config with a sufficient num_speculative_tokens." + ) + from vllm.benchmarks.forward.batch import build_batch from vllm.benchmarks.forward.capture import ForwardBench from vllm.benchmarks.forward.kv_init import touched_kv_views, zero_kv_blocks