From 7f7a32cfec0f1bc5b73c37200b86631523a1ea8f Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Wed, 12 Aug 2026 15:27:48 -0400 Subject: [PATCH] [Spec Decode] DSpark confidence-scheduled verification (#47808) Signed-off-by: Lucas Wilkinson Signed-off-by: Lucas Wilkinson Signed-off-by: Benjamin Chislett Signed-off-by: Lucas Wilkinson Signed-off-by: Nick Hill Co-authored-by: OpenAI Codex Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Benjamin Chislett Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill Co-authored-by: OpenAI Codex --- .buildkite/test_areas/lm_eval.yaml | 20 + docs/features/speculative_decoding/README.md | 2 + .../adaptive_verification.md | 47 ++ ...epSeek-V4-Flash-DSpark-confidence-TP4.yaml | 25 + .../gsm8k/configs/models-spec-decode.txt | 1 + tests/evals/gsm8k/test_gsm8k_correctness.py | 35 ++ .../spec_decode/test_adaptive_verification.py | 170 +++++++ .../test_gpu_model_runner_v2_streaming.py | 1 + tests/v1/worker/test_gpu_batch_ordering.py | 37 +- vllm/config/speculative.py | 11 + vllm/config/vllm.py | 28 + vllm/envs.py | 9 + .../layers/sparse_attn_indexer.py | 1 + vllm/model_executor/models/qwen3_dspark.py | 62 ++- vllm/models/deepseek_v4/amd/rocm.py | 8 +- vllm/models/deepseek_v4/attention.py | 3 + vllm/models/deepseek_v4/nvidia/dspark.py | 29 +- .../deepseek_v4/nvidia/flashinfer_sparse.py | 13 +- vllm/models/deepseek_v4/nvidia/flashmla.py | 20 +- vllm/models/deepseek_v4/sparse_mla.py | 30 +- vllm/models/deepseek_v4/xpu/xpu_sparse.py | 4 +- vllm/sampling_params.py | 14 + vllm/utils/deep_gemm.py | 15 +- vllm/v1/attention/backend.py | 45 +- vllm/v1/attention/backends/flashinfer.py | 6 + vllm/v1/attention/backends/mla/indexer.py | 95 +++- vllm/v1/attention/backends/mla/sparse_swa.py | 28 +- vllm/v1/attention/selector.py | 15 + vllm/v1/worker/gpu/async_utils.py | 94 ++++ vllm/v1/worker/gpu/attn_utils.py | 16 + vllm/v1/worker/gpu/cudagraph_utils.py | 45 +- vllm/v1/worker/gpu/dp_utils.py | 15 +- vllm/v1/worker/gpu/input_batch.py | 73 ++- vllm/v1/worker/gpu/model_runner.py | 150 +++++- vllm/v1/worker/gpu/model_states/default.py | 4 +- .../gpu/spec_decode/adaptive_verification.py | 477 ++++++++++++++++++ .../gpu/spec_decode/dflash/speculator.py | 10 +- .../gpu/spec_decode/dspark/speculator.py | 41 ++ .../gpu/spec_decode/rejection_sampler.py | 24 +- vllm/v1/worker/gpu/structured_outputs.py | 31 +- 40 files changed, 1643 insertions(+), 111 deletions(-) create mode 100644 docs/features/speculative_decoding/adaptive_verification.md create mode 100644 tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml create mode 100644 tests/evals/gsm8k/configs/models-spec-decode.txt create mode 100644 tests/v1/spec_decode/test_adaptive_verification.py create mode 100644 vllm/v1/worker/gpu/spec_decode/adaptive_verification.py diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 5507fa6023..84ebf0d13a 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -101,6 +101,26 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-pcp.txt +- label: LM Eval Spec Decode (4xB200) + key: lm-eval-spec-decode-4xb200 + timeout_in_minutes: 120 + device: b200-k8s + num_devices: 4 + optional: true + source_file_dependencies: + - tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml + - tests/evals/gsm8k/configs/models-spec-decode.txt + - vllm/models/deepseek_v4/ + - vllm/model_executor/models/qwen3_dspark.py + - vllm/v1/spec_decode/metrics.py + - vllm/v1/worker/gpu/spec_decode/adaptive_verification.py + - vllm/v1/worker/gpu/spec_decode/dflash/ + - vllm/v1/worker/gpu/spec_decode/dspark/ + - vllm/v1/worker/gpu/spec_decode/rejection_sampler.py + - vllm/v1/worker/gpu/spec_decode/speculator.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode.txt + - label: LM Eval Large Models EP (2xB200) key: lm-eval-large-models-ep-2xb200 timeout_in_minutes: 60 diff --git a/docs/features/speculative_decoding/README.md b/docs/features/speculative_decoding/README.md index 1ecc9ac719..8f3ac9fa13 100644 --- a/docs/features/speculative_decoding/README.md +++ b/docs/features/speculative_decoding/README.md @@ -18,6 +18,7 @@ vLLM supports a variety of methods of speculative decoding. Model-based methods - [Hidden State Extraction](extract_hidden_states.md) - [Custom Proposer Backend (Experimental)](#custom-proposer-backend-experimental) - [Dynamic Speculative Decoding](dynamic_speculative_decoding.md) +- [Adaptive Verification](adaptive_verification.md) ## Method Selection at a Glance @@ -35,6 +36,7 @@ depend on your model family, traffic pattern, hardware, and sampling settings. | Suffix decoding | Low to medium gain | Medium gain | No extra draft model; dynamic speculation depth. | | Custom Proposer | Varies | Varies | Bring your own proposer class (experimental). | | Dynamic Speculative Decoding | High gain | Higher than base SD method | Useful for RL or workload with fluctuating QPS | +| Adaptive Verification | High gain | Higher than base SD method | Sizes verification per request from drafter confidence; currently DSpark only. | For reproducible measurements in your environment, use [`examples/features/speculative_decoding/spec_decode_offline.py`](../../../examples/features/speculative_decoding/spec_decode_offline.py) diff --git a/docs/features/speculative_decoding/adaptive_verification.md b/docs/features/speculative_decoding/adaptive_verification.md new file mode 100644 index 0000000000..58c9d57cee --- /dev/null +++ b/docs/features/speculative_decoding/adaptive_verification.md @@ -0,0 +1,47 @@ +# Adaptive Verification + +Speculative decoding buys fewer decode steps with more compute. At batch size 1 that is a good trade: the GPU is memory-bound with spare compute, so the extra draft tokens are close to free. At batch size 256 it is a much more delicate one. Draft tokens now compete with real tokens for the same compute, and every rejected token is compute wasted; with enough of them, throughput drops. + +That matters because per-position acceptance decays fast. While the GPU is memory-bound that slot is effectively free and worth the gamble; once it saturates the gamble has a real throughput cost. The crossover moves with load and with workload-dependent acceptance rates, so no static `num_speculative_tokens` is right across concurrencies. + +Adaptive verification decides per step how much of the draft to verify instead. Every (request, position) draft slot is scored by its *survival probability*, the running product of that request's per-position confidences, and the highest-scoring slots are admitted until a global budget is spent. Slots compete across requests: position 5 of a confident request can outrank position 1 of a doubtful one, so one request keeps its full block while another could be trimmed after a token or two. + +The budget itself comes from a cost model profiled at startup. vLLM measures what a step costs at each shape, then picks the token count that maximizes expected accepted tokens per second. + +The practical effect is that one configuration holds up across the whole load range, which removes most of the need to tune `num_speculative_tokens` per deployment. + +## Support + +Adaptive verification needs per-position acceptance estimates, so today it is only supported for DSpark with a **confidence head**. + +## Usage + +It is off by default. Enable it in the speculative config: + +```bash +vllm serve deepseek-ai/DeepSeek-V4-Flash-DSpark \ + --tokenizer-mode deepseek_v4 --trust-remote-code \ + --speculative-config '{ + "method": "dspark", + "model": "deepseek-ai/DeepSeek-V4-Flash-DSpark", + "num_speculative_tokens": 7, + "draft_sample_method": "probabilistic", + "enable_adaptive_verification": true + }' +``` + +Set `enable_adaptive_verification: false` to verify the full block for every request. + +## Requirements and limitations + +- The attention backend must tolerate device-decided query lengths, since the CPU lengths only bound them from above. Backends that plan off the CPU lengths are excluded by the attention selector, and rejected at startup for models that hard-wire their backend. +- Full cudagraphs are required: step costs are profiled from captured graphs, so `--enforce-eager` is rejected at startup. +- Not supported with LoRA (the per-token LoRA mapping is built from CPU-side boundaries), pipeline parallelism (cost curves and confidences exist only on the last rank), or output logprobs (to be fixed). + +## Tuning the cost profile + +Step costs are profiled against a synthetic KV context, 8192 tokens by default. Deployments serving much longer contexts may want to raise it so the profiled step reads a more realistic amount of cache (this matters a bit less for sparse attention models like DeepSeek-v4 since the cheap indexer is the main cost that scales with context length). + +```bash +export VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN=131072 +``` diff --git a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml new file mode 100644 index 0000000000..61fbaf33e2 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml @@ -0,0 +1,25 @@ +model_name: "deepseek-ai/DeepSeek-V4-Flash-DSpark" +accuracy_threshold: 0.92 +# lower acceptance length since trimming drafts at higher concurrency reduces +# acceptance length, this is very loose lower bound for smoke testing +min_acceptance_length: 1.2 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1800 +server_args: >- + --tokenizer-mode deepseek_v4 + --trust-remote-code + --max-model-len 8192 + --tensor-parallel-size 4 + --enable-expert-parallel + --block-size 256 + --gpu-memory-utilization 0.5 + --kv-cache-dtype fp8 + --attention_config.use_fp4_indexer_cache=True + --max-num-batched-tokens 16384 + --max-num-seqs 128 + --speculative-config '{"method":"dspark", + "model":"deepseek-ai/DeepSeek-V4-Flash-DSpark", + "attention_backend":"FLASH_ATTN","num_speculative_tokens":7, + "draft_sample_method":"probabilistic", + "enable_adaptive_verification":true}' diff --git a/tests/evals/gsm8k/configs/models-spec-decode.txt b/tests/evals/gsm8k/configs/models-spec-decode.txt new file mode 100644 index 0000000000..05f620435c --- /dev/null +++ b/tests/evals/gsm8k/configs/models-spec-decode.txt @@ -0,0 +1 @@ +DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index e1ae6ec542..5a237cf947 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -12,6 +12,7 @@ pytest -s -v tests/evals/gsm8k/test_gsm8k_correctness.py \ import shlex import pytest +import requests import yaml from tests.utils import RemoteOpenAIServer @@ -65,6 +66,28 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict: return results +def get_acceptance_length(server_url: str) -> float: + """Mean tokens emitted per verification step, from the server's counters. + + 1.0 means every draft was rejected (speculation bought nothing); the + theoretical maximum is 1 + num_speculative_tokens. + """ + response = requests.get(f"{server_url.rstrip('/').removesuffix('/v1')}/metrics") + response.raise_for_status() + counters: dict[str, float] = {} + for line in response.text.splitlines(): + if line.startswith("vllm:spec_decode_num_"): + name, _, value = line.partition(" ") + counters[name.split("{")[0]] = float(value) + + num_drafts = counters.get("vllm:spec_decode_num_drafts_total", 0.0) + num_accepted = counters.get("vllm:spec_decode_num_accepted_tokens_total", 0.0) + assert num_drafts > 0, ( + "no drafts recorded; speculative decoding did not run for this config" + ) + return 1.0 + num_accepted / num_drafts + + def test_gsm8k_correctness(config_filename): """Test GSM8K correctness for a given model configuration.""" eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8")) @@ -168,4 +191,16 @@ def test_gsm8k_correctness(config_filename): f"{expected_metric:.4f} - {tol:.4f} = {expected_metric - tol:.4f}" ) + # Speculative configs additionally assert that drafts are actually + # landing: accuracy alone passes even when every draft is rejected. + min_acceptance_length = eval_config.get("min_acceptance_length") + if min_acceptance_length is not None: + acceptance_length = get_acceptance_length(server_url) + print(f" Mean acceptance length: {acceptance_length:.3f}") + print(f" Minimum acceptance length: {min_acceptance_length:.3f}") + assert acceptance_length >= min_acceptance_length, ( + f"Acceptance length too low: {acceptance_length:.3f} < " + f"{min_acceptance_length:.3f}" + ) + print(f"✅ GSM8K test passed for {eval_config['model_name']}") diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py new file mode 100644 index 0000000000..5fc88de998 --- /dev/null +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import numpy as np + +from vllm.v1.worker.gpu.async_utils import StepTimingSample +from vllm.v1.worker.gpu.spec_decode.adaptive_verification import ( + AdaptiveVerificationManager, +) + + +def make_manager( + confidences: np.ndarray, verify_cost_ms: np.ndarray +) -> AdaptiveVerificationManager: + num_reqs, num_steps = confidences.shape + manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) + manager.num_speculative_steps = num_steps + manager._stale_confidences = [SimpleNamespace(np=confidences)] + manager._stale_idx = 0 + manager.req_states = SimpleNamespace( + req_id_to_index={"low": 0, "high": 1}, + num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32), + prefill_len=SimpleNamespace(np=np.ones(num_reqs, dtype=np.int32)), + ) + manager.cost_tables = (np.zeros(num_reqs + 1), verify_cost_ms) + manager._max_total_logits = 1 << 30 + manager.num_bonus_tokens = 1 + return manager + + +def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves(): + # Verification is cheap up to two extra tokens, then jumps 100x; only the + # highest-confidence draft is worth the cheap slot. + manager = make_manager( + np.array([[0.1, 0.1], [0.9, 0.9]], dtype=np.float32), + np.array([1.0, 1.0, 1.0, 1.0, 100.0, 100.0, 100.0]), + ) + + manager.get_num_tokens( + {"low": 3, "high": 3}, + {"low": [1, 2], "high": [3, 4]}, + ) + valid_drafts, num_non_draft_tokens, draft_budget = manager._batch_budget + + assert draft_budget == 1 + assert valid_drafts == {"low": 2, "high": 2} + assert num_non_draft_tokens == {"low": 1, "high": 1} + + +def test_profiled_batches_seed_cost_curves_via_consumer(): + manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) + manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64) + manager.num_speculative_steps = 7 + manager.num_bonus_tokens = 1 + curves: dict[str, list[tuple[int, float]]] = {} + manager.set_cost_curves = lambda draft, verify: curves.update( + draft=draft, verify=verify + ) + + timings = [ + StepTimingSample( + forward_ms=float(batch["num_tokens"]), + drafter_ms=1.0, + num_target_tokens=batch["num_tokens"], + num_reqs=batch["num_tokens"] // 8, + # Only the captured sizes replay a graph; the tail sizes run eager. + full_cudagraph=batch["num_tokens"] <= 1024, + ) + for batch in manager.batches_to_profile([8, 1024]) + ] + manager.set_initial_cost_curves(timings) + + # Tail beyond the last capture size: 1.5x then doubling to the max. + assert curves["verify"] == [ + (8, 8.0), + (1024, 1024.0), + (1536, 1536.0), + (2048, 2048.0), + (4096, 4096.0), + ] + # Eager batches must not contribute to the draft curve: keyed by request + # count they would land inside the captured range and, once made monotonic, + # smear that eager cost across every larger request count. + assert curves["draft"] == [(1, 1.0), (128, 1.0)] + + +def test_compact_batch_preserves_totals_and_bounds(): + # The CPU placeholder layout must keep the batch total equal to the GPU + # total and every verification row within decode_query_len, or downstream + # CPU metadata desyncs from the reallocated GPU boundaries. + manager = make_manager( + np.array([[0.9, 0.9], [0.9, 0.9], [1.0, 1.0]], dtype=np.float32), + np.array([1.0] * 44 + [100.0] * 3), + ) + manager.req_states.req_id_to_index["prefill"] = 2 + manager.req_states.num_computed_tokens_np = np.zeros(3, dtype=np.int32) + manager.req_states.prefill_len.np = np.array([0, 0, 60], dtype=np.int32) + num_tokens = manager.get_num_tokens( + {"low": 3, "high": 3, "prefill": 40}, + {"low": [1, 2], "high": [3, 4]}, + ) + scheduled = np.array([3, 3, 40], dtype=np.int32) + drafts = np.array([2, 2, 0], dtype=np.int32) + cu_num_logits_np = np.array([0, 3, 6, 7], dtype=np.int32) + compacted, _ = manager.compact_batch(drafts, scheduled, cu_num_logits_np) + + assert int(compacted.sum()) == num_tokens + num_steps = manager.num_speculative_steps + assert (compacted[:2] <= 1 + num_steps).all() + assert compacted[2] == 40 + + +def test_budget_caps_at_one_rejection_sampler_chunk(): + # The chunked verification path cannot address the compacted logits + # layout, so the budget must keep total logits within a single chunk. + manager = make_manager( + np.array([[0.9, 0.9], [0.9, 0.9]], dtype=np.float32), + np.ones(7), + ) + manager._max_total_logits = 3 # 2 bonus logits + at most 1 draft + manager.get_num_tokens( + {"low": 3, "high": 3}, + {"low": [1, 2], "high": [3, 4]}, + ) + _, _, draft_budget = manager._batch_budget + assert draft_budget <= 1 + + +def test_zero_budget_rebuilds_cpu_cu_num_logits(): + # When one bonus row per request already overflows a verification chunk, the + # budget clamps to zero but the batch still needs chunking. Every capacity is + # zeroed on device, so the CPU can name that layout exactly -- and must, since + # _iter_request_chunks slices the compacted logits with these offsets. + # + # The third request is a chunked prefill (no drafts, still mid-prompt). The + # runner gives *every* request num_bonus_tokens logits rows regardless + # (num_logits = num_draft_tokens_per_req + num_bonus_tokens), so the rebuilt + # offsets stay uniform rather than skipping non-verification rows. + manager = make_manager( + np.array([[0.9, 0.9], [0.9, 0.9], [1.0, 1.0]], dtype=np.float32), + np.ones(64), + ) + manager.req_states.req_id_to_index["prefill"] = 2 + manager.req_states.num_computed_tokens_np = np.zeros(3, dtype=np.int32) + manager.req_states.prefill_len.np = np.array([0, 0, 60], dtype=np.int32) + manager._max_total_logits = 2 # < 3 requests * 1 bonus token + + manager.get_num_tokens( + {"low": 3, "high": 3, "prefill": 40}, + {"low": [1, 2], "high": [3, 4]}, + ) + _, _, draft_budget = manager._batch_budget + assert draft_budget == 0 + + scheduled = np.array([3, 3, 40], dtype=np.int32) + drafts = np.array([2, 2, 0], dtype=np.int32) + scheduled_cu_num_logits = np.array([0, 3, 6, 7], dtype=np.int32) + compacted, cu_num_logits_np = manager.compact_batch( + drafts, scheduled, scheduled_cu_num_logits + ) + + # One bonus row per request, matching cumsum(capacities + num_bonus_tokens) + # with every capacity zeroed -- the prefill row included. + expected = np.arange(4, dtype=np.int32) * manager.num_bonus_tokens + assert np.array_equal(cu_num_logits_np, expected) + assert cu_num_logits_np.dtype == scheduled_cu_num_logits.dtype + # The prefill keeps its scheduled tokens; only drafts are dropped. + assert np.array_equal(compacted, np.array([1, 1, 40], dtype=np.int32)) diff --git a/tests/v1/streaming_input/test_gpu_model_runner_v2_streaming.py b/tests/v1/streaming_input/test_gpu_model_runner_v2_streaming.py index 489284d667..31c1a6f484 100644 --- a/tests/v1/streaming_input/test_gpu_model_runner_v2_streaming.py +++ b/tests/v1/streaming_input/test_gpu_model_runner_v2_streaming.py @@ -42,6 +42,7 @@ def mock_model_runner_with_req_states(): runner.prompt_logprobs_worker = None runner.is_last_pp_rank = False runner.pooling_runner = None + runner.adaptive_verification = None # Mock staged writes — they use Triton kernels that require GPU runner.req_states.apply_staged_writes = Mock() diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index d0cd402ce7..5037dcc7a9 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -55,6 +55,7 @@ def _make_runner( num_computed = np.array([s[0] for s in req_states.values()], dtype=np.int32) runner: Any = GPUModelRunner.__new__(GPUModelRunner) runner.decode_query_len = decode_query_len + runner.adaptive_verification = None runner.req_states = SimpleNamespace( req_id_to_index={req_id: i for i, req_id in enumerate(req_states)}, # The runner keeps this as min(num_computed_tokens, prefill_len). @@ -73,6 +74,7 @@ def _uniform_token_count( scheduler_output = SimpleNamespace( num_scheduled_tokens={req_id: query_len for req_id in req_states}, total_num_scheduled_tokens=query_len * len(req_states), + scheduled_spec_decode_tokens={}, ) runner = _make_runner(req_states, decode_query_len=query_len) _, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, dummy_run) @@ -85,6 +87,33 @@ def test_spec_decode_batch_is_uniform_decode(): assert _uniform_token_count(decodes, 8) == 8 +def test_adaptive_verification_sizes_only_batches_with_drafts(): + decodes = {"d0": (16, 16), "d1": (16, 16)} + runner = _make_runner(decodes, decode_query_len=8) + manager = SimpleNamespace( + get_num_tokens=lambda _num_tokens_per_req, _draft_tokens: 12 + ) + runner.adaptive_verification = manager + scheduler_output = SimpleNamespace( + num_scheduled_tokens={req_id: 8 for req_id in decodes}, + total_num_scheduled_tokens=16, + scheduled_spec_decode_tokens={}, + ) + + state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False) + assert state is not None + assert state.num_tokens == 16 + assert uniform_tok_count == 8 + + scheduler_output.scheduled_spec_decode_tokens = { + req_id: [1, 2] for req_id in decodes + } + state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False) + assert state is not None + assert state.num_tokens == 12 + assert uniform_tok_count is None + + def test_prompt_chunk_of_decode_query_len_is_not_uniform_decode(): # Two prompt chunks whose query length coincides with the K+1 spec-decode # query length must reject the batch (issue #49918). @@ -116,14 +145,14 @@ def test_dummy_batches_stay_uniform_decode(): def test_sort_batch_req_ids_no_spec(): # decode_query_len == 1: plain ascending order (decodes first). num_tokens_per_req = {"p1": 100, "d1": 1, "p2": 7, "d2": 1} - assert sort_batch_req_ids(num_tokens_per_req, 1) == ["d1", "d2", "p2", "p1"] + assert sort_batch_req_ids(num_tokens_per_req, {}, 1) == ["d1", "d2", "p2", "p1"] def test_sort_batch_req_ids_spec_decode(): # decode_query_len == 2 (MTP k=1): uniform decodes lead, then the 1-token # chunked-prefill tail, then longer prefills. num_tokens_per_req = {"tail": 1, "d1": 2, "p1": 100, "d2": 2} - assert sort_batch_req_ids(num_tokens_per_req, 2) == ["d1", "d2", "tail", "p1"] + assert sort_batch_req_ids(num_tokens_per_req, {}, 2) == ["d1", "d2", "tail", "p1"] def test_spec_decodes_lead_short_prefill_tail(): @@ -131,7 +160,7 @@ def test_spec_decodes_lead_short_prefill_tail(): # uniform 2-token decodes as decodes even when a 1-token prefill tail is # in the batch (indexer-style: require_uniform, threshold=1+k). num_tokens_per_req = {"tail": 1, **{f"d{i}": 2 for i in range(8)}} - req_ids = sort_batch_req_ids(num_tokens_per_req, 2) + req_ids = sort_batch_req_ids(num_tokens_per_req, {}, 2) query_lens = [num_tokens_per_req[r] for r in req_ids] assert query_lens == [2] * 8 + [1] @@ -155,6 +184,7 @@ def test_uniform_decode_uses_state_index_not_batch_position(): """ runner: Any = GPUModelRunner.__new__(GPUModelRunner) runner.decode_query_len = 8 + runner.adaptive_verification = None # State arrays in state-index order: a prefilling request, then two decodes. runner.req_states = SimpleNamespace( req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2}, @@ -164,6 +194,7 @@ def test_uniform_decode_uses_state_index_not_batch_position(): scheduler_output = SimpleNamespace( num_scheduled_tokens={"decode_a": 8, "decode_b": 8}, total_num_scheduled_tokens=16, + scheduled_spec_decode_tokens={}, ) state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False) diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 177f5a262b..be70da78b3 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -238,6 +238,10 @@ class SpeculativeConfig: synthetic_acceptance_rates. Only valid when rejection_sample_method is 'synthetic'. Mutually exclusive with synthetic_acceptance_rates.""" + enable_adaptive_verification: bool = False + """Whether to adaptively size the draft-verification budget from per-request + confidence. Currently only supported for method="dspark".""" + @staticmethod def _acceptance_length_to_rates(length: float, n: int) -> list[float]: """Mean acceptance length to unconditional per-position rates, using @@ -1014,6 +1018,9 @@ class SpeculativeConfig: self.draft_model_config.hf_config.architectures = [ "DSparkDraftModel" ] + self.draft_model_config.quantization = ( + self.target_model_config.quantization + ) self.update_arch_() elif ( self.method == "dspark" @@ -1159,6 +1166,10 @@ class SpeculativeConfig: self.target_parallel_config, self.draft_tensor_parallel_size ) ) + + if self.method != "dspark" and self.enable_adaptive_verification: + raise ValueError("Adaptive verification only supported with DSpark") + return self def _validate_suffix_decoding(self): diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 2bbfd57330..67ac2c2b74 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2403,6 +2403,34 @@ class VllmConfig: ): unsupported.append("EAGLE3 with pipeline parallelism") + if ( + speculative_config.enable_adaptive_verification + and self.lora_config is not None + ): + # The per-token LoRA mapping is built from CPU placeholder boundaries, + # while the trimmed batch's true boundaries are decided on the GPU. + unsupported.append("adaptive verification with LoRA") + + if ( + speculative_config.enable_adaptive_verification + and self.compilation_config.cudagraph_mode == CUDAGraphMode.NONE + ): + # The draft budget divides by step costs profiled from captured + # cudagraphs; eager execution captures none. + unsupported.append( + "adaptive verification with enforce_eager/cudagraph_mode=none" + ) + + if ( + speculative_config.enable_adaptive_verification + and self.parallel_config.pipeline_parallel_size > 1 + ): + # Cost curves and confidences currently only exist on the last PP rank; + # earlier ranks would diverge on the trimmed batch shape. + # TODO: we should be able to support adaptive verification with PP by + # broadcasting the cost curves and confidences to all ranks. + unsupported.append("adaptive verification with pipeline parallelism") + if self.parallel_config.enable_dbo: unsupported.append("dual batch overlap") diff --git a/vllm/envs.py b/vllm/envs.py index 819c6d30cd..f8401f53c3 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -57,6 +57,7 @@ if TYPE_CHECKING: VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache") VLLM_XLA_CHECK_RECOMPILATION: bool = False VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: int = 512 + VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN: int = 8192 VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE: Literal["auto", "nccl", "shm"] = "auto" VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = False VLLM_USE_RAY_WRAPPED_PP_COMM: bool = True @@ -1067,6 +1068,14 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_SPARSE_INDEXER_MAX_LOGITS_MB": lambda: int( os.getenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "512") ), + # KV context length each adaptive-verification profiling request pretends to + # carry, so the profiled step reads a realistic amount of cache. + # Raise it for long-context deployments, where step cost is dominated by + # attention over a much larger KV cache than the default assumes. + # Clamped to max_model_len - query_len. Default: 8192 tokens + "VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN": lambda: int( + os.getenv("VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN", "8192") + ), # If set, the OpenAI API server will stay alive even after the underlying # AsyncLLMEngine errors and stops serving requests "VLLM_KEEP_ALIVE_ON_ENGINE_DEATH": lambda: bool( diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index 4081041198..d94b377dbe 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -609,6 +609,7 @@ def sparse_attn_indexer( decode_metadata.schedule_metadata, max_model_len=max_model_len, clean_logits=False, + indices=decode_metadata.indices, ) num_rows = logits.shape[0] topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens] diff --git a/vllm/model_executor/models/qwen3_dspark.py b/vllm/model_executor/models/qwen3_dspark.py index 03ad01e1d9..d15aa45461 100644 --- a/vllm/model_executor/models/qwen3_dspark.py +++ b/vllm/model_executor/models/qwen3_dspark.py @@ -10,8 +10,9 @@ The parallel backbone is a standard Qwen3 decoder stack reused from the DFlash Qwen3 draft (see qwen3_dflash.py). DSpark adds: * ``markov_head``: low-rank V x r / r x V transition bias added to the base logits, sampled left-to-right by the speculator (the sequential stage). + * ``confidence_head``: per-position acceptance-probability estimate. -DSparkMarkovHead is shared with the DSV4-style DSpark model. +DSparkMarkovHead and DSparkConfidenceHead are shared with the DSV4-style DSpark model. """ from collections.abc import Iterable @@ -21,6 +22,7 @@ import torch.nn as nn from vllm.config import VllmConfig from vllm.logger import init_logger +from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -102,8 +104,36 @@ class DSparkMarkovHead(nn.Module): return logits.scatter_(1, index, corrected.squeeze(-1)) +class DSparkConfidenceHead(nn.Module): + """DSpark acceptance-confidence head.""" + + def __init__( + self, + input_dim: int, + prefix: str, + bias: bool = False, + with_markov: bool = True, + ) -> None: + super().__init__() + self.with_markov = with_markov + self.proj = ReplicatedLinear( + input_dim, + 1, + bias=bias, + return_bias=False, + params_dtype=torch.float32, + prefix=maybe_prefix(prefix, "proj"), + ) + + def forward(self, hidden: torch.Tensor, markov_embed: torch.Tensor) -> torch.Tensor: + x = ( + torch.cat([hidden, markov_embed], dim=-1) if self.with_markov else hidden + ).float() + return self.proj(x).squeeze(-1) + + class Qwen3DSparkModel(DFlashQwen3Model): - """DFlash Qwen3 backbone + DSpark Markov head.""" + """DFlash Qwen3 backbone + DSpark Markov / confidence heads.""" def __init__( self, @@ -126,6 +156,18 @@ class Qwen3DSparkModel(DFlashQwen3Model): prefix=maybe_prefix(prefix, "markov_head"), quant_config=self.quant_config, ) + self.confidence_head: DSparkConfidenceHead | None = None + if getattr(config, "enable_confidence_head", False): + with_markov = getattr(config, "confidence_head_with_markov", False) + input_dim = config.hidden_size + if with_markov: + input_dim += config.markov_rank + self.confidence_head = DSparkConfidenceHead( + input_dim, + prefix=maybe_prefix(prefix, "confidence_head"), + bias=True, + with_markov=with_markov, + ) class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM): @@ -197,11 +239,19 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM): self.logits_processor.scale, ) + def compute_confidence( + self, head_hidden: torch.Tensor, markov_embed: torch.Tensor + ) -> torch.Tensor: + """Per-position acceptance probability for each drafted token.""" + assert self.model.confidence_head is not None + return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed)) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): model_weights = {} includes_embed_tokens = False includes_lm_head = False includes_draft_id_mapping = False + includes_confidence_head = False for name, loaded_weight in weights: # t2d is training-only; the draft remaps via d2t at sampling time. if "t2d" in name: @@ -215,22 +265,26 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM): includes_embed_tokens = True if "lm_head" in name: includes_lm_head = True + if "confidence_head" in name: + includes_confidence_head = True model_weights[name] = loaded_weight # Sets has_own_embed_tokens / has_own_lm_head so load_dspark_model # knows whether to keep these or alias the target's. process_eagle_weight(self, name) # mask_embedding is an unused placeholder param; DSpark masks via the vocab row. - # confidence_head is not wired into inference yet; skip its weights. # embed_tokens / lm_head are optional; when omitted they are shared from # the target by load_dspark_model, so skip the unloaded params here. - skip_substrs = ["mask_embedding", "confidence_head"] + skip_substrs = ["mask_embedding"] if not includes_embed_tokens: skip_substrs.append("embed_tokens") if not includes_lm_head: skip_substrs.append("lm_head") if not includes_draft_id_mapping: skip_substrs.append("draft_id_to_target_id") + if self.model.confidence_head is None or not includes_confidence_head: + self.model.confidence_head = None + skip_substrs.append("confidence_head") loader = AutoWeightsLoader(self, skip_substrs=skip_substrs) loader.load_weights(model_weights.items()) self.model._build_fused_kv_buffers() diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index 9b38409d1f..23223acd2d 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -14,9 +14,9 @@ from vllm.forward_context import get_forward_context from vllm.models.deepseek_v4.attention import DeepseekV4Attention from vllm.models.deepseek_v4.common.ops import dequantize_and_gather_k_cache from vllm.models.deepseek_v4.sparse_mla import ( - DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, - DeepseekV4FlashMLAMetadataBuilder, + DeepseekV4SparseMLABackend, + DeepseekV4SparseMLAMetadataBuilder, ) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton @@ -314,7 +314,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata): decode_swa_ragged_indptr: torch.Tensor | None = None -class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4FlashMLAMetadataBuilder): +class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.c128a_decode_topk_ragged_indices_buffer: torch.Tensor | None = None @@ -435,7 +435,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild ) -class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4FlashMLABackend): +class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4SparseMLABackend): @staticmethod def get_name() -> str: return "ROCM_FLASHMLA_SPARSE_DSV4" diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index 286ea11bf3..05576f0752 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -134,6 +134,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): # Provided by the platform subclass. backend_cls: ClassVar[type[AttentionBackend]] + # Backend for the SWA cache layer; None uses the default SWA backend. + swa_backend_cls: ClassVar[type[AttentionBackend] | None] = None # KV-cache per-token block format (both layouts are paged). True (default) # = fp8_ds_mla (UE8M0 block-scaled fp8 packed as uint8); False = plain # bf16 / per-tensor fp8 KV row. Backends can override the instance hook when @@ -322,6 +324,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): dtype=self.kv_cache_torch_dtype, prefix=f"{prefix}.swa_cache", cache_config=cache_config, + backend_cls=self.swa_backend_cls, ) # Register with compilation context for metadata lookup. diff --git a/vllm/models/deepseek_v4/nvidia/dspark.py b/vllm/models/deepseek_v4/nvidia/dspark.py index 5478c423a2..b702d7ec9c 100644 --- a/vllm/models/deepseek_v4/nvidia/dspark.py +++ b/vllm/models/deepseek_v4/nvidia/dspark.py @@ -39,6 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.qwen3_dspark import ( + DSparkConfidenceHead, DSparkMarkovHead, ) from vllm.model_executor.models.utils import maybe_prefix @@ -113,7 +114,7 @@ class DSparkDeepseekV4Model(nn.Module): ] ) - # Heads: final norm + hc_head, and the Markov head + # Heads: final norm + hc_head, and the Markov + confidence heads # Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) hc_dim = self.hc_mult * config.hidden_size @@ -136,6 +137,12 @@ class DSparkDeepseekV4Model(nn.Module): config.dspark_markov_rank, prefix=maybe_prefix(prefix, "markov_head"), ) + self.confidence_head: DSparkConfidenceHead | None = None + if getattr(config, "enable_confidence_head", True): + self.confidence_head = DSparkConfidenceHead( + config.hidden_size + config.dspark_markov_rank, + prefix=maybe_prefix(prefix, "confidence_head"), + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) @@ -369,6 +376,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor: return self.model.markov_head.bias(markov_embed, self.logits_processor) + def compute_confidence( + self, head_hidden: torch.Tensor, markov_embed: torch.Tensor + ) -> torch.Tensor: + """Per-position acceptance probability for each drafted token.""" + assert self.model.confidence_head is not None + return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed)) + # --- Weight loading ---------------------------------------------------- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: @@ -407,6 +421,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() + loaded_confidence_head = False tp_size = get_tensor_model_parallel_world_size() tp_rank = get_tensor_model_parallel_rank() @@ -419,6 +434,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): if mapped is None: continue name = mapped + if "confidence_head." in name: + loaded_confidence_head = True # ``.scale`` -> per-method scale suffix. if name.endswith(".scale"): @@ -461,8 +478,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): continue # Stacked rules only apply to decoder-layer weights. Head-stack params - # (main_proj/norm/hc_head/markov_head) load directly — otherwise e.g. - # "markov_w1" would collide with the "w1" shard rule. + # (main_proj/norm/hc_head/markov_head/confidence_head) load directly — + # otherwise e.g. "markov_w1" would collide with the "w1" shard rule. is_layer_param = name.startswith("model.layers.") for param_name, weight_name, stacked_shard_id in stacked_params_mapping: if not is_layer_param or weight_name not in name: @@ -488,6 +505,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): loaded_params.add(name) self._finalize_moe() + if self.model.confidence_head is not None and not loaded_confidence_head: + self.model.confidence_head = None logger.info_once("DSpark draft model loaded: %d params", len(loaded_params)) return loaded_params @@ -505,8 +524,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): return None stage = int(m.group(1)) rest = m.group(2) - # The confidence head is not wired into inference yet; drop its weights. - if rest.startswith("confidence_head."): + if rest.startswith("confidence_head.") and self.model.confidence_head is None: return None # Head-stack params live at model level (mtp.last), context combiner at # model level (mtp.0); everything else is a per-layer decoder block. @@ -516,6 +534,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): "hc_head_base", "hc_head_scale", "markov_head.", + "confidence_head.", ) if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith( head_prefixes diff --git a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py index d8742e6103..27031e02c5 100644 --- a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py @@ -18,8 +18,8 @@ from vllm.models.deepseek_v4.nvidia.ops.o_proj import ( deep_gemm_fp8_o_proj, ) from vllm.models.deepseek_v4.sparse_mla import ( - DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, + DeepseekV4SparseMLABackend, ) from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability @@ -74,11 +74,10 @@ def _pad_to_supported_q_heads(num_heads: int) -> int: ) -class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend): +class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend): """FlashInfer backend using the DSv4 sparse metadata/cache layout. - Inheriting from the FlashMLA V4 backend reuses its - ``DeepseekV4FlashMLAMetadata`` builder. + Inherits the base and backend reuses its``DeepseekV4SparseMLAMetadataBuilder`` """ supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] @@ -159,7 +158,7 @@ class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend): ) -> tuple[int, ...]: device_capability = current_platform.get_device_capability() if device_capability is not None and device_capability.major == 12: - return DeepseekV4FlashMLABackend.get_kv_cache_shape( + return DeepseekV4SparseMLABackend.get_kv_cache_shape( num_blocks, block_size, num_kv_heads, @@ -489,8 +488,6 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): # uniform-q batches, and this avoids flattening mixed batches into one call. if num_decode_tokens > 0: decode_cu = query_start_loc[: num_decodes + 1] - decode_cu_cpu = query_start_loc_cpu[: num_decodes + 1] - decode_lens_cpu = decode_cu_cpu[1:] - decode_cu_cpu[:-1] flashinfer_trtllm_batch_decode_sparse_mla_dsv4( query=query[:num_decode_tokens], swa_kv_cache=swa_k_cache, @@ -504,7 +501,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): bmm2_scale=bmm2_scale, sinks=self.attn_sink, cum_seq_lens_q=decode_cu, - max_q_len=int(decode_lens_cpu.max().item()), + max_q_len=swa_metadata.max_decode_query_len, ) if num_prefill_tokens > 0: diff --git a/vllm/models/deepseek_v4/nvidia/flashmla.py b/vllm/models/deepseek_v4/nvidia/flashmla.py index f0b00163d1..a6a24ae0cc 100644 --- a/vllm/models/deepseek_v4/nvidia/flashmla.py +++ b/vllm/models/deepseek_v4/nvidia/flashmla.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, ClassVar, cast import torch @@ -21,6 +21,11 @@ from vllm.models.deepseek_v4.sparse_mla import ( DeepseekV4FlashMLAMetadata, ) from vllm.utils.math_utils import round_up +from vllm.v1.attention.backend import AttentionCGSupport +from vllm.v1.attention.backends.mla.sparse_swa import ( + DeepseekSparseSWABackend, + DeepseekSparseSWAMetadataBuilder, +) from vllm.v1.attention.ops.flashmla import ( flash_mla_sparse_fwd, flash_mla_with_kvcache, @@ -31,10 +36,23 @@ if TYPE_CHECKING: from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata +class DeepseekSparseSWAFlashMLAMetadataBuilder(DeepseekSparseSWAMetadataBuilder): + """SWA metadata for the FlashMLA decode path, which allows varlen decode.""" + + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS + + +class DeepseekSparseSWAFlashMLABackend(DeepseekSparseSWABackend): + @staticmethod + def get_builder_cls() -> type[DeepseekSparseSWAFlashMLAMetadataBuilder]: + return DeepseekSparseSWAFlashMLAMetadataBuilder + + class DeepseekV4FlashMLAAttention(DeepseekV4Attention): """FlashMLA sparse MLA attention layer for DeepSeek V4 (CUDA).""" backend_cls = DeepseekV4FlashMLABackend + swa_backend_cls = DeepseekSparseSWAFlashMLABackend def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/vllm/models/deepseek_v4/sparse_mla.py b/vllm/models/deepseek_v4/sparse_mla.py index 4ac27bd59a..db8ab96e90 100644 --- a/vllm/models/deepseek_v4/sparse_mla.py +++ b/vllm/models/deepseek_v4/sparse_mla.py @@ -32,8 +32,8 @@ from vllm.v1.kv_cache_interface import AttentionSpec _C128A_TOPK_ALIGNMENT = 128 -class DeepseekV4FlashMLABackend(AttentionBackend): - """DeepSeek-V4 sparse-MLA backend. +class DeepseekV4SparseMLABackend(AttentionBackend): + """DeepSeek-V4 sparse-MLA backend base. Subclasses ``AttentionBackend`` directly (not the V3.2 ``FlashMLASparseBackend``): DeepSeek-V4 runs its own attention layer @@ -54,12 +54,8 @@ class DeepseekV4FlashMLABackend(AttentionBackend): return [256] @staticmethod - def get_name() -> str: - return "FLASHMLA_SPARSE_DSV4" - - @staticmethod - def get_builder_cls() -> type["DeepseekV4FlashMLAMetadataBuilder"]: - return DeepseekV4FlashMLAMetadataBuilder + def get_builder_cls() -> type["DeepseekV4SparseMLAMetadataBuilder"]: + return DeepseekV4SparseMLAMetadataBuilder @staticmethod def get_impl_cls() -> type[Any]: @@ -67,7 +63,7 @@ class DeepseekV4FlashMLABackend(AttentionBackend): # not the generic ``Attention``/``MLAAttention`` layer, so the backend's # impl class is never instantiated. raise NotImplementedError( - "DeepseekV4FlashMLABackend has no separate impl class; DeepSeek-V4 " + "DeepseekV4SparseMLABackend has no separate impl class; DeepSeek-V4 " "attention runs through DeepseekV4Attention." ) @@ -131,7 +127,7 @@ class DeepseekV4FlashMLAMetadata(AttentionMetadata): c128a_prefill_topk_indices: torch.Tensor | None = None -class DeepseekV4FlashMLAMetadataBuilder( +class DeepseekV4SparseMLAMetadataBuilder( AttentionMetadataBuilder[DeepseekV4FlashMLAMetadata] ): _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH @@ -294,6 +290,20 @@ class DeepseekV4FlashMLAMetadataBuilder( return result +class DeepseekV4FlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder): + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS + + +class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend): + @staticmethod + def get_name() -> str: + return "FLASHMLA_SPARSE_DSV4" + + @staticmethod + def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]: + return DeepseekV4FlashMLAMetadataBuilder + + def build_c128a_topk_metadata( positions: torch.Tensor, compress_ratio: int, diff --git a/vllm/models/deepseek_v4/xpu/xpu_sparse.py b/vllm/models/deepseek_v4/xpu/xpu_sparse.py index 73439d8646..9b39c716c7 100644 --- a/vllm/models/deepseek_v4/xpu/xpu_sparse.py +++ b/vllm/models/deepseek_v4/xpu/xpu_sparse.py @@ -19,8 +19,8 @@ from vllm.models.deepseek_v4.common.ops import ( dequantize_and_gather_k_cache, ) from vllm.models.deepseek_v4.sparse_mla import ( - DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, + DeepseekV4SparseMLABackend, ) from vllm.models.deepseek_v4.xpu.xpu_sparse_decode_fp8 import ( xpu_sparse_decode_fp8, @@ -32,7 +32,7 @@ if TYPE_CHECKING: from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata -class DeepseekV4XPUSparseBackend(DeepseekV4FlashMLABackend): +class DeepseekV4XPUSparseBackend(DeepseekV4SparseMLABackend): @staticmethod def get_name() -> str: return "XPU_V4_MLA_SPARSE" diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 8964f4047e..41cfabbecf 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -908,6 +908,20 @@ class SamplingParams( if speculative_config is None: return + # Adaptive verification compacts logits after the forward pass, while + # compute_topk_scores uses the scheduled layout in cu_num_logits_np. + # TODO(lucas): lift this restriction. The true boundaries exist on device as + # cu_num_logits; cu_num_generated_tokens is only read host-side after + # the logprobs D2H, so it could ride along on that copy. + if ( + speculative_config.enable_adaptive_verification + and self.num_logprobs is not None + ): + raise ValueError( + "Output logprobs are not supported with DSpark confidence-based " + "verification." + ) + # Some sampling parameters are not yet compatible with spec decoding. if self.min_p > _SAMPLING_EPS or self.logit_bias: raise VLLMValidationError( diff --git a/vllm/utils/deep_gemm.py b/vllm/utils/deep_gemm.py index 502f046ff9..29adb2a6f3 100644 --- a/vllm/utils/deep_gemm.py +++ b/vllm/utils/deep_gemm.py @@ -553,7 +553,10 @@ def fp8_fp4_mqa_logits( def get_paged_mqa_logits_metadata( - context_lens: torch.Tensor, block_size: int, num_sms: int + context_lens: torch.Tensor, + block_size: int, + num_sms: int, + indices: torch.Tensor | None = None, ) -> torch.Tensor: """Build scheduling metadata for paged MQA logits. @@ -562,6 +565,7 @@ def get_paged_mqa_logits_metadata( per batch element. block_size: KV-cache block size in tokens (e.g., 64). num_sms: Number of SMs available. 132 for Hopper + indices: Optional request index for each varlen row. Returns: Backend-specific tensor consumed by `fp8_fp4_paged_mqa_logits` to @@ -570,7 +574,10 @@ def get_paged_mqa_logits_metadata( _lazy_init() if _get_paged_mqa_logits_metadata_impl is None: return _missing() - return _get_paged_mqa_logits_metadata_impl(context_lens, block_size, num_sms) + kwargs = {} if indices is None else {"indices": indices} + return _get_paged_mqa_logits_metadata_impl( + context_lens, block_size, num_sms, **kwargs + ) def fp8_fp4_paged_mqa_logits( @@ -582,6 +589,7 @@ def fp8_fp4_paged_mqa_logits( schedule_metadata: torch.Tensor, max_model_len: int, clean_logits: bool, + indices: torch.Tensor | None = None, ) -> torch.Tensor: """Compute MQA logits using a paged KV-cache. @@ -606,6 +614,7 @@ def fp8_fp4_paged_mqa_logits( used to distribute work across SMs. max_model_len: Maximum sequence length used to size the logits output. clean_logits: Whether to clean the unfilled logits into `-inf`. + indices: Optional request index for each varlen row. Returns: Logits tensor of shape [B * next_n, max_model_len], dtype @@ -614,6 +623,7 @@ def fp8_fp4_paged_mqa_logits( _lazy_init() if _fp8_fp4_paged_mqa_logits_impl is None: return _missing() + kwargs = {} if indices is None else {"indices": indices} return _fp8_fp4_paged_mqa_logits_impl( q, kv_cache, @@ -623,6 +633,7 @@ def fp8_fp4_paged_mqa_logits( schedule_metadata, max_model_len, clean_logits=clean_logits, + **kwargs, ) diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index b860474035..e3f05df8dc 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -16,7 +16,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticTensorSym, kNvfp4Dynamic, ) -from vllm.utils.torch_utils import np_to_pinned_tensor if TYPE_CHECKING: from vllm.config import VllmConfig @@ -281,6 +280,21 @@ class AttentionBackend(ABC): def supports_kv_connector(cls) -> bool: return True + @classmethod + def supports_device_cpu_query_lens_mismatch(cls) -> bool: + """Whether this backend can run a batch whose device query_start_loc disagrees + with the CPU one; backends that plan off the CPU query lengths must opt out. + + Currently only verification requests are affected: adaptive verification trims + their drafts on device. On the CPU the draft budget is evenly distributed across + requests, so the total draft budget, the decode/prefill split point and the CPU + prefill query lengths all stay correct. + + SSM backends opt out: their recurrent-state planning is built from the CPU + per-request boundaries, which the trimmed batch no longer matches. + """ + return not cls.is_ssm() + @classmethod def supports_pcp(cls) -> bool: try: @@ -335,6 +349,7 @@ class AttentionBackend(ABC): use_batch_invariant: bool = False, use_kv_connector: bool = False, use_pcp: bool = False, + use_adaptive_verification: bool = False, ) -> list[str]: invalid_reasons = [] if not cls.supports_head_size(head_size): @@ -377,6 +392,14 @@ class AttentionBackend(ABC): invalid_reasons.append("KV connector not supported") if use_pcp and not cls.supports_pcp(): invalid_reasons.append("PCP not supported") + if ( + use_adaptive_verification + and not cls.supports_device_cpu_query_lens_mismatch() + ): + invalid_reasons.append( + "device-cpu query lens mismatch not supported, " + "this is needed for adaptive verification" + ) combination_reason = cls.supports_combination( head_size, dtype, @@ -549,17 +572,19 @@ class CommonAttentionMetadata: assert self._token_to_req_indices_cache.shape[0] >= num_tokens return self._token_to_req_indices_cache[:num_tokens] - starts = np.asarray(self.query_start_loc_cpu, dtype=np.int32) - query_lens = np.diff(starts) - token_to_req_indices = np.repeat( - np.arange(query_lens.shape[0], dtype=np.int32), query_lens - ) - num_mapped_tokens = token_to_req_indices.shape[0] + # Built from the device query_start_loc: adaptive verification decides the + # per-request draft split on device, so the CPU copy carries the right total + # but not the right per-request boundaries. Padding requests have a query + # length of zero and drop out of the repeat. + num_mapped_tokens = int(self.query_start_loc_cpu[-1]) + query_lens = self.query_start_loc[1:] - self.query_start_loc[:-1] assert buffer.shape[0] >= max(num_mapped_tokens, num_tokens) - # copy from CPU to GPU - buffer[:num_mapped_tokens].copy_( - np_to_pinned_tensor(token_to_req_indices), non_blocking=True + token_to_req_indices = torch.repeat_interleave( + torch.arange(query_lens.shape[0], dtype=torch.int32, device=buffer.device), + query_lens, + output_size=num_mapped_tokens, ) + buffer[:num_mapped_tokens].copy_(token_to_req_indices) if num_mapped_tokens < num_tokens: buffer[num_mapped_tokens:num_tokens].zero_() self._token_to_req_indices_cache = buffer[: max(num_mapped_tokens, num_tokens)] diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index e074afb3ea..5ef7759b70 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -428,6 +428,12 @@ class FlashInferBackend(AttentionBackend): def supports_non_causal(cls) -> bool: return True + @classmethod + def supports_device_cpu_query_lens_mismatch(cls) -> bool: + # The wrappers are planned from qo_indptr_cpu, so the CPU query offsets + # have to be the ones the kernel runs on. + return False + @classmethod def supports_sliding_window(cls) -> bool: return True diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 7051023a7c..30951ca4c1 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -128,6 +128,12 @@ class DeepseekV32IndexerBackend(AttentionBackend): def supports_pcp(cls) -> bool: return True + @classmethod + def supports_device_cpu_query_lens_mismatch(cls) -> bool: + # Only the varlen paged MQA logits kernel takes per-request query + # lengths from device tensors; otherwise the indexer needs uniform ones. + return _supports_varlen_paged_mqa_logits() + @staticmethod def get_name() -> str: return "DEEPSEEK_V32_INDEXER" @@ -418,6 +424,7 @@ class DeepSeekV32IndexerDecodeMetadata: requires_padding: bool schedule_metadata: torch.Tensor global_seq_lens: torch.Tensor | None = None + indices: torch.Tensor | None = None @dataclass @@ -452,6 +459,14 @@ def get_max_prefill_buffer_size(vllm_config: VllmConfig): return max_model_len * 40 +def _supports_varlen_paged_mqa_logits() -> bool: + return ( + current_platform.is_cuda() + and current_platform.is_device_capability_family(100) + and has_deep_gemm() + ) + + class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): # The indexer opts out of the shared reorder-threshold vote (see __init__), # so this is None; its own split uses self.decode_threshold. @@ -464,6 +479,8 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec, ) -> AttentionCGSupport: + if _supports_varlen_paged_mqa_logits(): + return AttentionCGSupport.ALWAYS return AttentionCGSupport.UNIFORM_BATCH def __init__(self, *args, block_table_width: int, **kwargs) -> None: @@ -515,10 +532,12 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): self.use_flattening = not current_platform.is_device_capability_family( 100 ) and next_n not in (1, 2) + self.supports_varlen = _supports_varlen_paged_mqa_logits() logger.info_once( - "DSA indexer decode path: use_flattening=%s " + "DSA indexer decode path: use_flattening=%s supports_varlen=%s " "(next_n=%d, use_fp4_indexer_cache=%s)", self.use_flattening, + self.supports_varlen, next_n, self.use_fp4_indexer_cache, ) @@ -547,6 +566,11 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): dtype=torch.int32, device=self.device, ) + self.decode_indices_buffer = torch.zeros( + (scheduler_config.max_num_batched_tokens,), + dtype=torch.int32, + device=self.device, + ) self.arange_buffer = torch.arange( max( scheduler_config.max_num_seqs * next_n, @@ -626,25 +650,16 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): next_n: int, max_decode_len: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, bool]: - """Expand seq_lens/block_table/decode_lens for the decode kernels. - - Flatten path (not use_native, max_decode_len > 1): - Each multi-token decode request is expanded into individual - single-token entries so the kernel always sees next_n=1. - - Native path (use_native or max_decode_len == 1): - Plain decode or spec-decode with 2D per-token context lengths. - - Returns (seq_lens, block_table, decode_lens, batch_size, requires_padding). - seq_lens is 1D (batch_size,) for flatten/plain, 2D (B, max_decode_len) - for native MTP. - """ + """Prepare native or per-token flattened decode tensors.""" min_decode_len = int(decode_lens_cpu.min().item()) - if not use_native and max_decode_len > 1: + if not use_native: assert self.decode_seq_lens_buffer.dim() == 1 - if min_decode_len == max_decode_len: - # Uniform decode lengths. - num_decode_tokens = num_decodes * max_decode_len + if ( + not self.supports_varlen + and min_decode_len == max_decode_len + and num_decodes * max_decode_len == num_decode_tokens + ): + # Uniform decode lengths with no cudagraph token padding. _prepare_uniform_decode_kernel[(num_decode_tokens,)]( seq_lens, self.decode_seq_lens_buffer, @@ -761,6 +776,28 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): self.global_decode_seq_lens_buffer[actual_expanded:num_decode_tokens] = 0 return self.global_decode_seq_lens_buffer[:num_decode_tokens] + def _build_varlen_decode_indices( + self, + decode_lens: torch.Tensor, + decode_lens_cpu: torch.Tensor, + num_decode_tokens: int, + ) -> torch.Tensor: + """Build request ids for flattened SM100 varlen rows.""" + indices = self.decode_indices_buffer[:num_decode_tokens] + actual_expanded = int(decode_lens_cpu.sum().item()) + num_decodes = decode_lens.shape[0] + indices[:actual_expanded] = torch.repeat_interleave( + self.arange_buffer[:num_decodes], + decode_lens, + output_size=actual_expanded, + ) + if actual_expanded < num_decode_tokens: + pad = num_decode_tokens - actual_expanded + indices[actual_expanded:num_decode_tokens] = ( + num_decodes + self.arange_buffer[:pad] + ) + return indices + def build( self, common_prefix_len: int, @@ -775,12 +812,11 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): slot_mapping = common_attn_metadata.slot_mapping block_table = common_attn_metadata.block_table_tensor dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens - num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( split_decodes_and_prefills( common_attn_metadata, decode_threshold=self.decode_threshold, - require_uniform=not self.use_flattening, + require_uniform=not (self.use_flattening or self.supports_varlen), treat_short_extends_as_decodes=not self.use_pcp, ) ) @@ -888,7 +924,10 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): max_decode_len = int(decode_lens_cpu.max().item()) next_n = 1 + self.num_speculative_tokens - use_native = not self.use_flattening and max_decode_len <= next_n + use_native = ( + not (self.use_flattening or self.supports_varlen) + and max_decode_len <= next_n + ) global_seq_lens_for_decode = self._prepare_global_decode_seq_lens( global_seq_lens=global_seq_lens_for_decode, @@ -900,6 +939,14 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): max_decode_len=max_decode_len, ) + decode_indices = None + if self.supports_varlen: + decode_indices = self._build_varlen_decode_indices( + decode_lens=decode_lens, + decode_lens_cpu=decode_lens_cpu, + num_decode_tokens=num_decode_tokens, + ) + seq_lens, block_table, decode_lens, batch_size, requires_padding = ( self._prepare_decode_tensors( seq_lens=seq_lens, @@ -915,9 +962,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): ) ) - seq_lens_is_buffer_view = (use_native and next_n > 1) or ( - not use_native and max_decode_len > 1 - ) + seq_lens_is_buffer_view = not use_native or next_n > 1 # DCP: localize the now-expanded per-token global bounds to this # rank's owned KV. Done here (after expansion) so each token's global @@ -952,6 +997,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): seq_lens, self.kv_cache_spec.storage_block_size, self.num_sms, + indices=decode_indices, ) decode_metadata = DeepSeekV32IndexerDecodeMetadata( @@ -960,6 +1006,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): decode_lens=decode_lens, requires_padding=requires_padding, schedule_metadata=self.scheduler_metadata_buffer, + indices=decode_indices, global_seq_lens=global_seq_lens_for_decode, ) diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index c11698335a..da0c125c62 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -61,8 +61,10 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase): dtype: torch.dtype, prefix: str, cache_config: CacheConfig, + backend_cls: "type[AttentionBackend] | None" = None, ): super().__init__() + self.backend_cls = backend_cls or DeepseekSparseSWABackend self.kv_cache = torch.tensor([]) self.head_dim = head_dim self.window_size = window_size @@ -104,7 +106,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase): def forward(self): ... def get_attn_backend(self) -> type[AttentionBackend]: - return DeepseekSparseSWABackend + return self.backend_cls class DeepseekSparseSWABackend(AttentionBackend): @@ -183,6 +185,7 @@ class DeepseekSparseSWAMetadata: num_prefills: int = 0 num_decode_tokens: int = 0 num_prefill_tokens: int = 0 + max_decode_query_len: int = 1 # Pre-computed prefill metadata shared across all DeepseekV4 attention layers. prefill_seq_lens: torch.Tensor | None = None @@ -644,6 +647,13 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): num_prefills=num_prefills, num_decode_tokens=num_decode_tokens, num_prefill_tokens=num_prefill_tokens, + # Upper bound on decode-split rows for the kernel's max_q_len + # hint. common max_query_len bounds every row (scheduled max under + # adaptive verification), clamped to what the split can admit so a + # mixed batch's prefill max does not inflate decode scheduling. + max_decode_query_len=min( + common_attn_metadata.max_query_len, self.decode_threshold + ), tile_sched_swaonly=tile_sched[_LAYER_TYPE_SWAONLY], tile_sched_c4a=tile_sched[_LAYER_TYPE_C4A], tile_sched_c128a=tile_sched[_LAYER_TYPE_C128A], @@ -787,6 +797,14 @@ def _compute_swa_indices_and_lens_kernel( is_valid = tl.load(is_valid_token_ptr + token_idx) if not is_valid: tl.store(swa_lens_ptr + pid, 0) + # Clear the row so a padded token cannot gather through stale indices. + for i in range(0, window_size, TRITON_BLOCK_SIZE): + offset = i + tl.arange(0, TRITON_BLOCK_SIZE) + tl.store( + swa_indices_ptr + pid * swa_indices_stride + offset, + -1, + mask=offset < window_size, + ) return req_idx = tl.load(token_to_req_indices_ptr + token_idx) @@ -853,6 +871,14 @@ def _compute_dspark_noncausal_swa_indices_kernel( is_valid = tl.load(is_valid_token_ptr + token_idx) if not is_valid: tl.store(swa_lens_ptr + pid, 0) + # Clear the row so a padded token cannot gather through stale indices. + for i in range(0, index_width, TRITON_BLOCK_SIZE): + offset = i + tl.arange(0, TRITON_BLOCK_SIZE) + tl.store( + swa_indices_ptr + pid * swa_indices_stride + offset, + -1, + mask=offset < index_width, + ) return req_idx = tl.load(token_to_req_indices_ptr + token_idx) diff --git a/vllm/v1/attention/selector.py b/vllm/v1/attention/selector.py index bff26e0266..00f2aed5c5 100644 --- a/vllm/v1/attention/selector.py +++ b/vllm/v1/attention/selector.py @@ -37,6 +37,7 @@ class AttentionSelectorConfig(NamedTuple): use_batch_invariant: bool = False use_kv_connector: bool = False use_pcp: bool = False + use_adaptive_verification: bool = False def __repr__(self): return ( @@ -54,6 +55,7 @@ class AttentionSelectorConfig(NamedTuple): f"use_non_causal={self.use_non_causal}, " f"use_batch_invariant={self.use_batch_invariant}, " f"use_kv_connector={self.use_kv_connector}, " + f"use_adaptive_verification={self.use_adaptive_verification}, " f"use_pcp={self.use_pcp})" ) @@ -136,6 +138,18 @@ def get_attn_backend( kv_transfer_config is not None and kv_transfer_config.is_kv_transfer_instance ) + speculative_config = vllm_config.speculative_config + use_adaptive_verification = ( + speculative_config is not None + and speculative_config.enable_adaptive_verification + ) + if use_adaptive_verification: + from vllm.compilation.backends import model_tag + + # The drafter always runs full-length blocks; only the verifier sees + # the trimmed, device-decided query lengths. + use_adaptive_verification = model_tag != "dspark_head" + attn_type = attn_type or AttentionType.DECODER attn_selector_config = AttentionSelectorConfig( head_size=head_size, @@ -153,6 +167,7 @@ def get_attn_backend( use_batch_invariant=envs.VLLM_BATCH_INVARIANT, use_kv_connector=use_kv_connector, use_pcp=vllm_config.parallel_config.prefill_context_parallel_size > 1, + use_adaptive_verification=use_adaptive_verification, ) # A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index fa009230f8..1a8f8f7a2d 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import contextlib +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import TYPE_CHECKING import numpy as np import torch @@ -17,6 +20,97 @@ from vllm.v1.outputs import ( from vllm.v1.worker.gpu.sample.output import SamplerOutput from vllm.v1.worker.utils import raise_if_nan_logits +if TYPE_CHECKING: + from vllm.v1.worker.gpu.input_batch import InputBatch + + +@dataclass(frozen=True) +class StepTimingSample: + forward_ms: float + drafter_ms: float + num_target_tokens: int + num_reqs: int + full_cudagraph: bool + + +def _timing_event() -> torch.cuda.Event: + return torch.cuda.Event(enable_timing=True) + + +@dataclass +class StepTimingEvents: + forward_start: torch.cuda.Event = field(default_factory=_timing_event) + forward_end: torch.cuda.Event = field(default_factory=_timing_event) + drafter_start: torch.cuda.Event = field(default_factory=_timing_event) + drafter_end: torch.cuda.Event = field(default_factory=_timing_event) + + +class StepTimingCollector: + """Times the steps run inside ``collect``; record calls no-op outside it. + + Every step gets its own events, so the steps queue back-to-back and the + block resolves them all behind one sync on the way out. + """ + + def __init__(self): + self._collecting = False + self._step: StepTimingEvents | None = None + self._batch = (False, 0, 0) + self._timed: list[tuple[StepTimingEvents, tuple[bool, int, int]]] = [] + + @contextlib.contextmanager + def collect(self) -> Iterator[list[StepTimingSample]]: + """Time every step run in this block. + + The yielded list holds one sample per timed step once the block exits; + it stays empty inside the block, where the timings are still on device. + """ + samples: list[StepTimingSample] = [] + self._collecting = True + try: + yield samples + finally: + self._collecting = False + timed, self._timed, self._step = self._timed, [], None + if not timed: + return + # Same stream, issue order: once the last step is done, so are the rest. + timed[-1][0].drafter_end.synchronize() + samples.extend( + StepTimingSample( + events.forward_start.elapsed_time(events.forward_end), + events.drafter_start.elapsed_time(events.drafter_end), + num_target_tokens, + num_reqs, + full_cudagraph, + ) + for events, (full_cudagraph, num_target_tokens, num_reqs) in timed + ) + + def record_batch(self, input_batch: "InputBatch", full_cudagraph: bool) -> None: + """Costs from different execution modes must not share a cost curve.""" + self._batch = (full_cudagraph, input_batch.num_tokens, input_batch.num_reqs) + + def forward_start(self) -> None: + if self._collecting: + self._step = StepTimingEvents() + self._step.forward_start.record() + + def forward_end(self) -> None: + if self._step is not None: + self._step.forward_end.record() + + def drafter_start(self) -> None: + if self._step is not None: + self._step.drafter_start.record() + + def drafter_end(self) -> None: + """Ends the step: only steps that reach here have a draft cost.""" + if self._step is not None: + self._step.drafter_end.record() + self._timed.append((self._step, self._batch)) + self._step = None + class AsyncOutput(AsyncModelRunnerOutput): def __init__( diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 4c137a8fe8..d2977870da 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -180,6 +180,22 @@ def init_attn_backend( return attn_groups, attn_cg_support_info, kernel_block_sizes +def get_query_lens_mismatch_unsupported_backend( + attn_groups: list[list[AttentionGroup]], +) -> str | None: + """Name the first backend needing the CPU query lengths to be exact, if any. + + The attention selector already excludes these when adaptive verification is + enabled, but models that hard-wire their backend never consult it. See + AttentionBackend.supports_device_cpu_query_lens_mismatch(). + """ + for groups in attn_groups: + for group in groups: + if not group.backend.supports_device_cpu_query_lens_mismatch(): + return group.backend.__name__ + return None + + def _allocate_kv_cache( kv_cache_config: KVCacheConfig, shared_layers: dict[str, str], device: torch.device ): diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 474c7e28aa..8e24a0ee02 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -54,6 +54,9 @@ class BatchExecutionDescriptor: num_tokens: int num_reqs: int | None # None means no request padding is needed (PIECEWISE graphs) uniform_token_count: int | None = None + # Upper bound on per-request query length. Varlen decode graphs leave + # uniform_token_count unset, so this is what keeps a prefill batch out of one. + max_query_len: int | None = None num_active_loras: int = 0 @@ -75,14 +78,21 @@ def _is_compatible( num_tokens: int, uniform_token_count: int | None, num_active_loras: int, + max_query_len: int | None, ) -> bool: # desc.uniform_token_count=None (PIECEWISE) can handle any uniform_token_count # desc.num_reqs=None means no request padding needed (PIECEWISE) + # desc.max_query_len=None means the graph does not constrain query length; a + # caller that does not track max_query_len must not match one that does return ( ( desc.uniform_token_count is None or desc.uniform_token_count == uniform_token_count ) + and ( + desc.max_query_len is None + or (max_query_len is not None and desc.max_query_len >= max_query_len) + ) and (desc.num_reqs is None or desc.num_reqs >= num_reqs) and desc.num_tokens >= num_tokens and desc.num_active_loras == num_active_loras @@ -97,6 +107,7 @@ class CudaGraphManager: cudagraph_mode: CUDAGraphMode, decode_query_len: int, lora_capture_cases: list[int] | None = None, + varlen_decode: bool = False, ): self.vllm_config = vllm_config self.device = device @@ -105,6 +116,7 @@ class CudaGraphManager: assert self.compilation_config is not None self.cudagraph_mode = cudagraph_mode self.decode_query_len = decode_query_len + self.varlen_decode = varlen_decode self.dp_size = vllm_config.parallel_config.data_parallel_size self.tp_size = vllm_config.parallel_config.tensor_parallel_size @@ -204,12 +216,27 @@ class CudaGraphManager: else: decode_query_lens = [self.decode_query_len] + capture_varlen_decode = ( + separate_decode_routine and bool(decode_mode) and self.varlen_decode + ) for num_tokens, num_active_loras in product( capture_sizes, self.lora_capture_cases ): + # Varlen decode graphs take any mix of 1..decode_query_len tokens per + # request, worst case 1 token per request (or max_num_reqs) + if capture_varlen_decode and num_tokens <= max_decode_tokens: + desc = BatchExecutionDescriptor( + cg_mode=decode_mode, + num_tokens=num_tokens, + num_reqs=min(num_tokens, self.max_num_reqs), + max_query_len=self.decode_query_len, + num_active_loras=num_active_loras, + ) + descs_by_mode[decode_mode].append(desc) + descs_by_token_lora[(num_tokens, num_active_loras)].append(desc) # Capture uniform decode specfifc graphs if required # (i.e. separate decode routine) - if separate_decode_routine and decode_mode: + elif separate_decode_routine and decode_mode and not self.varlen_decode: for decode_query_len in decode_query_lens: rounded_num_tokens = round_up(num_tokens, decode_query_len) rounded_num_reqs = rounded_num_tokens // decode_query_len @@ -346,12 +373,19 @@ class CudaGraphManager: compilation_counter.num_cudagraph_captured += 1 self._graphs_captured = True + def captured_token_counts(self) -> list[int]: + """Sorted token counts with a captured graph, ignoring LoRA variants.""" + return sorted( + {desc.num_tokens for desc in self.graphs if desc.num_active_loras == 0} + ) + def dispatch( self, num_reqs: int, num_tokens: int, uniform_token_count: int | None, num_active_loras: int, + max_query_len: int | None = None, ) -> BatchExecutionDescriptor: """Find matching cudagraph descriptor from priority-ordered candidates.""" @@ -365,6 +399,7 @@ class CudaGraphManager: num_tokens, uniform_token_count, effective_loras, + max_query_len, ): return desc return BatchExecutionDescriptor( @@ -413,6 +448,7 @@ class ModelCudaGraphManager(CudaGraphManager): cudagraph_mode: CUDAGraphMode, decode_query_len: int, lora_capture_cases: list[int] | None = None, + varlen_decode: bool = False, ): super().__init__( vllm_config, @@ -420,6 +456,7 @@ class ModelCudaGraphManager(CudaGraphManager): cudagraph_mode, decode_query_len, lora_capture_cases=lora_capture_cases, + varlen_decode=varlen_decode, ) self.hidden_states: torch.Tensor | None = None self.aux_hidden_states: list[torch.Tensor] = [] @@ -483,6 +520,7 @@ class ModelCudaGraphManager(CudaGraphManager): attn_groups, kv_cache_config, full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL, + max_query_len=desc.max_query_len, ) # Capture with dummy rows marked as padding. @@ -574,8 +612,11 @@ def prepare_inputs_to_capture( attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, full_cudagraph: bool, + max_query_len: int | None = None, ) -> AttentionState: - input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers) + input_batch = InputBatch.make_dummy( + num_reqs, num_tokens, input_buffers, max_query_len=max_query_len + ) input_block_tables = block_tables.get_dummy_block_tables(num_reqs) slot_mappings = block_tables.get_dummy_slot_mappings(num_tokens) slot_mappings_by_layer = build_slot_mappings_by_layer( diff --git a/vllm/v1/worker/gpu/dp_utils.py b/vllm/v1/worker/gpu/dp_utils.py index ee9b924ba1..fcdb4e28cf 100644 --- a/vllm/v1/worker/gpu/dp_utils.py +++ b/vllm/v1/worker/gpu/dp_utils.py @@ -21,6 +21,7 @@ def sync_cudagraph_and_dp_padding( uniform_token_count: int | None, dp_size: int, dp_rank: int, + max_query_len: int | None = None, num_active_loras: int = 0, ) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]: """ @@ -30,15 +31,17 @@ def sync_cudagraph_and_dp_padding( """ assert dp_size > 1, "DP size must be greater than 1" group = get_dp_group().cpu_group - tensor = torch.zeros(3, dp_size, dtype=torch.int32, device="cpu") + tensor = torch.zeros(4, dp_size, dtype=torch.int32, device="cpu") tensor[0][dp_rank] = num_tokens tensor[1][dp_rank] = desired_batch_desc.cg_mode.value tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None) + tensor[3][dp_rank] = max_query_len or -1 # (-1 means None) dist.all_reduce(tensor, group=group) num_tokens_across_dp = tensor[0] cg_mode_across_dp = tensor[1] uniform_token_counts_across_dp = tensor[2] + max_query_lens_across_dp = tensor[3] if torch.all(num_tokens_across_dp == 0).item(): synced_desc = BatchExecutionDescriptor( @@ -69,6 +72,12 @@ def sync_cudagraph_and_dp_padding( ): synced_uniform_token_count = None + # Varlen decode graphs are selected by the query-length bound, so ranks must agree + # on it or they pad to different token counts below. + synced_max_query_len: int | None = None + if bool(torch.all(max_query_lens_across_dp != -1).item()): + synced_max_query_len = int(max_query_lens_across_dp.max().item()) + # Dispatch for the final synced values, use num_reqs instead of synced_num_reqs # so we don't perform request padding for PIECEWISE graphs. # num_active_loras is per-rank and doesn't need cross-rank agreement. @@ -77,6 +86,7 @@ def sync_cudagraph_and_dp_padding( synced_num_tokens, synced_uniform_token_count, num_active_loras=num_active_loras, + max_query_len=synced_max_query_len, ) # Update num_tokens_across_dp to reflect padded size. @@ -92,6 +102,7 @@ def dispatch_cg_and_sync_dp( uniform_token_count: int | None, dp_size: int, dp_rank: int, + max_query_len: int | None = None, need_eager: bool = False, num_active_loras: int = 0, ) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]: @@ -112,6 +123,7 @@ def dispatch_cg_and_sync_dp( num_tokens, uniform_token_count, num_active_loras=num_active_loras, + max_query_len=max_query_len, ) if dp_size == 1: @@ -125,5 +137,6 @@ def dispatch_cg_and_sync_dp( uniform_token_count, dp_size, dp_rank, + max_query_len=max_query_len, num_active_loras=num_active_loras, ) diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 6362e86c1d..9a574a4faa 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -1,12 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass +from typing import TYPE_CHECKING import numpy as np import torch from vllm.triton_utils import tl, triton from vllm.utils import random_uuid +from vllm.utils.math_utils import cdiv + +if TYPE_CHECKING: + from vllm.v1.worker.gpu.block_table import BlockTables class InputBuffers: @@ -49,9 +54,10 @@ class InputBatch: expanded_local_pos: torch.Tensor # [num_reqs] - # batch_idx -> num_scheduled_tokens + # batch_idx -> num_scheduled_tokens, (upper bound when using adaptive verification) num_scheduled_tokens: np.ndarray - # sum(num_scheduled_tokens) + # number of tokens in the batch, + # may be < sum(num_scheduled_tokens) when using adaptive verification num_tokens: int num_tokens_after_padding: int # Sum of draft tokens scheduled across requests. @@ -101,9 +107,18 @@ class InputBatch: # [num_reqs] per-request prompt length, only populated for R-SWA. prompt_lens: torch.Tensor | None + # Longest query the batch may contain. Set when a cudagraph descriptor promises + # a query length this batch's own split does not reach, so attention metadata + # stays valid for every replay the graph serves. + max_query_len: int | None = None + @classmethod def make_dummy( - cls, num_reqs: int, num_tokens: int, input_buffers: InputBuffers + cls, + num_reqs: int, + num_tokens: int, + input_buffers: InputBuffers, + max_query_len: int | None = None, ) -> "InputBatch": assert 0 < num_reqs <= num_tokens device = input_buffers.device @@ -115,9 +130,12 @@ class InputBatch: expanded_local_pos = torch.zeros(num_reqs, dtype=torch.int32, device=device) # Distribute the remainder evenly so that no dummy request exceeds - # ceil(num_tokens / num_reqs) <= max_model_len tokens. + # ceil(num_tokens / num_reqs) <= max_model_len tokens. Varlen graphs + # accept any split with non-empty slots, so this shape works for them + # too; attention metadata is built from the promised max_query_len. base_tokens = num_tokens // num_reqs num_extra = num_tokens % num_reqs + assert max_query_len is None or base_tokens + (num_extra > 0) <= max_query_len num_scheduled_tokens = np.full(num_reqs, base_tokens, dtype=np.int32) if num_extra > 0: num_scheduled_tokens[-num_extra:] += 1 @@ -150,7 +168,8 @@ class InputBatch: logits_indices = query_start_loc[1:] - 1 cu_num_logits = torch.arange(num_reqs + 1, device=device, dtype=torch.int32) cu_num_logits_np = np.arange(num_reqs + 1, dtype=np.int32) - # Dummy: seq_len == query_len (fresh-prefill shape). + # Copy so set_dummy_context can add context in place without touching + # num_scheduled_tokens. seq_lens_cpu_upper_bound = torch.from_numpy(num_scheduled_tokens.copy()) return cls( req_ids=req_ids, @@ -184,9 +203,53 @@ class InputBatch: cu_num_logits_np=cu_num_logits_np, has_structured_output_reqs=False, prompt_lens=None, + max_query_len=max_query_len, ) +def set_dummy_context( + input_batch: InputBatch, + block_tables: "BlockTables", + context_len: int, + num_kv_blocks: int, + max_model_len: int, +) -> None: + """Give each dummy request context_len of context, used when profiling step cost.""" + if not block_tables.input_block_tables: + # Attention-free models have no KV context to fabricate. + return + num_reqs = input_batch.num_reqs + query_len = input_batch.max_query_len or int(input_batch.num_scheduled_tokens.max()) + context_len = max(min(context_len, max_model_len - query_len), 0) + if not context_len: + return + + # Decode-like shape: each request continues after context_len + # already-computed tokens. + input_batch.seq_lens += context_len + input_batch.seq_lens_cpu_upper_bound += context_len + input_batch.num_computed_tokens_np.fill(context_len) + input_batch.num_computed_prefill_tokens_np.fill(context_len) + local_pos = np.arange(input_batch.num_tokens, dtype=np.int64) - np.repeat( + input_batch.query_start_loc_np[:-1], input_batch.num_scheduled_tokens + ) + input_batch.positions.copy_(torch.from_numpy(local_pos + context_len)) + + seq_len = context_len + query_len + for block_table, block_size, bpk in zip( + block_tables.input_block_tables, + block_tables.kernel_block_sizes, + block_tables.blocks_per_kv_block, + ): + num_blocks = min(cdiv(seq_len, block_size), block_table.shape[1]) + # Spans are disjoint until the pool runs out, then they wrap and share + # blocks: profiling only needs the reads to be realistic, not distinct. + block_ids = torch.arange( + num_reqs * num_blocks, dtype=block_table.dtype, device=block_table.device + ) % (num_kv_blocks * bpk) + block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks) + + @triton.jit def _prepare_prefill_inputs_kernel( input_ids_ptr, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index a9bc7db103..23a037bef9 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -72,7 +72,11 @@ from vllm.v1.outputs import ( from vllm.v1.worker.block_table import get_block_table_width from vllm.v1.worker.cp_utils import check_attention_cp_compatibility from vllm.v1.worker.gpu import pcp_manager as pcp -from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput +from vllm.v1.worker.gpu.async_utils import ( + AsyncOutput, + AsyncPoolingOutput, + StepTimingCollector, +) from vllm.v1.worker.gpu.attn_utils import ( build_slot_mappings_by_layer, get_kv_cache_spec, @@ -101,6 +105,7 @@ from vllm.v1.worker.gpu.input_batch import ( post_update_num_computed_tokens, prepare_pos_seq_lens, prepare_prefill_inputs, + set_dummy_context, ) from vllm.v1.worker.gpu.kv_connector import ( NO_OP_KV_CONNECTOR, @@ -123,10 +128,17 @@ from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker from vllm.v1.worker.gpu.sample.sampler import Sampler from vllm.v1.worker.gpu.shutdown import free_before_shutdown from vllm.v1.worker.gpu.spec_decode import init_speculator +from vllm.v1.worker.gpu.spec_decode.adaptive_verification import ( + AdaptiveVerificationManager, + maybe_create_adaptive_verification_manager, +) from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import ( set_eagle3_aux_hidden_state_layers, ) -from vllm.v1.worker.gpu.spec_decode.rejection_sampler import RejectionSampler +from vllm.v1.worker.gpu.spec_decode.rejection_sampler import ( + RejectionSampler, + get_max_chunk_logits, +) from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler from vllm.v1.worker.gpu.states import RequestState @@ -252,6 +264,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): and self.speculative_config.use_multi_module_mtp() else 1 ) + + self.step_timing = StepTimingCollector() + # General request states. self.req_states = RequestState( max_num_reqs=self.max_num_reqs, @@ -262,6 +277,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): device=self.device, num_prefill_lookahead=num_prefill_lookahead, ) + self.adaptive_verification: AdaptiveVerificationManager | None = None self.input_buffers = InputBuffers( max_num_reqs=self.max_num_reqs, max_num_tokens=self.max_num_tokens, @@ -401,6 +417,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): max_num_logits=self.max_num_reqs * self.decode_query_len, vocab_size=self.vocab_size, device=self.device, + mask_stride=self.decode_query_len, ) if self.is_pooling_model and self.is_last_pp_rank: @@ -510,6 +527,20 @@ class GPUModelRunner(LoRAModelRunnerMixin): attn_cg_support = attn_cg_support.narrow( *self.model_state.get_additional_cg_support() ) + # The speculator clears the flag at load time when the checkpoint has + # no confidence head, so it holds the effective value. + self.adaptive_verification = maybe_create_adaptive_verification_manager( + enable_adaptive_verification=getattr( + self.speculator, "enable_adaptive_verification", False + ), + attn_groups=self.attn_groups, + attn_cg_support=attn_cg_support, + req_states=self.req_states, + query_start_loc=self.input_buffers.query_start_loc, + num_bonus_tokens=self.model_state.num_new_sampled_tokens_per_step, + max_total_logits=get_max_chunk_logits(self.vocab_size), + ) + self.block_tables = BlockTables( block_sizes=block_sizes, max_num_reqs=self.max_num_reqs, @@ -532,6 +563,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): initialize_mamba_ssu_backend( self.vllm_config.mamba_config, self.kv_cache_config ) + if self.adaptive_verification is not None: + self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE cudagraph_mode = self.compilation_config.resolve_cudagraph_mode_and_sizes( attn_cg_support.min_cg_support, attn_cg_support.min_cg_attn_backend, @@ -547,6 +580,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): cudagraph_mode, decode_query_len=self.decode_query_len, lora_capture_cases=self.lora_capture_cases, + varlen_decode=self.adaptive_verification is not None, ) check_attention_cp_compatibility(self.vllm_config) if isinstance(self.speculator, DraftModelSpeculator): @@ -594,6 +628,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): *args, skip_attn: bool = False, uniform_decode: bool = False, + context_len: int = 0, skip_eplb: bool = False, is_profile: bool = False, **kwargs, @@ -655,6 +690,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): dummy_run=True, skip_attn_for_dummy_run=skip_attn, is_profile=is_profile, + context_len=context_len, ) self.kv_connector.set_disabled(False) @@ -670,9 +706,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): aux_hidden_states = self.execute_model_state.aux_hidden_states self.execute_model_state = None + self.step_timing.forward_end() + # dummy run the eagle speculator's propose to ensure DP/EP sync. if self.speculator is not None: assert self.sampler is not None + self.step_timing.drafter_start() mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None if self.speculator.supports_mm_inputs: mm_inputs = ( @@ -713,6 +752,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): mm_inputs=mm_inputs, is_profile=is_profile, ) + self.step_timing.drafter_end() assert hidden_states is not None # Last PP rank always has hidden_states sample_hidden_states = hidden_states[input_batch.logits_indices] @@ -830,6 +870,13 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) if self.speculator is not None: self.speculator.capture() + if self.adaptive_verification is not None: + with self.step_timing.collect() as timings: + for batch in self.adaptive_verification.batches_to_profile( + self.cudagraph_manager.captured_token_counts() + ): + self._dummy_run(**batch) + self.adaptive_verification.set_initial_cost_curves(timings) end_time = time.perf_counter() end_free_gpu_memory = torch.accelerator.get_memory_info()[0] @@ -906,6 +953,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): max_tokens=sampling_params.max_tokens if sampling_params else 1, # type: ignore[arg-type] ) req_index = self.req_states.req_id_to_index[req_id] + if self.adaptive_verification is not None: + self.adaptive_verification.add_request(req_index) if self.pooling_runner is not None: assert new_req_data.pooling_params is not None @@ -994,8 +1043,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_reqs, num_toks, max_query_len, has_prefill=False ) + draft_tokens = scheduler_output.scheduled_spec_decode_tokens # batch_idx -> req_id - req_ids = sort_batch_req_ids(num_tokens_per_req, self.decode_query_len) + req_ids = sort_batch_req_ids( + num_tokens_per_req, draft_tokens, self.decode_query_len + ) + numtoks_iter = map(num_tokens_per_req.__getitem__, req_ids) num_scheduled_tokens = np.fromiter(numtoks_iter, dtype=np.int32, count=num_reqs) @@ -1007,9 +1060,15 @@ class GPUModelRunner(LoRAModelRunnerMixin): ] is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np + if self.adaptive_verification is not None and draft_tokens: + num_toks = self.adaptive_verification.get_num_tokens( + num_tokens_per_req, draft_tokens + ) + batch_state = BatchReqState( req_ids=req_ids, num_scheduled_tokens=num_scheduled_tokens, + num_tokens=num_toks, idx_mapping_np=idx_mapping_np, prefill_len_np=prefill_len_np, num_computed_prefill_tokens_np=num_computed_prefill_tokens_np, @@ -1026,7 +1085,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): batch_req_state: "BatchReqState", batch_desc: BatchExecutionDescriptor, ) -> InputBatch: - num_tokens = scheduler_output.total_num_scheduled_tokens + num_tokens = batch_req_state.num_tokens num_tokens_after_padding = batch_desc.num_tokens assert num_tokens > 0 if envs.VLLM_MOE_SKIP_PADDING: @@ -1072,9 +1131,20 @@ class GPUModelRunner(LoRAModelRunnerMixin): np.cumsum(num_logits, out=cu_num_logits_np[1:]) cu_num_logits = async_copy_to_gpu(cu_num_logits_np, device=self.device) - max_expand_len = self.decode_query_len - expanded_idx_mapping, expanded_local_pos = expand_idx_mapping( - idx_mapping, total_num_logits, cu_num_logits, max_expand_len + adaptive_verification = ( + self.adaptive_verification if num_draft_tokens_per_req is not None else None + ) + num_scheduled_tokens_upper_bound = num_scheduled_tokens_np + if adaptive_verification is not None: + # num_scheduled_tokens represents the draft budget evenly distributed across + # all verification requests, `reallocate_drafts` will unevenly assign the + # draft budget to requests on the GPU side only. + num_scheduled_tokens_np, cu_num_logits_np = ( + adaptive_verification.compact_batch( + num_draft_tokens_per_req, + num_scheduled_tokens_np, + cu_num_logits_np, + ) ) # Get query_start_loc. @@ -1086,9 +1156,19 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Pad for full CUDA graph mode. # Some attention backends like FA3 require query_start_loc to be non-decreasing. query_start_loc_np[num_reqs + 1 :] = num_tokens - async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc) + query_start_loc = self.input_buffers.query_start_loc + async_copy_to_gpu(query_start_loc_np, out=query_start_loc) + if adaptive_verification is not None: + cu_num_logits, query_start_loc, total_num_draft_tokens = ( + adaptive_verification.reallocate_drafts(req_ids, idx_mapping) + ) + total_num_logits = num_reqs * num_bonus_tokens + total_num_draft_tokens + if draft_tokens: + expanded_idx_mapping, expanded_local_pos = expand_idx_mapping( + idx_mapping, total_num_logits, cu_num_logits, self.decode_query_len + ) query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1] - query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1] + query_start_loc = query_start_loc[: num_reqs_padded + 1] # Get prefill tokens if any. if batch_req_state.has_prefill: @@ -1145,7 +1225,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32) np.add( num_computed_tokens_np, - num_scheduled_tokens_np, + num_scheduled_tokens_upper_bound, out=seq_lens_cpu_upper_bound_np[:num_reqs], ) seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np) @@ -1168,7 +1248,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): idx_mapping_np=idx_mapping_np, expanded_idx_mapping=expanded_idx_mapping, expanded_local_pos=expanded_local_pos, - num_scheduled_tokens=num_scheduled_tokens_np, + num_scheduled_tokens=num_scheduled_tokens_upper_bound, num_tokens=num_tokens, num_tokens_after_padding=num_tokens_after_padding, num_draft_tokens=total_num_draft_tokens, @@ -1192,6 +1272,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): cu_num_logits_np=cu_num_logits_np, has_structured_output_reqs=scheduler_output.has_structured_output_requests, prompt_lens=prompt_lens, + max_query_len=( + int(num_scheduled_tokens_upper_bound.max()) + if adaptive_verification is not None + else None + ), ) return pcp.maybe_partition_pcp_batch(self.pcp_manager, input_batch) @@ -1298,6 +1383,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): dummy_run: bool = False, skip_attn_for_dummy_run: bool = False, is_profile: bool = False, + context_len: int = 0, ) -> ModelRunnerOutput | IntermediateTensors | None: if not dummy_run: # Update the request states. @@ -1315,9 +1401,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Get batch descriptor and sync across DP ranks. num_reqs = len(scheduler_output.num_scheduled_tokens) num_toks = scheduler_output.total_num_scheduled_tokens + max_query_len = max(scheduler_output.num_scheduled_tokens.values()) batch_req_state, uniform_tok_count = self.gather_batch_req_state( scheduler_output, dummy_run ) + if batch_req_state is not None: + num_toks = batch_req_state.num_tokens num_active_loras = 0 if self.lora_config: @@ -1340,6 +1429,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): uniform_tok_count, self.dp_size, self.dp_rank, + max_query_len=max_query_len, need_eager=is_profile or skip_compiled, num_active_loras=num_active_loras, ) @@ -1378,13 +1468,23 @@ class GPUModelRunner(LoRAModelRunnerMixin): self._set_active_loras(*lora_inputs) else: # No actual tokens to run. A dummy run for DP or memory profiling. + dummy_num_reqs = batch_desc.num_reqs or num_reqs input_batch = InputBatch.make_dummy( - batch_desc.num_reqs or num_reqs, + dummy_num_reqs, batch_desc.num_tokens, self.input_buffers, + max_query_len=batch_desc.max_query_len, ) if not skip_attn_for_dummy_run: block_tables, slot_mappings = self.prepare_dummy_attn(input_batch) + if context_len: + set_dummy_context( + input_batch, + self.block_tables, + context_len, + self.kv_cache_config.num_blocks, + self.max_model_len, + ) else: assert batch_desc.cg_mode != CUDAGraphMode.FULL, ( "Attention metadata must be prepared for dummy runs when using " @@ -1488,6 +1588,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Update the EPLB meta. self.eplb.prepare_forward(self.model_config, input_batch.num_tokens) + self.step_timing.record_batch( + input_batch, batch_desc.cg_mode == CUDAGraphMode.FULL + ) + self.step_timing.forward_start() + # Run model. if batch_desc.cg_mode == CUDAGraphMode.FULL: # Use explicit cudagraph replay for FULL mode. @@ -1698,6 +1803,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): mm_inputs=mm_inputs, ) self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens + if self.adaptive_verification is not None: + self.adaptive_verification.record_confidences( + self.speculator.draft_token_confidence_probs, input_batch + ) if self.num_speculative_steps > 0: # Spec-decode and diffusion LLMs both use draft tokens but the latter does @@ -1839,6 +1948,9 @@ class BatchReqState(NamedTuple): req_ids: list[str] num_scheduled_tokens: np.ndarray # [num_reqs] + # May be less than scheduler_output.total_num_scheduled_tokens: + # adaptive verification trims the draft budget before running. + num_tokens: int idx_mapping_np: np.ndarray # [num_reqs] prefill_len_np: np.ndarray # [num_reqs] num_computed_prefill_tokens_np: np.ndarray # [num_reqs] @@ -1847,9 +1959,15 @@ class BatchReqState(NamedTuple): def sort_batch_req_ids( - num_tokens_per_req: dict[str, int], decode_query_len: int + num_tokens_per_req: dict[str, int], + draft_tokens: dict[str, list[int]], + decode_query_len: int, ) -> list[str]: - # Order decode -> short_extend -> prefill; split_decodes_and_prefills - # relies on uniform decodes (query_len == decode_query_len) leading. - key = lambda r: ((num := num_tokens_per_req[r]) != decode_query_len, num) + # Order verification/decode -> short_extend -> prefill; + # split_decodes_and_prefills relies on decode-like requests leading. + key = lambda r: ( + not draft_tokens.get(r), + (num := num_tokens_per_req[r]) != decode_query_len, + num, + ) return sorted(num_tokens_per_req, key=key) diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index db688fa7ec..31802c017b 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -144,7 +144,9 @@ class DefaultModelState(ModelState): input_batch.query_start_loc_np[: num_reqs + 1] ) query_start_loc_gpu = input_batch.query_start_loc[: num_reqs + 1] - max_query_len = input_batch.num_scheduled_tokens.max().item() + max_query_len = input_batch.max_query_len + if max_query_len is None: + max_query_len = input_batch.num_scheduled_tokens.max().item() seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound if for_capture: # Capture with worst-case max_seq_len so the graph is valid at any replay. diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py new file mode 100644 index 0000000000..12dc8c7c65 --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -0,0 +1,477 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adaptive verification for DSpark speculative decoding.""" + +from collections import defaultdict +from collections.abc import Iterable, Iterator +from typing import TYPE_CHECKING + +import numpy as np +import torch + +import vllm.envs as envs +from vllm.distributed.parallel_state import get_tp_group +from vllm.logger import init_logger +from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.v1.attention.backend import AttentionCGSupport +from vllm.v1.utils import CpuGpuBuffer +from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream +from vllm.v1.worker.gpu.attn_utils import ( + get_query_lens_mismatch_unsupported_backend, +) +from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu + +logger = init_logger(__name__) +_PROFILE_REPLAYS = 5 + +if TYPE_CHECKING: + from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo + from vllm.v1.worker.gpu.input_batch import InputBatch + from vllm.v1.worker.gpu.states import RequestState + from vllm.v1.worker.utils import AttentionGroup + + +def _assign_draft_token_budget( + confidence_probs: torch.Tensor, + idx_mapping: torch.Tensor, + capacities: torch.Tensor, + draft_budget: int, + num_steps: int, +) -> None: + """Admit the globally best `draft_budget` draft slots, in place. + + Every (request, step) slot is scored by its survival probability, the running + product of that request's per-position confidences, and the highest scores win. + Survival only decreases along a request, so a global top-k always admits + continuously along steps with a request. + + `capacities` enters holding each request's scheduled draft count (which bounds its + eligible slots) and leaves holding the admitted count. The caller only calls this + when `draft_budget < capacities.sum()`, so every winner is a real draft slot. + """ + survival = confidence_probs[idx_mapping].cumprod(dim=1) + steps = torch.arange(num_steps, device=survival.device) + # Out-of-range slots score -inf so they never outrank a real draft. + out_of_range = steps[None, :] >= capacities[:, None] + survival = survival.masked_fill(out_of_range, -float("inf")) + flat = survival.flatten() + winners = flat.topk(draft_budget).indices + admitted = torch.zeros_like(flat, dtype=torch.bool).index_fill_(0, winners, True) + torch.sum(admitted.view_as(survival), dim=1, dtype=capacities.dtype, out=capacities) + + +_assign_draft_token_budget_compiled = torch.compile( + _assign_draft_token_budget, dynamic=True +) + + +def build_cost_tables_from_curves( + draft_curve: list[tuple[int, float]], + verify_curve: list[tuple[int, float]], + max_num_reqs: int, + max_batch_tokens: int, + cudagraph_limit: int = 0, +) -> tuple[np.ndarray, np.ndarray]: + """Build cost tables: graph-padded below the capture limit, smooth above. + + Args: + cudagraph_limit: Largest cudagraph-captured size. At or below it, + execution pads up to the next captured size, so cost is a step + function. Above it there is no padding, so cost is continuous. + """ + + def build_table(limit: int, curve: list[tuple[int, float]]) -> np.ndarray: + xs, ys = np.asarray(curve, dtype=np.float64).T + ys = np.maximum.accumulate(ys) + values = np.arange(limit + 1) + # Execution pads to the next captured size, so cost is a step + # function of the padded size: smooth interpolation would invent + # marginal per-token costs that don't exist within a pad bucket. + idx = np.searchsorted(xs, values, side="left") + result = ys[np.minimum(idx, len(xs) - 1)] + # Past the capture limit nothing pads, so cost really is continuous in + # size; snapping to the next profiled point overestimates badly when + # the profiled points are far apart. Interpolate only between points + # that are themselves past the limit: crossing the limit loses + # cudagraphs entirely, which is a genuine discontinuity, so the first + # point above it must not be blended with the last one below. + if cudagraph_limit: + smooth = values > cudagraph_limit + above = xs > cudagraph_limit + if smooth.any() and above.any(): + result[smooth] = np.interp(values[smooth], xs[above], ys[above]) + if len(xs) > 1: + after = values > xs[-1] + slope = (ys[-1] - ys[-2]) / (xs[-1] - xs[-2]) + result[after] = ys[-1] + slope * (values[after] - xs[-1]) + return result + + draft_table = np.maximum(build_table(max_num_reqs, draft_curve), 0.0) + verify_table = np.maximum(build_table(max_batch_tokens, verify_curve), 1e-6) + return draft_table, verify_table + + +class AdaptiveVerificationManager: + def __init__( + self, + req_states: "RequestState", + query_start_loc: torch.Tensor, + num_bonus_tokens: int, + max_total_logits: int, + ): + self.req_states = req_states + self.num_speculative_steps = req_states.num_speculative_steps + device = req_states.device + self._copy_stream = torch.cuda.Stream(device) + + self.num_bonus_tokens = num_bonus_tokens + # Rejection sampling verifies logits in one contiguous chunk; the + # chunked path indexes by scheduled (untrimmed) offsets and cannot + # address the compacted layout, so the budget must fit one chunk. + self._max_total_logits = max_total_logits + self.query_start_loc = query_start_loc + self.cost_tables: tuple[np.ndarray, np.ndarray] | None = None + # Largest cudagraph-captured token count; above it nothing pads. + self._cudagraph_limit = 0 + self._batch_budget: tuple[dict[str, int], dict[str, int], int] | None = None + max_num_reqs = req_states.max_num_reqs + # Current per-slot confidences + self._confidence_probs = torch.empty( + (max_num_reqs, self.num_speculative_steps), + dtype=torch.float32, + device=device, + ) + self._batch_draft_capacity = torch.empty( + max_num_reqs, dtype=torch.int32, device=device + ) + self._num_non_draft_tokens = torch.empty_like(query_start_loc[:-1]) + self._cu_num_logits = torch.empty_like(query_start_loc) + + # Two D2H slots preserve stale inputs for budget selection. + self._stale_confidences = [ + CpuGpuBuffer( + max_num_reqs, + self.num_speculative_steps, + dtype=torch.float32, + device=device, + ) + for _ in range(2) + ] + self._copy_events = [torch.cuda.Event(blocking=True) for _ in range(2)] + self._pending_resets: list[int] = [] + self._stale_idx = 0 + for slot in self._stale_confidences: + slot.np.fill(1.0) + + def add_request(self, req_idx: int) -> None: + self._stale_confidences[self._stale_idx].np[req_idx].fill(1.0) + self._pending_resets.append(req_idx) + self._confidence_probs[req_idx].fill_(1.0) + + def batches_to_profile(self, capture_sizes: list[int]) -> Iterator[dict[str, int]]: + """Dummy-run kwargs whose step timings seed the cost tables. + + Run these inside StepTimingCollector.collect(), then hand the block's + timings to set_initial_cost_curves.""" + max_num_tokens = self.req_states.max_num_batched_tokens + size = self._cudagraph_limit = capture_sizes[-1] if capture_sizes else 0 + # Also profile beyond the capture limit: real steps run there + # (piecewise/eager) and linear extrapolation badly underestimates + # them. These runs double as JIT warmup for the piecewise shapes. + tail_sizes: set[int] = set() + if size: + tail_sizes.add(min(size + size // 2, max_num_tokens)) + while size < max_num_tokens: + size = min(size * 2, max_num_tokens) + tail_sizes.add(size) + tail_sizes -= set(capture_sizes) + for num_tokens in capture_sizes + sorted(tail_sizes): + for _ in range(_PROFILE_REPLAYS): + yield { + "num_tokens": num_tokens, + "context_len": envs.VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN, + } + + def set_initial_cost_curves(self, samples: list[StepTimingSample]) -> None: + def median_curve( + points: Iterable[tuple[int, float]], + ) -> list[tuple[int, float]]: + grouped: defaultdict[int, list[float]] = defaultdict(list) + for key, value in points: + grouped[key].append(value) + return [(k, float(np.median(v))) for k, v in sorted(grouped.items())] + + # Draft curve: eager-target steps inflate drafter timings (the CPU is + # still launching kernels, opening gaps between the drafter's events), + # and request counts — unlike token counts — collide across execution + # modes, so only graph-replay samples may price the draft curve. + draft_curve = median_curve( + (s.num_reqs, s.drafter_ms) for s in samples if s.full_cudagraph + ) + verify_curve = median_curve( + (s.num_target_tokens, s.forward_ms) for s in samples + ) + self.set_cost_curves(draft_curve, verify_curve) + + def set_cost_curves( + self, + draft_curve: list[tuple[int, float]], + verify_curve: list[tuple[int, float]], + ) -> None: + draft_curve, verify_curve = get_tp_group().broadcast_object( + (draft_curve, verify_curve), src=0 + ) + if not draft_curve or not verify_curve: + raise RuntimeError( + "Adaptive verification could not profile step costs. Pass " + "`enable_adaptive_verification=false` in the speculative config to " + "verify a fixed number of drafts instead." + ) + self.cost_tables = build_cost_tables_from_curves( + draft_curve, + verify_curve, + self.req_states.max_num_reqs, + self.req_states.max_num_batched_tokens, + self._cudagraph_limit, + ) + logger.debug("DSpark cost tables: %s", self.cost_tables) + + def record_confidences( + self, + confidence_probs: torch.Tensor, + input_batch: "InputBatch", + ) -> None: + """Publish this step's raw confidences for the ranking kernel and start + copying them to the CPU, where a later step's budget reads them.""" + num_reqs = input_batch.num_reqs + ready_idx = self._stale_idx ^ 1 + with gpu_sync_allowed(): + self._copy_events[ready_idx].synchronize() + if self._pending_resets: + self._stale_confidences[ready_idx].np[self._pending_resets] = 1.0 + self._pending_resets.clear() + # Last step's copy has landed: budgets read it, this step overwrites the + # slot they were reading before. + self._stale_idx, write_idx = ready_idx, self._stale_idx + + self._confidence_probs[input_batch.idx_mapping] = confidence_probs[:num_reqs] + write_slot = self._stale_confidences[write_idx] + write_slot.gpu.copy_(self._confidence_probs) + + current_stream = torch.cuda.current_stream(self.req_states.device) + self._copy_stream.wait_stream(current_stream) + with stream(self._copy_stream, current_stream): + write_slot.copy_to_cpu() + self._copy_events[write_idx].record() + + def get_num_tokens( + self, + num_tokens_per_req: dict[str, int], + draft_tokens: dict[str, list[int]], + ) -> int: + """Token count once the draft budget is trimmed to fit. + + Stashes the chosen budget in ``_batch_budget`` for the compaction and + reallocation that follow in the same step. + """ + assert self.cost_tables is not None + req_ids = list(num_tokens_per_req) + num_reqs = len(req_ids) + scheduled_tokens = np.fromiter( + num_tokens_per_req.values(), dtype=np.int32, count=num_reqs + ) + scheduled_drafts = np.fromiter( + (len(draft_tokens.get(req_id, ())) for req_id in req_ids), + dtype=np.int32, + count=num_reqs, + ) + num_non_draft_tokens = scheduled_tokens - scheduled_drafts + slots = np.fromiter( + (self.req_states.req_id_to_index[req_id] for req_id in req_ids), + dtype=np.int32, + count=len(req_ids), + ) + stale_confidences = self._stale_confidences[self._stale_idx].np[slots] + survival_probability = np.cumprod(stale_confidences.astype(np.float64), axis=1) + steps = np.arange(self.num_speculative_steps) + valid = steps[None, :] < scheduled_drafts[:, None] + scores = np.sort(survival_probability[valid])[::-1] + num_non_draft_tokens_total = int(num_non_draft_tokens.sum()) + max_draft_budget = min( + int(scheduled_drafts.sum()), + max(0, self._max_total_logits - num_reqs * self.num_bonus_tokens), + ) + scores = scores[:max_draft_budget] + draft_cost_ms, verify_cost_ms = self.cost_tables + num_sampling_requests = np.count_nonzero( + self.req_states.num_computed_tokens_np[slots] + num_non_draft_tokens + >= self.req_states.prefill_len.np[slots] + ) + num_tokens_to_estimated_accepted_tokens = np.concatenate( + ([num_sampling_requests], num_sampling_requests + np.cumsum(scores)) + ) + costs = ( + draft_cost_ms[len(req_ids)] + + verify_cost_ms[ + num_non_draft_tokens_total : num_non_draft_tokens_total + + max_draft_budget + + 1 + ] + ) + num_drafts_per_req = { + req_id: int(num_drafts) + for req_id, num_drafts in zip(req_ids, scheduled_drafts, strict=True) + } + num_non_draft_tokens_per_req = { + req_id: int(num_tokens) + for req_id, num_tokens in zip(req_ids, num_non_draft_tokens, strict=True) + } + draft_budget = int(np.argmax(num_tokens_to_estimated_accepted_tokens / costs)) + self._batch_budget = ( + num_drafts_per_req, + num_non_draft_tokens_per_req, + draft_budget, + ) + return sum(num_non_draft_tokens_per_req.values()) + draft_budget + + def compact_batch( + self, + num_draft_tokens_per_req: np.ndarray, + num_scheduled_tokens: np.ndarray, + cu_num_logits_np: np.ndarray, + ) -> tuple[np.ndarray, np.ndarray]: + """Compact the CPU batch to the chosen draft budget. + + Returns the compacted per-request token counts and the CPU cu_num_logits_np. + If the draft budget is 0, we can know cu_num_logits_np exactly, otherwise + its unchanged/an-upper-bound. + """ + batch_budget = self._batch_budget + assert batch_budget is not None + _, _, draft_budget = batch_budget + num_drafts = int(num_draft_tokens_per_req.sum()) + if draft_budget == num_drafts: + return num_scheduled_tokens, cu_num_logits_np + + num_non_draft_tokens = num_scheduled_tokens - num_draft_tokens_per_req + if draft_budget == 0: + # The draft budget is 0, so we can know cu_num_logits_np exactly. This helps + # when we would exceed the sampler logit chunk size. + num_reqs = num_scheduled_tokens.shape[0] + cu_num_logits_np = ( + np.arange(num_reqs + 1, dtype=cu_num_logits_np.dtype) + * self.num_bonus_tokens + ) + return num_non_draft_tokens, cu_num_logits_np + + is_verification_request = num_draft_tokens_per_req > 0 + num_verification_reqs = int(is_verification_request.sum()) + # sort_batch_req_ids keeps verification requests at the front. + assert np.all(is_verification_request[:num_verification_reqs]) + # for the CPU side buffer we distribute draft tokens evenly + draft_lens_cpu = np.zeros_like(num_non_draft_tokens) + draft_lens_cpu[:num_verification_reqs] = draft_budget // num_verification_reqs + draft_lens_cpu[: draft_budget % num_verification_reqs] += 1 + return num_non_draft_tokens + draft_lens_cpu, cu_num_logits_np + + def reallocate_drafts( + self, req_ids: list[str], idx_mapping: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, int]: + batch_budget, self._batch_budget = self._batch_budget, None + assert batch_budget is not None + num_drafts_per_req, num_non_draft_tokens_per_req, draft_budget = batch_budget + num_reqs = idx_mapping.shape[0] + scheduled_drafts = np.fromiter( + (num_drafts_per_req[req_id] for req_id in req_ids), + dtype=np.int32, + count=num_reqs, + ) + num_non_draft_tokens = np.fromiter( + (num_non_draft_tokens_per_req[req_id] for req_id in req_ids), + dtype=np.int32, + count=num_reqs, + ) + num_tokens = int(num_non_draft_tokens.sum()) + draft_budget + + # Rank draft slots by survival probability and admit the best prefix. + # capacities enters holding each request's valid draft count (the kernel + # uses it to bound eligible slots) and leaves holding the admitted count. + capacities = self._batch_draft_capacity[:num_reqs] + if draft_budget == 0: + capacities.zero_() + else: + async_copy_to_gpu(scheduled_drafts, out=capacities) + if draft_budget < int(scheduled_drafts.sum()): + _assign_draft_token_budget_compiled( + self._confidence_probs, + idx_mapping, + capacities, + draft_budget, + self.num_speculative_steps, + ) + + num_non_draft_tokens_gpu = self._num_non_draft_tokens[:num_reqs] + async_copy_to_gpu( + num_non_draft_tokens, + out=num_non_draft_tokens_gpu, + ) + self._cu_num_logits[:1].zero_() + torch.cumsum( + capacities + self.num_bonus_tokens, + dim=0, + out=self._cu_num_logits[1 : num_reqs + 1], + ) + self.query_start_loc[:1].zero_() + torch.cumsum( + capacities + num_non_draft_tokens_gpu, + dim=0, + out=self.query_start_loc[1 : num_reqs + 1], + ) + self.query_start_loc[num_reqs + 1 :].fill_(num_tokens) + return ( + self._cu_num_logits[: num_reqs + 1], + self.query_start_loc, + draft_budget, + ) + + +def maybe_create_adaptive_verification_manager( + *, + enable_adaptive_verification: bool, + attn_groups: list[list["AttentionGroup"]], + attn_cg_support: "AttentionCGSupportInfo", + req_states: "RequestState", + query_start_loc: torch.Tensor, + num_bonus_tokens: int, + max_total_logits: int, +) -> AdaptiveVerificationManager | None: + if not enable_adaptive_verification: + return None + + # The selector rejects unsupported backends, but models that + # hard-wire theirs (e.g. DeepSeek-V4) never go through it. + backend = get_query_lens_mismatch_unsupported_backend(attn_groups) + if backend is not None: + raise ValueError( + "Adaptive verification trims verification requests on device, which" + f" the {backend} attention backend does not support. Pass " + "enable_adaptive_verification=false in the speculative config, or " + "use a backend that does." + ) + + if attn_cg_support.min_cg_support != AttentionCGSupport.ALWAYS: + raise ValueError( + "Adaptive verification captures varlen decode cudagraphs, so every" + " attention builder must report AttentionCGSupport.ALWAYS, but " + f"{attn_cg_support.min_cg_attn_backend} reports " + f"{attn_cg_support.min_cg_support}. Pass " + "enable_adaptive_verification=false in the speculative config, or " + "use a backend that does." + ) + + return AdaptiveVerificationManager( + req_states, + query_start_loc, + num_bonus_tokens, + max_total_logits=max_total_logits, + ) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index 333648637b..97c284c03d 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -135,11 +135,10 @@ class DFlashSpeculator(DraftModelSpeculator): def capture(self) -> None: logger.info("Capturing model for %s speculator...", self._speculator_name) - # Reset sampling indices to zero to prevent stale values from prior - # dummy runs from being baked into the captured graph. + # Padded sample rows must not scatter into a live request during capture. self.sample_indices.zero_() self.sample_pos.zero_() - self.sample_idx_mapping.zero_() + self.sample_idx_mapping.fill_(-1) assert self.query_cudagraph_manager is not None self.query_cudagraph_manager.capture( self._generate_draft, @@ -589,7 +588,10 @@ def _prepare_dflash_inputs_kernel( # seq_lens is the absolute sequence length the draft attention # reads up to (context + query), not just the count of accepted # tokens this step. - tl.store(out_seq_lens_ptr + req_idx, last_valid_pos + 1 + num_query_per_req) + tl.store( + out_seq_lens_ptr + req_idx, + tl.minimum(last_valid_pos + 1 + num_query_per_req, max_model_len), + ) # Copy sampling state. tl.store( out_temperature_ptr + req_state_idx, diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index 7f48a3689f..19fc452018 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -29,10 +29,13 @@ import torch from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode +from vllm.logger import init_logger from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator from vllm.v1.worker.gpu.spec_decode.dspark.utils import load_dspark_model +logger = init_logger(__name__) + class DSparkSpeculator(DFlashSpeculator): _speculator_name = "DSpark" @@ -76,6 +79,13 @@ class DSparkSpeculator(DFlashSpeculator): self.draft_model_config.hf_config, "dspark_draft_topk", None ) + self.draft_token_confidence_probs = torch.empty_like( + self.draft_tokens, dtype=torch.float32 + ) + self.enable_adaptive_verification = ( + self.speculative_config.enable_adaptive_verification + ) + def load_draft_model( self, target_model: torch.nn.Module, @@ -98,6 +108,13 @@ class DSparkSpeculator(DFlashSpeculator): dtype=self.draft_logits.dtype, device=self.device, ) + if self.enable_adaptive_verification and model.model.confidence_head is None: + raise ValueError( + "Adaptive verification needs a DSpark checkpoint with a confidence " + "head, and this one has none. Pass " + "enable_adaptive_verification=false in the speculative config to verify" + " a fixed number of drafts instead." + ) return model def _sample_logits( @@ -149,6 +166,7 @@ class DSparkSpeculator(DFlashSpeculator): idx_map = self.sample_idx_mapping[:num_sample].view(num_reqs, n_spec) sample_pos = self.sample_pos[:num_sample].view(num_reqs, n_spec) + confidence_markov_embeds = [] # Anchor (bonus) token per request = the input id at query offset 0, # read via the precomputed persistent index (fixed buffer for capture). @@ -157,6 +175,8 @@ class DSparkSpeculator(DFlashSpeculator): for i in range(n_spec): # Sequential stage: Markov bias from the previously sampled token. markov_embed = self.model.markov_embed(prev) + if self.enable_adaptive_verification: + confidence_markov_embeds.append(markov_embed) bias = self.model.markov_bias(markov_embed) logits_i = base_logits[:, i] + bias draft_sampled_i = self._sample_logits( @@ -165,6 +185,15 @@ class DSparkSpeculator(DFlashSpeculator): self.draft_tokens[:num_reqs, i] = draft_sampled_i prev = draft_sampled_i + if self.enable_adaptive_verification: + confidence = self.model.compute_confidence( + sample_hidden, + torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1), + ) + self.draft_token_confidence_probs[:num_reqs] = confidence.view( + num_reqs, n_spec + ) + def _sample_sequential_topk(self, num_reqs: int, head_hidden: torch.Tensor) -> None: """Apply the sequential Markov head only to top-k base-logit candidates. @@ -186,10 +215,13 @@ class DSparkSpeculator(DFlashSpeculator): base_logits.fill_(float("-inf")) idx_map = self.sample_idx_mapping[:num_sample].view(num_reqs, n_spec) sample_pos = self.sample_pos[:num_sample].view(num_reqs, n_spec) + confidence_markov_embeds = [] prev = self.input_buffers.input_ids[self._anchor_idx[:num_reqs]] for i in range(n_spec): markov_embed = self.model.markov_embed(prev) + if self.enable_adaptive_verification: + confidence_markov_embeds.append(markov_embed) logits_i = self.model.apply_markov_bias_gathered( markov_embed, base_logits[:, i], @@ -202,6 +234,15 @@ class DSparkSpeculator(DFlashSpeculator): self.draft_tokens[:num_reqs, i] = draft_sampled_i prev = draft_sampled_i + if self.enable_adaptive_verification: + confidence = self.model.compute_confidence( + sample_hidden, + torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1), + ) + self.draft_token_confidence_probs[:num_reqs] = confidence.view( + num_reqs, n_spec + ) + def _generate_draft( self, num_reqs: int, diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index a4be95be00..8410b6ce87 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Iterator +from collections.abc import Iterable, Iterator import numpy as np import torch @@ -32,6 +32,11 @@ MAX_CHUNK_BYTES = 2**30 # 1GB _FP32_BYTES = 4 +def get_max_chunk_logits(vocab_size: int) -> int: + """Largest number of logits rows one verification chunk may hold.""" + return max(1, MAX_CHUNK_BYTES // (vocab_size * _FP32_BYTES)) + + def _iter_request_chunks( cu_num_logits: np.ndarray, max_chunk_logits: int ) -> Iterator[tuple[int, int]]: @@ -177,11 +182,22 @@ class RejectionSampler: ) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]: cu_num_logits_np = input_batch.cu_num_logits_np use_processed_logits = self.sampler.logprobs_mode in PROCESSED_LOGPROBS_MODES + num_reqs = input_batch.num_reqs + + if logits.shape[0] <= max_chunk_logits: + # One chunk covers the batch. Adaptive verification compacts the logits + # without updating cu_num_logits_np (it keeps the pre-compacted layout), + # so the stale sums must not pick chunk boundaries; its budget cap + # guarantees the compacted batch always lands here. + request_chunks: Iterable[tuple[int, int]] = ((0, num_reqs),) + else: + request_chunks = _iter_request_chunks(cu_num_logits_np, max_chunk_logits) + sampled_chunks: list[torch.Tensor] = [] num_sampled_chunks: list[torch.Tensor] = [] logprobs_chunks: list[LogprobsTensors] = [] - for start, end in _iter_request_chunks(cu_num_logits_np, max_chunk_logits): + for start, end in request_chunks: lo = int(cu_num_logits_np[start]) hi = int(cu_num_logits_np[end]) chunk_cu_num_logits_np = cu_num_logits_np[start : end + 1] - lo @@ -246,14 +262,14 @@ class RejectionSampler: max_num_logprobs = self.sampler.sampling_states.max_num_logprobs( input_batch.idx_mapping_np ) - max_chunk_logits = max(1, MAX_CHUNK_BYTES // (logits.shape[1] * _FP32_BYTES)) + chunk_logit_limit = get_max_chunk_logits(logits.shape[1]) sampled, num_sampled, logprobs_tensors = self._verify_in_chunks( logits, input_batch, draft_logits, draft_sampled, pos, - max_chunk_logits, + chunk_logit_limit, max_num_logprobs, ) diff --git a/vllm/v1/worker/gpu/structured_outputs.py b/vllm/v1/worker/gpu/structured_outputs.py index 34f00086d2..6b55b65c44 100644 --- a/vllm/v1/worker/gpu/structured_outputs.py +++ b/vllm/v1/worker/gpu/structured_outputs.py @@ -10,7 +10,13 @@ from vllm.v1.worker.gpu.input_batch import InputBatch class StructuredOutputsWorker: - def __init__(self, max_num_logits: int, vocab_size: int, device: torch.device): + def __init__( + self, + max_num_logits: int, + vocab_size: int, + device: torch.device, + mask_stride: int, + ): self.logits_indices = torch.zeros( max_num_logits, dtype=torch.int32, device=device ) @@ -19,6 +25,7 @@ class StructuredOutputsWorker: ) self.device = device self.copy_stream = torch.cuda.Stream() + self.mask_stride = mask_stride def apply_grammar_bitmask( self, @@ -45,7 +52,13 @@ class StructuredOutputsWorker: req_idx = req_id_to_idx[grammar_req_id] logits_start_idx = cu_num_logits[req_idx] logits_end_idx = cu_num_logits[req_idx + 1] - mapping.extend(range(logits_start_idx, logits_end_idx)) + # Key by (request, position) rather than absolute logit index: + # adaptive verification finalizes per-request logit offsets on + # device, so the kernel resolves them from the GPU cu_num_logits. + mapping.extend( + req_idx * self.mask_stride + position + for position in range(logits_end_idx - logits_start_idx) + ) # Asynchronously copy the mapping to GPU. with torch.cuda.stream(self.copy_stream): @@ -69,9 +82,11 @@ class StructuredOutputsWorker: logits, logits.stride(0), logits_indices, + input_batch.cu_num_logits, bitmask, bitmask.stride(0), vocab_size, + MASK_STRIDE=self.mask_stride, BLOCK_SIZE=BLOCK_SIZE, ) @@ -87,13 +102,21 @@ def _apply_grammar_bitmask_kernel( logits_ptr, logits_stride, logits_indices_ptr, + cu_num_logits_ptr, bitmask_ptr, bitmask_stride, vocab_size, + MASK_STRIDE: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): bitmask_idx = tl.program_id(0) - logits_idx = tl.load(logits_indices_ptr + bitmask_idx) + mapping_idx = tl.load(logits_indices_ptr + bitmask_idx) + req_idx = mapping_idx // MASK_STRIDE + position_idx = mapping_idx % MASK_STRIDE + logits_idx = tl.load(cu_num_logits_ptr + req_idx) + num_req_logits = tl.load(cu_num_logits_ptr + req_idx + 1) - logits_idx + logits_idx += position_idx + position_is_active = position_idx < num_req_logits # Load the bitmask. block_id = tl.program_id(1) @@ -111,5 +134,5 @@ def _apply_grammar_bitmask_kernel( tl.store( logits_ptr + logits_idx * logits_stride + block_offset, -float("inf"), - mask=bitmask & (block_offset < vocab_size), + mask=position_is_active & bitmask & (block_offset < vocab_size), )