From a4eb3f25d6f9b3cad7ecf5390423d853935fcaeb Mon Sep 17 00:00:00 2001 From: qizixi <22851944+zixi-qi@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:01:17 -0700 Subject: [PATCH] [Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263) Signed-off-by: zixi-qi Co-authored-by: OpenAI Codex Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Nick Hill --- docs/design/cuda_graphs.md | 2 + .../test_dcp_direct_a2a_lse_reduce.py | 22 +- .../models/language/generation/test_gemma.py | 69 +++- tests/test_config.py | 12 + tests/transformers_utils/test_config.py | 45 +++ tests/v1/attention/test_attention_backends.py | 376 ++++++++++++++++++ tests/v1/cudagraph/test_cudagraph_manager.py | 24 ++ .../spec_decode/test_adaptive_verification.py | 120 ++++-- tests/v1/worker/test_attn_utils.py | 43 +- vllm/config/speculative.py | 2 + vllm/model_executor/models/gemma4_dspark.py | 25 +- vllm/models/minimax_m3/amd/indexer_aiter.py | 11 +- vllm/transformers_utils/config.py | 15 +- vllm/v1/attention/backend.py | 51 ++- vllm/v1/attention/backends/flashinfer.py | 74 +++- vllm/v1/attention/backends/mla/index_group.py | 12 +- vllm/v1/attention/backends/mla/sparse_swa.py | 11 +- vllm/v1/attention/ops/dcp.py | 14 +- vllm/v1/hisparse/runtime.py | 17 +- vllm/v1/worker/gpu/attn_utils.py | 34 ++ vllm/v1/worker/gpu/cudagraph_utils.py | 3 +- vllm/v1/worker/gpu/dp_utils.py | 3 +- vllm/v1/worker/gpu/model_runner.py | 7 +- .../gpu/spec_decode/adaptive_verification.py | 37 +- 24 files changed, 894 insertions(+), 135 deletions(-) diff --git a/docs/design/cuda_graphs.md b/docs/design/cuda_graphs.md index e274b68c70..abebdd54ae 100644 --- a/docs/design/cuda_graphs.md +++ b/docs/design/cuda_graphs.md @@ -172,6 +172,8 @@ class AttentionCGSupport(enum.Enum): Suppose we have hybrid attention backends (e.g., in mamba mixer models). In that case, we seek the minimum capability of all backends to determine the final capability of the model, and we might resolve the incompatible CUDA Graphs mode by downgrading the mode to the best fit one. For example, downgrading `FULL` mode to `FULL_AND_PIECEWISE` mode if the minimum capability is `UNIFORM_BATCH`, or `PIECEWISE` mode if the minimum capability is `NEVER` for -O3 compilation mode. For the complete fallback policy, please see the code for [this][vllm.v1.worker.gpu_model_runner.GPUModelRunner._check_and_update_cudagraph_mode]. +Variable-length decode batches, where each request carries a different number of query tokens read from the device `query_start_loc` (as in adaptive verification), are declared separately and not ordered against the enum: `AttentionMetadataBuilder.get_varlen_cudagraph_max_query_len()` returns the largest per-request query length a FULL decode graph can replay. It returns `None` for builders reporting `ALWAYS`, which replay any batch, and for builders that cannot replay variable-length batches; FlashInfer returns `1 + num_speculative_tokens` when its TRTLLM-GEN varlen decode path is active. Batches with a prefill never replay these graphs. + The following table lists backends that support full CUDA Graphs at the time of writing. | Attention Backend | cudagraph_support | Comments | diff --git a/tests/distributed/test_dcp_direct_a2a_lse_reduce.py b/tests/distributed/test_dcp_direct_a2a_lse_reduce.py index e5764954bd..b3d9fdddd3 100644 --- a/tests/distributed/test_dcp_direct_a2a_lse_reduce.py +++ b/tests/distributed/test_dcp_direct_a2a_lse_reduce.py @@ -455,13 +455,29 @@ def test_mla_dcp_manager_selects_fallback_backends(monkeypatch): ) -def test_dcp_workspace_covers_parallel_drafting(): +@pytest.mark.parametrize( + ("parallel_drafting", "use_v2_model_runner", "expected"), + [ + # Model runner V1's parallel drafter widens requests to 1 + 2 * 3. + (True, False, 28), + (True, True, 16), + # A diffusion canvas of 3 without a speculative config. + (None, True, 16), + ], +) +def test_dcp_workspace_covers_decode_width( + parallel_drafting, use_v2_model_runner, expected +): config = _manager_config() config.scheduler_config.max_num_batched_tokens = 128 config.num_speculative_tokens = 3 - config.speculative_config = MagicMock(parallel_drafting=True) + config.use_v2_model_runner = use_v2_model_runner + if parallel_drafting is not None: + config.speculative_config = MagicMock( + num_speculative_tokens=3, parallel_drafting=parallel_drafting + ) - assert dcp.get_dcp_workspace_max_num_tokens(config) == 28 + assert dcp.get_dcp_workspace_max_num_tokens(config) == expected def test_mla_dcp_manager_selects_pcp_combine(monkeypatch): diff --git a/tests/models/language/generation/test_gemma.py b/tests/models/language/generation/test_gemma.py index 5cffe6fb19..a8fe4ab951 100644 --- a/tests/models/language/generation/test_gemma.py +++ b/tests/models/language/generation/test_gemma.py @@ -7,7 +7,8 @@ import numpy as np import pytest import torch -from vllm.config import VllmConfig +from vllm.config import CompilationConfig, VllmConfig +from vllm.config.compilation import CompilationMode from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding from vllm.model_executor.models import gemma from vllm.model_executor.models.gemma3n import ( @@ -18,10 +19,72 @@ from vllm.model_executor.models.gemma4 import ( Gemma4ForCausalLM, _gemma4_layer_weights_mapper, ) +from vllm.model_executor.models.gemma4_dspark import ( + Gemma4DSparkForCausalLM, + Gemma4DSparkModel, +) MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"] +@pytest.mark.usefixtures("dist_init") +@pytest.mark.parametrize( + "enabled,has_weights,with_markov", + [ + pytest.param(True, True, False, id="hidden-only"), + pytest.param(True, True, True, id="with-markov"), + pytest.param(True, False, True, id="missing-weights"), + pytest.param(False, True, True, id="disabled-head"), + ], +) +def test_gemma4_dspark_loads_confidence_head( + monkeypatch, enabled, has_weights, with_markov +) -> None: + """Use checkpoint confidence parameters, or disable an unavailable head.""" + config = SimpleNamespace( + vocab_size=64, + hidden_size=8, + target_layer_ids=[0, 1], + num_hidden_layers=0, + rms_norm_eps=1e-6, + markov_rank=4, + enable_confidence_head=enabled, + confidence_head_with_markov=with_markov, + ) + vllm_config = SimpleNamespace( + compilation_config=CompilationConfig(mode=CompilationMode.NONE), + model_config=SimpleNamespace(dtype=torch.bfloat16), + speculative_config=SimpleNamespace( + draft_model_config=SimpleNamespace(hf_config=config), + ), + ) + monkeypatch.setattr(Gemma4DSparkModel, "_build_fused_kv_buffers", lambda _: None) + model = Gemma4DSparkForCausalLM(vllm_config=cast(VllmConfig, vllm_config)) + width = config.hidden_size + (config.markov_rank if with_markov else 0) + weight = torch.arange(width, dtype=torch.bfloat16).reshape(1, -1) / 16 + bias = torch.tensor([-0.5], dtype=torch.bfloat16) + loaded = model.load_weights( + [("confidence_head.proj.weight", weight), ("confidence_head.proj.bias", bias)] + if has_weights + else [] + ) + if not (enabled and has_weights): + assert model.model.confidence_head is None + assert not loaded + return + + assert loaded == { + "model.confidence_head.proj.weight", + "model.confidence_head.proj.bias", + } + assert model.model.confidence_head.proj.weight.dtype == torch.float32 + hidden = torch.full((3, config.hidden_size), 0.25, dtype=torch.bfloat16) + markov = torch.full((3, config.markov_rank), -0.5, dtype=torch.bfloat16) + inputs = torch.cat([hidden, markov], dim=-1) if with_markov else hidden + expected = (inputs.float() @ weight.float().T + bias.float()).sigmoid().squeeze(-1) + torch.testing.assert_close(model.compute_confidence(hidden, markov), expected) + + @pytest.mark.cpu_test @pytest.mark.usefixtures("dist_init") def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None: @@ -164,7 +227,9 @@ def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None: ) as llm: if model == "google/gemma-3-4b-it": normalizers = llm.llm.collective_rpc( - lambda self: self.model_runner.model.language_model.model.normalizer.cpu().item() # noqa: E501 + lambda self: ( + self.model_runner.model.language_model.model.normalizer.cpu().item() + ) # noqa: E501 ) config = llm.llm.llm_engine.model_config.hf_config.text_config else: diff --git a/tests/test_config.py b/tests/test_config.py index fe4d8ef343..bbb12e3917 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -92,6 +92,16 @@ def test_nested_rope_validation_patch_preserves_flat_rope_parameters(monkeypatch assert len(calls) == 2 +def test_dspark_adaptive_verification_separates_graph_cache(): + config = object.__new__(SpeculativeConfig) + config.method = "dspark" + config.draft_model_config = None + config.enable_adaptive_verification = False + fixed_hash = config.compute_hash() + config.enable_adaptive_verification = True + assert config.compute_hash() != fixed_hash + + def _write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value), encoding="utf-8") @@ -878,6 +888,8 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1( ("FULL_AND_PIECEWISE", False, "ALWAYS", "FULL_DECODE_ONLY"), ("FULL_DECODE_ONLY", False, "ALWAYS", "FULL_DECODE_ONLY"), ("FULL_DECODE_ONLY", False, "NEVER", "NONE"), + ("FULL_AND_PIECEWISE", True, "UNIFORM_BATCH", "FULL_AND_PIECEWISE"), + ("FULL", True, "UNIFORM_BATCH", "FULL_DECODE_ONLY"), ], ) def test_resolve_cudagraph_mode_uses_loaded_piecewise_provider( diff --git a/tests/transformers_utils/test_config.py b/tests/transformers_utils/test_config.py index 4dee8fab60..19773dfda4 100644 --- a/tests/transformers_utils/test_config.py +++ b/tests/transformers_utils/test_config.py @@ -5,6 +5,7 @@ only get the `eos_token_id` from the tokenizer as defined by `BaseRenderer.get_eos_token_id`. """ +import json import math from types import SimpleNamespace from typing import cast @@ -31,6 +32,50 @@ from vllm.transformers_utils.configs.glm5_next import ( from vllm.transformers_utils.configs.mistral import adapt_config_dict +@pytest.mark.parametrize("layout", ["mixed", "flat"]) +def test_gemma4_dspark_rope_config_preserves_parameters(tmp_path, layout): + """Remove redundant shared entries while preserving per-layer and flat RoPE.""" + from transformers import Gemma4TextConfig + + per_layer = { + "full_attention": { + "rope_type": "proportional", + "partial_rotary_factor": 0.25, + "rope_theta": 1000000.0, + }, + "sliding_attention": {"rope_type": "default", "rope_theta": 10000.0}, + } + rope_parameters: dict[str, object] = dict(per_layer) + if layout == "mixed": + rope_parameters.update(rope_type="default", rope_theta=None) + else: + rope_parameters = {"rope_type": "default", "rope_theta": 12345.0} + (tmp_path / "config.json").write_text( + json.dumps( + { + "model_type": "gemma4_text", + "architectures": ["Gemma4DSparkModel"], + "num_hidden_layers": 1, + "layer_types": ["full_attention"], + "rope_parameters": rope_parameters, + } + ) + ) + _, config = config_module.HFConfigParser().parse( + tmp_path, trust_remote_code=False, max_position_embeddings=8192 + ) + assert isinstance(config, Gemma4TextConfig) + assert config.name_or_path == str(tmp_path) + assert config.max_position_embeddings == 8192 + if layout == "flat": + assert config.rope_parameters["rope_theta"] == 12345.0 + else: + for layer_type, expected in per_layer.items(): + for key, value in expected.items(): + assert config.rope_parameters[layer_type][key] == value + assert set(config.rope_parameters) == set(per_layer) + + def test_patch_legacy_rope_type_preserves_nope_layers(): """NoPE layers stay disabled while later RoPE layers are normalized.""" rope_parameters = { diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py index b02c85f682..2f8b80a7c1 100644 --- a/tests/v1/attention/test_attention_backends.py +++ b/tests/v1/attention/test_attention_backends.py @@ -2,12 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Tests for v1 attention backends without GPUModelRunner dependency.""" +from dataclasses import replace from functools import partial from types import SimpleNamespace import pytest import torch from torch.nn.attention.flex_attention import create_block_mask, flex_attention +from transformers import LlamaConfig from tests.v1.attention.utils import ( BatchSpec, @@ -28,8 +30,10 @@ from vllm.utils.torch_utils import ( ) from vllm.v1.attention.backend import ( AttentionCGSupport, + AttentionMetadataBuilder, AttentionType, CommonAttentionMetadata, + max_decode_query_len, ) from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout @@ -1066,6 +1070,378 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets( assert q_cu_seq_lens.tolist() == qo_indptr_values +@pytest.mark.skipif( + AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, + reason="FlashInfer is not available.", +) +@pytest.mark.parametrize( + "adaptive,decode_kernel,dcp_size,expected_bound", + [ + (None, "TRTLLM_GEN", 1, None), + (False, "TRTLLM_GEN", 1, None), + (True, "TRTLLM_GEN", 1, 8), + (True, "XQA", 1, None), + (True, "TRTLLM_GEN", 2, None), + ], +) +def test_flashinfer_varlen_cudagraph_capability( + monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound +): + """Only the trtllm-gen path that adaptive verification enables replays + varlen decode graphs, up to the decode width; the uniform level is fixed.""" + from vllm.model_executor.layers.attention.chunked_local_attention import ( + create_chunked_local_attention_backend, + ) + from vllm.v1.attention.backends import flashinfer as fi + + config = SimpleNamespace( + attention_config=SimpleNamespace(use_non_causal=False), + speculative_config=SimpleNamespace( + enable_adaptive_verification=adaptive, + num_speculative_tokens=7, + parallel_drafting=True, + ) + if adaptive is not None + else None, + parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size), + model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32), + use_v2_model_runner=True, + ) + monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True) + monkeypatch.setattr( + fi.FlashInferMetadataBuilder, + "_get_flashinfer_trtllm_api_decode_kernel", + staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]), + ) + spec = FullAttentionSpec( + block_size=16, + num_kv_heads=4, + head_size=64, + dtype=torch.bfloat16, + ) + builder_cls = fi.FlashInferMetadataBuilder + assert builder_cls.get_cudagraph_support(config, spec) == ( + AttentionCGSupport.UNIFORM_BATCH + if dcp_size == 1 + else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + ) + bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec) + assert bound == expected_bound + if bound is not None: + # A wrapper forcing NEVER inherits the override but not the bound. + chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16) + wrapped_cls = chunked_local.get_builder_cls() + assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None + + +@pytest.mark.parametrize( + "use_v2_model_runner,parallel_drafting,expected", + [ + (True, None, 1), + (True, False, 8), + (True, True, 8), + (False, False, 8), + (False, True, 15), + ], +) +def test_spec_as_decode_width_follows_model_runner( + use_v2_model_runner, parallel_drafting, expected +): + """Only model runner V1's parallel drafter widens decode requests.""" + + class Builder(AttentionMetadataBuilder): + def __init__(self, vllm_config): + self.vllm_config = vllm_config + + def build(self, common_prefix_len, common_attn_metadata, fast_build=False): + raise NotImplementedError + + config = SimpleNamespace( + speculative_config=None + if parallel_drafting is None + else SimpleNamespace( + num_speculative_tokens=7, parallel_drafting=parallel_drafting + ), + parallel_config=SimpleNamespace(decode_context_parallel_size=1), + use_v2_model_runner=use_v2_model_runner, + ) + assert max_decode_query_len(config) == expected + builder = Builder(config) + builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True) + assert builder.reorder_batch_threshold == expected + + +@pytest.mark.skipif( + AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST + or not current_platform.is_device_capability_family(100), + reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs", +) +@pytest.mark.parametrize( + "head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads", + [ + pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"), + pytest.param( + 256, + 1024, + 40, + 32, + 16, + 8, + id="gemma4-sliding-gqa", + ), + pytest.param( + 512, + None, + 40, + 32, + 16, + 1, + id="gemma4-global-mqa", + ), + ], +) +def test_flashinfer_varlen_decode_graph_replays_changed_layout( + tmp_path, + monkeypatch, + head_size, + sliding_window, + num_tokens, + num_reqs, + num_heads, + num_kv_heads, +): + """Replay changes device offsets, active slots, context, and KV writes. + + The graph is captured with evenly spread dummy queries and an explicit + bound of eight. CPU offsets then remain stale while device offsets change. + Compare real output rows with independent FP32 causal/windowed attention, + and verify cache writes including preservation of the null padding block. + """ + from vllm.v1.attention.backends import flashinfer as fi + from vllm.v1.attention.backends.utils import PerLayerParameters + + set_random_seed(0) + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + block_size = 16 + dtype = torch.bfloat16 + LlamaConfig( + architectures=["LlamaForCausalLM"], + hidden_size=num_heads * head_size, + head_dim=head_size, + num_attention_heads=num_heads, + num_key_value_heads=num_kv_heads, + num_hidden_layers=1, + max_position_embeddings=4096, + ).save_pretrained(tmp_path) + num_blocks = 1 + num_reqs * cdiv(2112, block_size) + config = create_vllm_config( + model_name=str(tmp_path), + max_model_len=4096, + dtype=dtype, + num_gpu_blocks=num_blocks, + block_size=block_size, + max_num_seqs=32, + max_num_batched_tokens=256, + ) + config.cache_config.kv_cache_layout = "BLHNC" + config.speculative_config = SimpleNamespace( + num_speculative_tokens=7, + enable_adaptive_verification=True, + parallel_drafting=False, + ) + spec = create_standard_kv_cache_spec(config) + scale = head_size**-0.5 + monkeypatch.setattr( + fi, + "get_per_layer_parameters", + lambda *_: { + "layer": PerLayerParameters( + window_left=-1 if sliding_window is None else sliding_window - 1, + logits_soft_cap=0.0, + sm_scale=scale, + has_sinks=False, + ), + }, + ) + device = torch.device("cuda") + layout = KVCacheLayout.BLHNC + query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device) + key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device) + value = torch.empty_like(key) + output = torch.empty_like(query) + dummy_lens = [ + num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs) + ] + common = create_common_attn_metadata( + BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens), + block_size, + device, + ) + common.max_query_len = 8 + cache = None + + def prepare(query_lens): + nonlocal cache + query_lens = query_lens + [0] * (num_reqs - len(query_lens)) + query.normal_() + key.normal_() + value.normal_() + # Alternating short prefills, nonempty decode contexts, and contexts + # beyond the sliding window; inactive requests have zero sequence length. + contexts = [ + ([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens) + ] + ks = [ + torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device) + for c in contexts + ] + vs = [torch.randn_like(k) for k in ks] + actual = create_common_attn_metadata( + BatchSpec( + seq_lens=[c + q for c, q in zip(contexts, query_lens)], + query_lens=query_lens, + ), + block_size, + device, + ) + fresh_cache = create_and_prepopulate_kv_cache( + ks, + vs, + block_size, + num_kv_heads, + head_size, + dtype, + device, + num_blocks, + actual, + layout, + randomize_blocks=True, + ) + if cache is None: + cache = fresh_cache + else: + cache.copy_(fresh_cache) + common.query_start_loc.copy_(actual.query_start_loc) + common.seq_lens.copy_(actual.seq_lens) + common.block_table_tensor.zero_() + common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_( + actual.block_table_tensor + ) + common.slot_mapping.fill_(-1) + common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping) + return contexts, ks, vs + + prepare(dummy_lens) + with set_current_vllm_config(config): + builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device) + metadata = builder.build(0, common) + assert metadata.num_prefills == 0 + assert metadata.decode.q_len_per_req == 8 + impl = fi.FlashInferImpl( + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_size=head_size, + scale=scale, + alibi_slopes=None, + sliding_window=sliding_window, + kv_cache_dtype="auto", + attn_type=AttentionType.DECODER, + ) + layer = MockAttentionLayer(device) + + def forward(attn_metadata=metadata): + impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping) + impl.forward(layer, query, key, value, cache, attn_metadata, output=output) + + for _ in range(3): + forward() + torch.accelerator.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + forward() + + layouts = [[8], [1] * num_reqs] + if num_tokens >= 16: + layouts.append([8, 4, 3, 1]) + if num_tokens >= 40: + layouts.append([8] * 5) + + def check(query_lens, run): + contexts, ks, vs = prepare(query_lens) + assert cache is not None + output.fill_(float("nan")) + run() + start = 0 + for i, q_len in enumerate(query_lens): + if q_len == 0: + continue + end = start + q_len + full_k = ( + torch.cat((ks[i], key[start:end])) + .float() + .repeat_interleave(num_heads // num_kv_heads, dim=1) + ) + full_v = ( + torch.cat((vs[i], value[start:end])) + .float() + .repeat_interleave(num_heads // num_kv_heads, dim=1) + ) + scores = ( + torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale + ) + q_pos = torch.arange(q_len, device=device) + contexts[i] + k_pos = torch.arange(full_k.shape[0], device=device) + allowed = k_pos[None, :] <= q_pos[:, None] + if sliding_window is not None: + allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window + probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1) + reference = torch.einsum("hqk,khd->qhd", probs, full_v) + torch.testing.assert_close( + output[start:end].float(), + reference, + atol=2e-2, + rtol=2e-2, + ) + slots = common.slot_mapping[start:end] + written = cache[slots // block_size, :, slots % block_size, :] + torch.testing.assert_close(written[..., :head_size], key[start:end]) + torch.testing.assert_close(written[..., head_size:], value[start:end]) + start = end + assert torch.count_nonzero(cache[0]) == 0 + + for query_lens in layouts: + check(query_lens, graph.replay) + + if num_tokens == 40: + # A longer prefill forces a separate launch in eager/piecewise mode. + # Its length must not become the decode kernel's query bound, and + # compacted CPU placeholders still need not match the GPU allocation. + common.num_reqs = 5 + common.max_query_len = 24 + common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40]) + common.query_start_loc_cpu[6:] = 40 + + def mixed_forward(): + with set_current_vllm_config(config): + metadata = builder.build( + 0, + replace( + common, + query_start_loc=common.query_start_loc[:6], + query_start_loc_cpu=common.query_start_loc_cpu[:6], + seq_lens=common.seq_lens[:5], + seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5], + block_table_tensor=common.block_table_tensor[:5], + ), + ) + assert metadata.num_decodes == 4 + assert metadata.num_prefills == 1 + assert metadata.decode.q_len_per_req == 8 + forward(metadata) + + check([8, 4, 3, 1, 24], mixed_forward) + + @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index 76c4732158..d46476f8d0 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -294,6 +294,7 @@ def _make_spec_decode_manager( dynamic_spec_schedule: list[tuple[int, int, int]] | None = None, max_num_seqs: int = 8, use_kda_recoverssm: bool = False, + varlen_decode: bool = False, ) -> gpu_cudagraph_utils.CudaGraphManager: monkeypatch.setattr( gpu_cudagraph_utils, @@ -316,6 +317,7 @@ def _make_spec_decode_manager( device=torch.device("cpu"), cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, decode_query_len=decode_query_len, + varlen_decode=varlen_decode, ) manager._graphs_captured = True return manager @@ -373,6 +375,28 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch): assert desc.num_tokens == 16 +def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch): + """A varlen decode graph replays any decode batch within its query-length + bound. The runner passes no bound for a batch with a prefill, which must not + replay one however short the prefill is.""" + manager = _make_spec_decode_manager(monkeypatch, varlen_decode=True) + + def dispatch(max_query_len): + return manager.dispatch( + num_reqs=3, + num_tokens=6, + uniform_token_count=None, + num_active_loras=0, + max_query_len=max_query_len, + ) + + desc = dispatch(_DECODE_QUERY_LEN) + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.max_query_len == _DECODE_QUERY_LEN + assert dispatch(None).cg_mode == CUDAGraphMode.PIECEWISE + assert dispatch(_DECODE_QUERY_LEN + 1).cg_mode == CUDAGraphMode.PIECEWISE + + def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch): """A mixed batch must not fall to eager where only decode graphs are staged. diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index 116f4a7c36..392b64d992 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -1,15 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from contextlib import nullcontext from types import SimpleNamespace import numpy as np import pytest from vllm.config.compilation import CUDAGraphMode -from vllm.v1.attention.backend import AttentionCGSupport +from vllm.v1.attention.backend import AttentionCGSupport, AttentionMetadataBuilder from vllm.v1.worker.gpu.async_utils import StepTimingSample -from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_module from vllm.v1.worker.gpu.spec_decode.adaptive_verification import ( AdaptiveVerificationManager, @@ -58,37 +58,90 @@ def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expe ) -def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch): - class Backend: - @classmethod - def supports_device_cpu_query_lens_mismatch(cls): - return True +@pytest.mark.parametrize( + "target_support,target_bound,device_offsets,additional,error", + [ + pytest.param(AttentionCGSupport.ALWAYS, None, True, None, None, id="always"), + pytest.param( + AttentionCGSupport.UNIFORM_BATCH, 8, True, None, None, id="bounded" + ), + pytest.param( + AttentionCGSupport.UNIFORM_BATCH, + None, + True, + None, + "TargetBackend allows none", + id="unsupported", + ), + pytest.param( + AttentionCGSupport.UNIFORM_BATCH, + 7, + True, + None, + "up to 8, but TargetBackend allows at most 7", + id="too-narrow", + ), + pytest.param( + AttentionCGSupport.UNIFORM_BATCH, + 8, + False, + None, + "trims verification requests", + id="host-query-lens", + ), + pytest.param( + AttentionCGSupport.ALWAYS, + None, + True, + (AttentionCGSupport.UNIFORM_BATCH, "EncoderBackend"), + "EncoderBackend allows none", + id="additional-group", + ), + ], +) +def test_manager_checks_target_varlen_cudagraph_bound( + monkeypatch, target_support, target_bound, device_offsets, additional, error +): + """Target builders must replay varlen decode graphs of the verification + width; draft-only groups are not checked.""" - class Builder: - def __init__(self, support): - self.support = support + def group(backend_name, layer_name, support, bound=None, mismatch=True): + class Builder(AttentionMetadataBuilder): + _cudagraph_support = support - def get_cudagraph_support(self, *_args): - return self.support + def __init__(self): + pass - def group(layer_name, support): - builder = Builder(support) + def build(self, common_prefix_len, common_attn_metadata, fast_build=False): + raise NotImplementedError + + if bound is not None: + + @classmethod + def get_varlen_cudagraph_max_query_len(cls, *_args): + return bound + + backend = type( + backend_name, + (), + {"supports_device_cpu_query_lens_mismatch": staticmethod(lambda: mismatch)}, + ) + builder = Builder() return SimpleNamespace( layer_names=[layer_name], - backend=Backend, + backend=backend, kv_cache_spec=None, get_metadata_builder=lambda _index: builder, ) groups = [ [ - group("target", AttentionCGSupport.ALWAYS), - group("draft", AttentionCGSupport.UNIFORM_BATCH), + group( + "TargetBackend", "target", target_support, target_bound, device_offsets + ), + group("DraftBackend", "draft", AttentionCGSupport.UNIFORM_BATCH), ] ] - runner_support = AttentionCGSupportInfo( - AttentionCGSupport.UNIFORM_BATCH, "DraftBackend" - ) created = object() monkeypatch.setattr( adaptive_module, @@ -96,20 +149,19 @@ def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatc lambda *_args, **_kwargs: created, ) - manager = maybe_create_adaptive_verification_manager( - enable_adaptive_verification=True, - attn_groups=groups, - attn_cg_support=runner_support, - req_states=object(), - query_start_loc=object(), - num_bonus_tokens=1, - max_total_logits=1, - vllm_config=None, - target_layer_names={"target"}, - ) - - assert manager is created - assert runner_support.min_cg_support == AttentionCGSupport.UNIFORM_BATCH + with pytest.raises(ValueError, match=error) if error else nullcontext(): + manager = maybe_create_adaptive_verification_manager( + enable_adaptive_verification=True, + attn_groups=groups, + req_states=SimpleNamespace(num_speculative_steps=7), + query_start_loc=object(), + num_bonus_tokens=1, + max_total_logits=1, + vllm_config=None, + target_layer_names={"target"}, + additional_attn_cg_support=additional, + ) + assert manager is created def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves(): diff --git a/tests/v1/worker/test_attn_utils.py b/tests/v1/worker/test_attn_utils.py index 99481a1293..1bff5ae466 100644 --- a/tests/v1/worker/test_attn_utils.py +++ b/tests/v1/worker/test_attn_utils.py @@ -8,6 +8,7 @@ never addressed by the logical view. """ from types import SimpleNamespace +from typing import Any import numpy as np import pytest @@ -99,12 +100,16 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size( class _FakeMetadataBuilder: - def __init__(self, support: AttentionCGSupport): + def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None): self.support = support + self.varlen_bound = varlen_bound def get_cudagraph_support(self, *_args): return self.support + def get_varlen_cudagraph_max_query_len(self, *_args): + return self.varlen_bound + class _TargetBackend: @classmethod @@ -183,6 +188,42 @@ def test_attention_checks_preserve_global_and_target_scoped_support(): ) +def test_varlen_cudagraph_unsupported_backend_checks_scoped_bounds(): + """ALWAYS passes without a bound, other builders need one at least as wide as + the requested length, and NEVER fails whatever bound a builder reports.""" + config: Any = SimpleNamespace() + spec = FullAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + ) + + def group( + backend: Any, + layer_name: str, + support: AttentionCGSupport, + bound: int | None = None, + ): + builder: Any = _FakeMetadataBuilder(support, bound) + attn_group = AttentionGroup(backend, [layer_name], spec, 0) + attn_group.metadata_builders = [builder] + return attn_group + + target = group(_TargetBackend, "target", AttentionCGSupport.ALWAYS) + draft = group(_DraftBackend, "draft", AttentionCGSupport.UNIFORM_BATCH, 8) + never = group(_DraftBackend, "never", AttentionCGSupport.NEVER, 8) + + unsupported = attn_utils.get_varlen_cudagraph_unsupported_backend + assert unsupported([[target, draft]], config, 8) is None + assert unsupported([[target, draft]], config, 9) == ("_DraftBackend", 8) + assert ( + unsupported([[target, draft]], config, 9, checked_layer_names={"target"}) + is None + ) + assert unsupported([[target, never]], config, 1) == ("_DraftBackend", None) + + def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyPatch): """Fast prefill applies to the contiguous suffix of KV-sharing layers. diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 7fb2a09dd3..dbec9fd743 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -631,6 +631,8 @@ class SpeculativeConfig: "dspark", ) factors.append(uses_aux_hidden_states) + if self.method == "dspark": + factors.append(self.enable_adaptive_verification) if self.draft_model_config is not None: factors.append(self.draft_model_config.compute_hash()) diff --git a/vllm/model_executor/models/gemma4_dspark.py b/vllm/model_executor/models/gemma4_dspark.py index 6b6d5d2731..33b1a1aada 100644 --- a/vllm/model_executor/models/gemma4_dspark.py +++ b/vllm/model_executor/models/gemma4_dspark.py @@ -24,7 +24,11 @@ from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config from .gemma4_mtp import Gemma4MTPAttention, Gemma4MTPDecoderLayer from .qwen3_dflash import DFlashQwen3Model, _dflash_layer_causal -from .qwen3_dspark import DSparkMarkovHead, Qwen3DSparkForCausalLM +from .qwen3_dspark import ( + DSparkConfidenceHead, + DSparkMarkovHead, + Qwen3DSparkForCausalLM, +) from .utils import extract_layer_index, maybe_prefix @@ -187,6 +191,18 @@ class Gemma4DSparkModel(DFlashQwen3Model): config.markov_rank, prefix=maybe_prefix(prefix, "markov_head"), ) + self.confidence_head: DSparkConfidenceHead | None = None + if getattr(config, "enable_confidence_head", False): + with_markov = getattr(config, "confidence_head_with_markov", False) + input_dim = config.hidden_size + if with_markov: + input_dim += config.markov_rank + self.confidence_head = DSparkConfidenceHead( + input_dim, + prefix=maybe_prefix(prefix, "confidence_head"), + bias=True, + with_markov=with_markov, + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) * self.normalizer @@ -295,9 +311,8 @@ class Gemma4DSparkForCausalLM(Qwen3DSparkForCausalLM): params = dict(self.named_parameters()) params.update(dict(self.named_buffers())) loaded: set[str] = set() + includes_confidence_head = False for name, w in weights: - if "confidence_head" in name: - continue if "lm_head" not in name: name = "model." + name for pn, wn, shard in stacked: @@ -310,5 +325,9 @@ class Gemma4DSparkForCausalLM(Qwen3DSparkForCausalLM): p = params[name] getattr(p, "weight_loader", default_weight_loader)(p, w) loaded.add(name) + if "confidence_head" in name: + includes_confidence_head = True + if not includes_confidence_head: + self.model.confidence_head = None self.model._build_fused_kv_buffers() return loaded diff --git a/vllm/models/minimax_m3/amd/indexer_aiter.py b/vllm/models/minimax_m3/amd/indexer_aiter.py index adf505aa7d..d46db51363 100644 --- a/vllm/models/minimax_m3/amd/indexer_aiter.py +++ b/vllm/models/minimax_m3/amd/indexer_aiter.py @@ -52,7 +52,11 @@ from vllm.models.minimax_m3.common.sparse_attention import ( _minimax_m3_aiter_sparse_pa_requested, ) from vllm.platforms import current_platform -from vllm.v1.attention.backend import AttentionBackend, CommonAttentionMetadata +from vllm.v1.attention.backend import ( + AttentionBackend, + CommonAttentionMetadata, + max_decode_query_len, +) from vllm.v1.attention.backends.utils import split_decodes_and_prefills from vllm.v1.kv_cache_interface import AttentionSpec @@ -97,10 +101,7 @@ def aiter_indexer_max_decode_query_len(vllm_config: VllmConfig) -> int: since that is what the builder splits the batch on and therefore what the decode kernel will actually be handed. """ - spec = vllm_config.speculative_config - if spec is None or spec.num_speculative_tokens is None: - return 1 - return 1 + (2 if spec.parallel_drafting else 1) * spec.num_speculative_tokens + return max_decode_query_len(vllm_config) @cache diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index b7f4f54d93..432fb1379f 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -387,7 +387,20 @@ class HFConfigParser(ConfigParserBase): if extra_layer_types := _PATCH_HF_ALLOWED_LAYER_TYPES.get(model_type): _patch_hf_transformers_allowed_layer_types(extra_layer_types) - if model_type in _SPECULATIVE_DECODING_CONFIGS: + rope_parameters = config_dict.get("rope_parameters") or {} + if model_type == "gemma4_text" and "full_attention" in rope_parameters: + from transformers import Gemma4TextConfig + + # Published DSpark configs mix redundant scalar entries with per-layer + # RoPE dicts. Remove them before Transformers' constructor validates. + config_dict["rope_parameters"] = { + k: v + for k, v in rope_parameters.items() + if k not in ("rope_type", "rope_theta") + } + kwargs.setdefault("name_or_path", str(model)) + config = Gemma4TextConfig.from_dict(config_dict, **kwargs) + elif model_type in _SPECULATIVE_DECODING_CONFIGS: config_class = _CONFIG_REGISTRY[model_type] config = config_class.from_pretrained( model, diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 8b51e39169..b5cd6e4b32 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -573,6 +573,26 @@ class AttentionCGSupport(Enum): """NO cudagraph support""" +def max_decode_query_len(vllm_config: "VllmConfig") -> int: + """Widest request a spec-as-decode builder treats as a decode. + + On model runner V2 this is a verification request, 1 + + num_speculative_tokens: no V2 speculator builds a wider query. The reorder + threshold and the code that mirrors the builders' decode/prefill split all + read this, so they cannot drift apart. + """ + speculative_config = getattr(vllm_config, "speculative_config", None) + if speculative_config is None or speculative_config.num_speculative_tokens is None: + return 1 + num_speculative_tokens = speculative_config.num_speculative_tokens + max_query_len = 1 + num_speculative_tokens + if speculative_config.parallel_drafting and not vllm_config.use_v2_model_runner: + # Model runner V1 only; remove with it. Its parallel drafter appends up + # to num_speculative_tokens mask slots to each verified request. + max_query_len += num_speculative_tokens + return max_query_len + + class AttentionMetadataBuilder(ABC, Generic[M]): # Does this backend/builder support CUDA Graphs for attention (default: no). # Do not access directly. Call get_cudagraph_support() instead. @@ -616,6 +636,30 @@ class AttentionMetadataBuilder(ABC, Generic[M]): """Get the cudagraph support level of this builder class.""" return cls._cudagraph_support + @classmethod + def get_varlen_cudagraph_max_query_len( + cls: type["AttentionMetadataBuilder"], + vllm_config: "VllmConfig", + kv_cache_spec: "KVCacheSpec", + ) -> int | None: + """Get the largest per-request query length L of the variable-length + decode batches a FULL cudagraph of this builder class can replay. + + The graph must replay any decode batch in which every real request has + between 1 and L query tokens and every padding request has 0. Lengths + come from the device query_start_loc; host metadata carries only the + token count and an upper bound on the per-request length. Batches with + a prefill never replay these graphs. Whether host metadata may + understate device query lengths at all is a separate backend question; + see supports_device_cpu_query_lens_mismatch(). + + Returns: + L, or None for builders reporting ALWAYS, which replay any batch, + and for builders that cannot replay variable-length batches. + + """ + return None + def _init_reorder_batch_threshold( self, reorder_batch_threshold: int | None = 1, @@ -632,14 +676,9 @@ class AttentionMetadataBuilder(ABC, Generic[M]): speculative_config is not None and speculative_config.num_speculative_tokens is not None ): - max_num_queries_for_spec = ( - 1 - + (2 if speculative_config.parallel_drafting else 1) - * speculative_config.num_speculative_tokens - ) self.reorder_batch_threshold = max( self.reorder_batch_threshold, - max_num_queries_for_spec, + max_decode_query_len(self.vllm_config), ) if ( diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 08920fa3ca..f40c75b474 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -67,6 +67,7 @@ from vllm.v1.attention.backend import ( AttentionType, CommonAttentionMetadata, MultipleOf, + max_decode_query_len, ) from vllm.v1.attention.backends.utils import ( get_dcp_local_seq_lens, @@ -856,15 +857,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self.use_xqa = ( self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA ) - # Adaptive verification trims drafts on device, so decode query lengths - # must come from the device qo_indptr; only trtllm-gen supports that - # (the selector already rejects the other configurations). - self.use_trtllm_gen_varlen_decode = ( - speculative_config is not None - and speculative_config.enable_adaptive_verification - and self.flashinfer_trtllm_api_decode_kernel - == FlashInferDecodeKernel.TRTLLM_GEN - and not self.use_dcp + self.use_trtllm_gen_varlen_decode = self._uses_trtllm_gen_varlen_decode( + vllm_config, self.flashinfer_trtllm_api_decode_kernel, self.use_dcp ) self._init_reorder_batch_threshold( 1, @@ -1011,6 +1005,31 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): else: return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + @override # type: ignore[misc] + @classmethod + def get_varlen_cudagraph_max_query_len( + cls: type["FlashInferMetadataBuilder"], + vllm_config: VllmConfig, + kv_cache_spec: KVCacheSpec, + ) -> int | None: + # The uniform gate already covers DCP, head geometry, the XQA head-dim + # limit and causality, and yields None for subclasses that force NEVER. + if ( + cls.get_cudagraph_support(vllm_config, kv_cache_spec) + != AttentionCGSupport.UNIFORM_BATCH + ): + return None + # The kernel selector asserts off the trtllm-capable architectures, so + # call it only past the gate, as get_cudagraph_support does. + if not cls._uses_trtllm_gen_varlen_decode( + vllm_config, + cls._get_flashinfer_trtllm_api_decode_kernel(), + use_dcp=vllm_config.parallel_config.decode_context_parallel_size > 1, + ): + return None + # Longer requests are split off as prefills. + return max_decode_query_len(vllm_config) + def _get_workspace_buffer(self): if self._workspace_buffer is None: buffer_size = envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE @@ -1039,6 +1058,24 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): def set_workspace_buffer(self, workspace_buffer: torch.Tensor): self._workspace_buffer = workspace_buffer + @staticmethod + def _uses_trtllm_gen_varlen_decode( + vllm_config: VllmConfig, + decode_kernel: FlashInferDecodeKernel | None, + use_dcp: bool, + ) -> bool: + """Whether decode reads per-request query lengths from the device + qo_indptr. Adaptive verification trims drafts on device, and only + trtllm-gen supports that (the selector already rejects the other + configurations).""" + speculative_config = vllm_config.speculative_config + return ( + speculative_config is not None + and speculative_config.enable_adaptive_verification + and decode_kernel == FlashInferDecodeKernel.TRTLLM_GEN + and not use_dcp + ) + @staticmethod def _get_flashinfer_trtllm_api_decode_kernel() -> FlashInferDecodeKernel: if current_platform.is_device_capability( @@ -1302,7 +1339,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): split_decodes_and_prefills( common_attn_metadata, decode_threshold=self.reorder_batch_threshold, - require_uniform=not self.use_xqa, + require_uniform=not ( + self.use_xqa or self.use_trtllm_gen_varlen_decode + ), ) ) else: @@ -1704,15 +1743,14 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): bool(causal), ) elif self.use_trtllm_gen_varlen_decode: - # CPU lens are only an upper bound, device qo_indptr is - # the truth. - decode_q_lens = ( - qo_indptr_cpu[1 : num_decodes + 1] - qo_indptr_cpu[:num_decodes] + # CPU placeholders may underestimate query lengths; + # use a safe upper bound. + q_len_per_req = min( + common_attn_metadata.max_query_len, self.reorder_batch_threshold ) - max_q_upper = int(decode_q_lens.max()) - if max_q_upper > 1: - q_len_per_req = max_q_upper - q_cu_seq_lens = qo_indptr[: num_decodes + 1] + # Always pass GPU offsets so a graph captured with uniform + # query lengths can replay batches with variable query lengths. + q_cu_seq_lens = qo_indptr[: num_decodes + 1] attn_metadata.decode = FlashInferTrtllmAPIDecode( kernel=self.flashinfer_trtllm_api_decode_kernel, block_tables=block_table_tensor[:num_decodes], diff --git a/vllm/v1/attention/backends/mla/index_group.py b/vllm/v1/attention/backends/mla/index_group.py index 1b8455a232..a3c6df6fe6 100644 --- a/vllm/v1/attention/backends/mla/index_group.py +++ b/vllm/v1/attention/backends/mla/index_group.py @@ -12,6 +12,7 @@ from vllm import _custom_ops as ops from vllm.config import VllmConfig from vllm.forward_context import in_piecewise_cudagraph from vllm.utils.torch_utils import current_stream +from vllm.v1.attention.backend import max_decode_query_len from vllm.v1.attention.backends.mla.sparse_utils import ( triton_convert_req_index_to_global_index, ) @@ -496,17 +497,8 @@ class SparseMLAIndexGroupBuilder: def get_sparse_mla_index_group_max_rows(vllm_config: VllmConfig) -> int: - max_query_len = 1 - speculative_config = vllm_config.speculative_config - if ( - speculative_config is not None - and speculative_config.num_speculative_tokens is not None - ): - max_query_len += speculative_config.num_speculative_tokens * ( - 2 if speculative_config.parallel_drafting else 1 - ) scheduler_config = vllm_config.scheduler_config return min( scheduler_config.max_num_batched_tokens, - scheduler_config.max_num_seqs * max_query_len, + scheduler_config.max_num_seqs * max_decode_query_len(vllm_config), ) diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index 78727c2eb5..b0ef9e65a3 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -26,6 +26,7 @@ from vllm.v1.attention.backend import ( AttentionMetadataBuilder, CommonAttentionMetadata, MultipleOf, + max_decode_query_len, ) from vllm.v1.attention.backends.mla.compressor_utils import ( get_dspark_swa_index_width, @@ -456,14 +457,10 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): self.num_speculative_tokens = ( spec_config.num_speculative_tokens if spec_config else 0 ) - # Decode can have query_len up to - # 1 + (2 if parallel drafting else 1) * num_speculative_tokens. # sparse_swa has no MQA-vs-dense-MHA routing, so multi-token queries take - # the prefill path and the decode/prefill split stays at that width. - spec_mult = ( - 2 if (spec_config is not None and spec_config.parallel_drafting) else 1 - ) - self.decode_threshold = 1 + spec_mult * self.num_speculative_tokens + # the prefill path and the decode/prefill split stays at the widest + # decode. + self.decode_threshold = max_decode_query_len(self.vllm_config) self.reorder_batch_threshold = None hf_config = self.vllm_config.model_config.hf_config diff --git a/vllm/v1/attention/ops/dcp.py b/vllm/v1/attention/ops/dcp.py index 8f844b6476..71e1fdae07 100644 --- a/vllm/v1/attention/ops/dcp.py +++ b/vllm/v1/attention/ops/dcp.py @@ -27,6 +27,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import ( triton_scalar_specialization_rep, ) from vllm.triton_utils import tl, triton +from vllm.v1.attention.backend import max_decode_query_len from vllm.v1.attention.ops.cp_common import ( DirectCPWorkspace, direct_cp_enabled, @@ -1011,16 +1012,9 @@ def dcp_a2a_lse_reduce( def get_dcp_workspace_max_num_tokens(vllm_config: VllmConfig) -> int: scheduler_config = vllm_config.scheduler_config - speculative_config = vllm_config.speculative_config - speculative_tokens = vllm_config.num_speculative_tokens - tokens_per_seq = ( - 1 - + ( - 2 - if speculative_config is not None and speculative_config.parallel_drafting - else 1 - ) - * speculative_tokens + # num_speculative_tokens also counts a diffusion canvas. + tokens_per_seq = max( + 1 + vllm_config.num_speculative_tokens, max_decode_query_len(vllm_config) ) return min( scheduler_config.max_num_batched_tokens, diff --git a/vllm/v1/hisparse/runtime.py b/vllm/v1/hisparse/runtime.py index 5100379083..4614e1aa32 100644 --- a/vllm/v1/hisparse/runtime.py +++ b/vllm/v1/hisparse/runtime.py @@ -21,6 +21,7 @@ from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.math_utils import round_up from vllm.utils.torch_utils import current_stream +from vllm.v1.attention.backend import max_decode_query_len from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor @@ -46,20 +47,8 @@ def is_hisparse_decode_batch(attn_metadata: Any | None) -> bool: ) -def _get_max_decode_query_len(vllm_config: VllmConfig) -> int: - speculative_config = getattr(vllm_config, "speculative_config", None) - if ( - speculative_config is not None - and speculative_config.num_speculative_tokens is not None - ): - return 1 + speculative_config.num_speculative_tokens * ( - 2 if speculative_config.parallel_drafting else 1 - ) - return 1 - - def _get_max_swap_rows(vllm_config: VllmConfig) -> int: - max_query_len = _get_max_decode_query_len(vllm_config) + max_query_len = max_decode_query_len(vllm_config) scheduler_config = vllm_config.scheduler_config return min( scheduler_config.max_num_batched_tokens, @@ -120,7 +109,7 @@ class ResolvedHiSparseConfig: if config is None: return None - max_query_len = _get_max_decode_query_len(vllm_config) + max_query_len = max_decode_query_len(vllm_config) min_device_buffer_size = max_query_len * model_top_k # Retain the worst-case union of every speculative step's top-k and # leave one additional top-k of LRU slack. diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index e8b4da8f36..423bcebb2d 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -328,6 +328,40 @@ def get_query_lens_mismatch_unsupported_backend( return None +def get_varlen_cudagraph_unsupported_backend( + attn_groups: list[list[AttentionGroup]], + vllm_config: VllmConfig, + max_query_len: int, + checked_layer_names: set[str] | None = None, +) -> tuple[str, int | None] | None: + """Name the first backend whose FULL cudagraphs cannot replay decode batches + of 1 to max_query_len tokens per request, with its bound, if any. + + ALWAYS builders replay any batch; the others need a bound of at least + max_query_len. See AttentionMetadataBuilder.get_varlen_cudagraph_max_query_len(). + """ + for groups in attn_groups: + for group in groups: + if checked_layer_names is not None and checked_layer_names.isdisjoint( + group.layer_names + ): + continue + builder = group.get_metadata_builder(0) + spec = group.kv_cache_spec + support = builder.get_cudagraph_support(vllm_config, spec) + if support == AttentionCGSupport.ALWAYS: + continue + bound = ( + # NEVER means never, whatever an inherited override says. + None + if support == AttentionCGSupport.NEVER + else builder.get_varlen_cudagraph_max_query_len(vllm_config, spec) + ) + if bound is None or bound < max_query_len: + return group.backend.__name__, bound + return None + + def init_kv_cache( forward_context: dict[str, Any], kv_cache_config: KVCacheConfig, diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 8f8658077c..fa1dc4a04b 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -69,7 +69,8 @@ class BatchExecutionDescriptor: num_reqs: int | None # None means no request padding is needed (PIECEWISE graphs) uniform_token_count: int | None = None # Upper bound on per-request query length. Varlen decode graphs leave - # uniform_token_count unset, so this is what keeps a prefill batch out of one. + # uniform_token_count unset, so this is what keeps a prefill batch out of one: + # the runner passes None for any batch with a prefill. max_query_len: int | None = None num_active_loras: int = 0 # Number of microbatches the batch is split into (DBO). 1 means no splitting. diff --git a/vllm/v1/worker/gpu/dp_utils.py b/vllm/v1/worker/gpu/dp_utils.py index 8eabc3adbf..15db7c197c 100644 --- a/vllm/v1/worker/gpu/dp_utils.py +++ b/vllm/v1/worker/gpu/dp_utils.py @@ -243,7 +243,8 @@ def dispatch_cg_and_sync_dp( dp_size: Data-parallel world size. 1 skips all cross-rank work. dp_rank: This rank's index in the DP group. max_query_len: Upper bound on per-request query length, for selecting - varlen decode graphs. None means the graph must not constrain it. + varlen decode graphs. None, as for a batch with a prefill, keeps + the batch out of graphs that constrain it. need_eager: Force `CUDAGraphMode.NONE` instead of dispatching. num_active_loras: Active LoRA count for this rank. Does not need cross-rank agreement; it never changes a bucket's token count. diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index c4dbe25843..61ceaeef2e 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -640,7 +640,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.speculator, "enable_adaptive_verification", False ), attn_groups=self.attn_groups, - attn_cg_support=attn_cg_support, req_states=self.req_states, query_start_loc=self.input_buffers.query_start_loc, num_bonus_tokens=self.model_state.num_new_sampled_tokens_per_step, @@ -1646,12 +1645,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Get batch descriptor and sync across DP ranks. num_reqs = len(scheduler_output.num_scheduled_tokens) num_toks = scheduler_output.total_num_scheduled_tokens - max_query_len = max(scheduler_output.num_scheduled_tokens.values()) + max_query_len: int | None = max(scheduler_output.num_scheduled_tokens.values()) batch_req_state, uniform_tok_count = self.gather_batch_req_state( scheduler_output, dummy_run ) if batch_req_state is not None: num_toks = batch_req_state.num_tokens + if batch_req_state.has_prefill: + # Varlen decode graphs replay decodes only, and their bound + # alone would admit a short prefill. + max_query_len = None if self.pcp_manager is not None: num_toks = self.pcp_manager.get_num_tokens_for_dispatch( batch_req_state.num_scheduled_tokens, diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index f47b4b7cbf..5f91af59e5 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -19,8 +19,8 @@ from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.utils import CpuGpuBuffer from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream from vllm.v1.worker.gpu.attn_utils import ( - get_attn_cg_support, get_query_lens_mismatch_unsupported_backend, + get_varlen_cudagraph_unsupported_backend, ) logger = init_logger(__name__) @@ -28,7 +28,6 @@ _PROFILE_REPLAYS = 5 if TYPE_CHECKING: from vllm.config import VllmConfig - from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.utils import AttentionGroup @@ -452,7 +451,6 @@ def maybe_create_adaptive_verification_manager( *, enable_adaptive_verification: bool, attn_groups: list[list["AttentionGroup"]], - attn_cg_support: "AttentionCGSupportInfo", req_states: "RequestState", query_start_loc: torch.Tensor, num_bonus_tokens: int, @@ -477,25 +475,30 @@ def maybe_create_adaptive_verification_manager( "use a backend that does." ) - target_attn_cg_support = attn_cg_support - if target_layer_names is not None: - target_attn_cg_support = get_attn_cg_support( + # The runner's decode_query_len, the width varlen decode graphs capture. + max_query_len = req_states.num_speculative_steps + num_bonus_tokens + unsupported: tuple[str | None, int | None] | None = ( + get_varlen_cudagraph_unsupported_backend( attn_groups, vllm_config, + max_query_len, checked_layer_names=target_layer_names, ) - if additional_attn_cg_support is not None: - target_attn_cg_support = target_attn_cg_support.narrow( - *additional_attn_cg_support - ) - if target_attn_cg_support.min_cg_support != AttentionCGSupport.ALWAYS: + ) + if unsupported is None and additional_attn_cg_support is not None: + # Groups built outside init_attn_backend report only their support + # level, and without a bound only ALWAYS replays varlen batches. + additional_support, additional_backend = additional_attn_cg_support + if additional_support != AttentionCGSupport.ALWAYS: + unsupported = additional_backend, None + if unsupported is not None: + backend, bound = unsupported + allowed = "none" if bound is None else f"at most {bound}" raise ValueError( - "Adaptive verification captures varlen decode cudagraphs, so every" - " target attention builder must report AttentionCGSupport.ALWAYS, but " - f"{target_attn_cg_support.min_cg_attn_backend} reports " - f"{target_attn_cg_support.min_cg_support}. Pass " - "enable_adaptive_verification=false in the speculative config, or " - "use a backend that does." + "Adaptive verification replays decode cudagraphs whose per-request " + f"query lengths vary up to {max_query_len}, but {backend} allows " + f"{allowed}. Pass enable_adaptive_verification=false in the " + "speculative config, or use a backend that does." ) return AdaptiveVerificationManager(