[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-09-26 17:01:17 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5.5 Nick Hill
parent 3b4566c5cf
commit a4eb3f25d6
24 changed files with 894 additions and 135 deletions
+2
View File
@@ -172,6 +172,8 @@ class AttentionCGSupport(enum.Enum):
Suppose we have hybrid attention backends (e.g., in mamba mixer models). In that case, we seek the minimum capability of all backends to determine the final capability of the model, and we might resolve the incompatible CUDA Graphs mode by downgrading the mode to the best fit one. For example, downgrading `FULL` mode to `FULL_AND_PIECEWISE` mode if the minimum capability is `UNIFORM_BATCH`, or `PIECEWISE` mode if the minimum capability is `NEVER` for -O3 compilation mode. For the complete fallback policy, please see the code for [this][vllm.v1.worker.gpu_model_runner.GPUModelRunner._check_and_update_cudagraph_mode].
Variable-length decode batches, where each request carries a different number of query tokens read from the device `query_start_loc` (as in adaptive verification), are declared separately and not ordered against the enum: `AttentionMetadataBuilder.get_varlen_cudagraph_max_query_len()` returns the largest per-request query length a FULL decode graph can replay. It returns `None` for builders reporting `ALWAYS`, which replay any batch, and for builders that cannot replay variable-length batches; FlashInfer returns `1 + num_speculative_tokens` when its TRTLLM-GEN varlen decode path is active. Batches with a prefill never replay these graphs.
The following table lists backends that support full CUDA Graphs at the time of writing.
| Attention Backend | cudagraph_support | Comments |
@@ -455,13 +455,29 @@ def test_mla_dcp_manager_selects_fallback_backends(monkeypatch):
)
def test_dcp_workspace_covers_parallel_drafting():
@pytest.mark.parametrize(
("parallel_drafting", "use_v2_model_runner", "expected"),
[
# Model runner V1's parallel drafter widens requests to 1 + 2 * 3.
(True, False, 28),
(True, True, 16),
# A diffusion canvas of 3 without a speculative config.
(None, True, 16),
],
)
def test_dcp_workspace_covers_decode_width(
parallel_drafting, use_v2_model_runner, expected
):
config = _manager_config()
config.scheduler_config.max_num_batched_tokens = 128
config.num_speculative_tokens = 3
config.speculative_config = MagicMock(parallel_drafting=True)
config.use_v2_model_runner = use_v2_model_runner
if parallel_drafting is not None:
config.speculative_config = MagicMock(
num_speculative_tokens=3, parallel_drafting=parallel_drafting
)
assert dcp.get_dcp_workspace_max_num_tokens(config) == 28
assert dcp.get_dcp_workspace_max_num_tokens(config) == expected
def test_mla_dcp_manager_selects_pcp_combine(monkeypatch):
+67 -2
View File
@@ -7,7 +7,8 @@ import numpy as np
import pytest
import torch
from vllm.config import VllmConfig
from vllm.config import CompilationConfig, VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models import gemma
from vllm.model_executor.models.gemma3n import (
@@ -18,10 +19,72 @@ from vllm.model_executor.models.gemma4 import (
Gemma4ForCausalLM,
_gemma4_layer_weights_mapper,
)
from vllm.model_executor.models.gemma4_dspark import (
Gemma4DSparkForCausalLM,
Gemma4DSparkModel,
)
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize(
"enabled,has_weights,with_markov",
[
pytest.param(True, True, False, id="hidden-only"),
pytest.param(True, True, True, id="with-markov"),
pytest.param(True, False, True, id="missing-weights"),
pytest.param(False, True, True, id="disabled-head"),
],
)
def test_gemma4_dspark_loads_confidence_head(
monkeypatch, enabled, has_weights, with_markov
) -> None:
"""Use checkpoint confidence parameters, or disable an unavailable head."""
config = SimpleNamespace(
vocab_size=64,
hidden_size=8,
target_layer_ids=[0, 1],
num_hidden_layers=0,
rms_norm_eps=1e-6,
markov_rank=4,
enable_confidence_head=enabled,
confidence_head_with_markov=with_markov,
)
vllm_config = SimpleNamespace(
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
model_config=SimpleNamespace(dtype=torch.bfloat16),
speculative_config=SimpleNamespace(
draft_model_config=SimpleNamespace(hf_config=config),
),
)
monkeypatch.setattr(Gemma4DSparkModel, "_build_fused_kv_buffers", lambda _: None)
model = Gemma4DSparkForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
width = config.hidden_size + (config.markov_rank if with_markov else 0)
weight = torch.arange(width, dtype=torch.bfloat16).reshape(1, -1) / 16
bias = torch.tensor([-0.5], dtype=torch.bfloat16)
loaded = model.load_weights(
[("confidence_head.proj.weight", weight), ("confidence_head.proj.bias", bias)]
if has_weights
else []
)
if not (enabled and has_weights):
assert model.model.confidence_head is None
assert not loaded
return
assert loaded == {
"model.confidence_head.proj.weight",
"model.confidence_head.proj.bias",
}
assert model.model.confidence_head.proj.weight.dtype == torch.float32
hidden = torch.full((3, config.hidden_size), 0.25, dtype=torch.bfloat16)
markov = torch.full((3, config.markov_rank), -0.5, dtype=torch.bfloat16)
inputs = torch.cat([hidden, markov], dim=-1) if with_markov else hidden
expected = (inputs.float() @ weight.float().T + bias.float()).sigmoid().squeeze(-1)
torch.testing.assert_close(model.compute_confidence(hidden, markov), expected)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
@@ -164,7 +227,9 @@ def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
) as llm:
if model == "google/gemma-3-4b-it":
normalizers = llm.llm.collective_rpc(
lambda self: self.model_runner.model.language_model.model.normalizer.cpu().item() # noqa: E501
lambda self: (
self.model_runner.model.language_model.model.normalizer.cpu().item()
) # noqa: E501
)
config = llm.llm.llm_engine.model_config.hf_config.text_config
else:
+12
View File
@@ -92,6 +92,16 @@ def test_nested_rope_validation_patch_preserves_flat_rope_parameters(monkeypatch
assert len(calls) == 2
def test_dspark_adaptive_verification_separates_graph_cache():
config = object.__new__(SpeculativeConfig)
config.method = "dspark"
config.draft_model_config = None
config.enable_adaptive_verification = False
fixed_hash = config.compute_hash()
config.enable_adaptive_verification = True
assert config.compute_hash() != fixed_hash
def _write_json(path: Path, value: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value), encoding="utf-8")
@@ -878,6 +888,8 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
("FULL_AND_PIECEWISE", False, "ALWAYS", "FULL_DECODE_ONLY"),
("FULL_DECODE_ONLY", False, "ALWAYS", "FULL_DECODE_ONLY"),
("FULL_DECODE_ONLY", False, "NEVER", "NONE"),
("FULL_AND_PIECEWISE", True, "UNIFORM_BATCH", "FULL_AND_PIECEWISE"),
("FULL", True, "UNIFORM_BATCH", "FULL_DECODE_ONLY"),
],
)
def test_resolve_cudagraph_mode_uses_loaded_piecewise_provider(
+45
View File
@@ -5,6 +5,7 @@ only get the `eos_token_id` from the tokenizer as defined by
`BaseRenderer.get_eos_token_id`.
"""
import json
import math
from types import SimpleNamespace
from typing import cast
@@ -31,6 +32,50 @@ from vllm.transformers_utils.configs.glm5_next import (
from vllm.transformers_utils.configs.mistral import adapt_config_dict
@pytest.mark.parametrize("layout", ["mixed", "flat"])
def test_gemma4_dspark_rope_config_preserves_parameters(tmp_path, layout):
"""Remove redundant shared entries while preserving per-layer and flat RoPE."""
from transformers import Gemma4TextConfig
per_layer = {
"full_attention": {
"rope_type": "proportional",
"partial_rotary_factor": 0.25,
"rope_theta": 1000000.0,
},
"sliding_attention": {"rope_type": "default", "rope_theta": 10000.0},
}
rope_parameters: dict[str, object] = dict(per_layer)
if layout == "mixed":
rope_parameters.update(rope_type="default", rope_theta=None)
else:
rope_parameters = {"rope_type": "default", "rope_theta": 12345.0}
(tmp_path / "config.json").write_text(
json.dumps(
{
"model_type": "gemma4_text",
"architectures": ["Gemma4DSparkModel"],
"num_hidden_layers": 1,
"layer_types": ["full_attention"],
"rope_parameters": rope_parameters,
}
)
)
_, config = config_module.HFConfigParser().parse(
tmp_path, trust_remote_code=False, max_position_embeddings=8192
)
assert isinstance(config, Gemma4TextConfig)
assert config.name_or_path == str(tmp_path)
assert config.max_position_embeddings == 8192
if layout == "flat":
assert config.rope_parameters["rope_theta"] == 12345.0
else:
for layer_type, expected in per_layer.items():
for key, value in expected.items():
assert config.rope_parameters[layer_type][key] == value
assert set(config.rope_parameters) == set(per_layer)
def test_patch_legacy_rope_type_preserves_nope_layers():
"""NoPE layers stay disabled while later RoPE layers are normalized."""
rope_parameters = {
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for v1 attention backends without GPUModelRunner dependency."""
from dataclasses import replace
from functools import partial
from types import SimpleNamespace
import pytest
import torch
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
from transformers import LlamaConfig
from tests.v1.attention.utils import (
BatchSpec,
@@ -28,8 +30,10 @@ from vllm.utils.torch_utils import (
)
from vllm.v1.attention.backend import (
AttentionCGSupport,
AttentionMetadataBuilder,
AttentionType,
CommonAttentionMetadata,
max_decode_query_len,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
@@ -1066,6 +1070,378 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
assert q_cu_seq_lens.tolist() == qo_indptr_values
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
@pytest.mark.parametrize(
"adaptive,decode_kernel,dcp_size,expected_bound",
[
(None, "TRTLLM_GEN", 1, None),
(False, "TRTLLM_GEN", 1, None),
(True, "TRTLLM_GEN", 1, 8),
(True, "XQA", 1, None),
(True, "TRTLLM_GEN", 2, None),
],
)
def test_flashinfer_varlen_cudagraph_capability(
monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound
):
"""Only the trtllm-gen path that adaptive verification enables replays
varlen decode graphs, up to the decode width; the uniform level is fixed."""
from vllm.model_executor.layers.attention.chunked_local_attention import (
create_chunked_local_attention_backend,
)
from vllm.v1.attention.backends import flashinfer as fi
config = SimpleNamespace(
attention_config=SimpleNamespace(use_non_causal=False),
speculative_config=SimpleNamespace(
enable_adaptive_verification=adaptive,
num_speculative_tokens=7,
parallel_drafting=True,
)
if adaptive is not None
else None,
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32),
use_v2_model_runner=True,
)
monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True)
monkeypatch.setattr(
fi.FlashInferMetadataBuilder,
"_get_flashinfer_trtllm_api_decode_kernel",
staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]),
)
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=4,
head_size=64,
dtype=torch.bfloat16,
)
builder_cls = fi.FlashInferMetadataBuilder
assert builder_cls.get_cudagraph_support(config, spec) == (
AttentionCGSupport.UNIFORM_BATCH
if dcp_size == 1
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec)
assert bound == expected_bound
if bound is not None:
# A wrapper forcing NEVER inherits the override but not the bound.
chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16)
wrapped_cls = chunked_local.get_builder_cls()
assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None
@pytest.mark.parametrize(
"use_v2_model_runner,parallel_drafting,expected",
[
(True, None, 1),
(True, False, 8),
(True, True, 8),
(False, False, 8),
(False, True, 15),
],
)
def test_spec_as_decode_width_follows_model_runner(
use_v2_model_runner, parallel_drafting, expected
):
"""Only model runner V1's parallel drafter widens decode requests."""
class Builder(AttentionMetadataBuilder):
def __init__(self, vllm_config):
self.vllm_config = vllm_config
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
raise NotImplementedError
config = SimpleNamespace(
speculative_config=None
if parallel_drafting is None
else SimpleNamespace(
num_speculative_tokens=7, parallel_drafting=parallel_drafting
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
use_v2_model_runner=use_v2_model_runner,
)
assert max_decode_query_len(config) == expected
builder = Builder(config)
builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
assert builder.reorder_batch_threshold == expected
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
or not current_platform.is_device_capability_family(100),
reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs",
)
@pytest.mark.parametrize(
"head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads",
[
pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"),
pytest.param(
256,
1024,
40,
32,
16,
8,
id="gemma4-sliding-gqa",
),
pytest.param(
512,
None,
40,
32,
16,
1,
id="gemma4-global-mqa",
),
],
)
def test_flashinfer_varlen_decode_graph_replays_changed_layout(
tmp_path,
monkeypatch,
head_size,
sliding_window,
num_tokens,
num_reqs,
num_heads,
num_kv_heads,
):
"""Replay changes device offsets, active slots, context, and KV writes.
The graph is captured with evenly spread dummy queries and an explicit
bound of eight. CPU offsets then remain stale while device offsets change.
Compare real output rows with independent FP32 causal/windowed attention,
and verify cache writes including preservation of the null padding block.
"""
from vllm.v1.attention.backends import flashinfer as fi
from vllm.v1.attention.backends.utils import PerLayerParameters
set_random_seed(0)
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
block_size = 16
dtype = torch.bfloat16
LlamaConfig(
architectures=["LlamaForCausalLM"],
hidden_size=num_heads * head_size,
head_dim=head_size,
num_attention_heads=num_heads,
num_key_value_heads=num_kv_heads,
num_hidden_layers=1,
max_position_embeddings=4096,
).save_pretrained(tmp_path)
num_blocks = 1 + num_reqs * cdiv(2112, block_size)
config = create_vllm_config(
model_name=str(tmp_path),
max_model_len=4096,
dtype=dtype,
num_gpu_blocks=num_blocks,
block_size=block_size,
max_num_seqs=32,
max_num_batched_tokens=256,
)
config.cache_config.kv_cache_layout = "BLHNC"
config.speculative_config = SimpleNamespace(
num_speculative_tokens=7,
enable_adaptive_verification=True,
parallel_drafting=False,
)
spec = create_standard_kv_cache_spec(config)
scale = head_size**-0.5
monkeypatch.setattr(
fi,
"get_per_layer_parameters",
lambda *_: {
"layer": PerLayerParameters(
window_left=-1 if sliding_window is None else sliding_window - 1,
logits_soft_cap=0.0,
sm_scale=scale,
has_sinks=False,
),
},
)
device = torch.device("cuda")
layout = KVCacheLayout.BLHNC
query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device)
key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device)
value = torch.empty_like(key)
output = torch.empty_like(query)
dummy_lens = [
num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs)
]
common = create_common_attn_metadata(
BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens),
block_size,
device,
)
common.max_query_len = 8
cache = None
def prepare(query_lens):
nonlocal cache
query_lens = query_lens + [0] * (num_reqs - len(query_lens))
query.normal_()
key.normal_()
value.normal_()
# Alternating short prefills, nonempty decode contexts, and contexts
# beyond the sliding window; inactive requests have zero sequence length.
contexts = [
([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens)
]
ks = [
torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device)
for c in contexts
]
vs = [torch.randn_like(k) for k in ks]
actual = create_common_attn_metadata(
BatchSpec(
seq_lens=[c + q for c, q in zip(contexts, query_lens)],
query_lens=query_lens,
),
block_size,
device,
)
fresh_cache = create_and_prepopulate_kv_cache(
ks,
vs,
block_size,
num_kv_heads,
head_size,
dtype,
device,
num_blocks,
actual,
layout,
randomize_blocks=True,
)
if cache is None:
cache = fresh_cache
else:
cache.copy_(fresh_cache)
common.query_start_loc.copy_(actual.query_start_loc)
common.seq_lens.copy_(actual.seq_lens)
common.block_table_tensor.zero_()
common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_(
actual.block_table_tensor
)
common.slot_mapping.fill_(-1)
common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping)
return contexts, ks, vs
prepare(dummy_lens)
with set_current_vllm_config(config):
builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device)
metadata = builder.build(0, common)
assert metadata.num_prefills == 0
assert metadata.decode.q_len_per_req == 8
impl = fi.FlashInferImpl(
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
scale=scale,
alibi_slopes=None,
sliding_window=sliding_window,
kv_cache_dtype="auto",
attn_type=AttentionType.DECODER,
)
layer = MockAttentionLayer(device)
def forward(attn_metadata=metadata):
impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
impl.forward(layer, query, key, value, cache, attn_metadata, output=output)
for _ in range(3):
forward()
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
forward()
layouts = [[8], [1] * num_reqs]
if num_tokens >= 16:
layouts.append([8, 4, 3, 1])
if num_tokens >= 40:
layouts.append([8] * 5)
def check(query_lens, run):
contexts, ks, vs = prepare(query_lens)
assert cache is not None
output.fill_(float("nan"))
run()
start = 0
for i, q_len in enumerate(query_lens):
if q_len == 0:
continue
end = start + q_len
full_k = (
torch.cat((ks[i], key[start:end]))
.float()
.repeat_interleave(num_heads // num_kv_heads, dim=1)
)
full_v = (
torch.cat((vs[i], value[start:end]))
.float()
.repeat_interleave(num_heads // num_kv_heads, dim=1)
)
scores = (
torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale
)
q_pos = torch.arange(q_len, device=device) + contexts[i]
k_pos = torch.arange(full_k.shape[0], device=device)
allowed = k_pos[None, :] <= q_pos[:, None]
if sliding_window is not None:
allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window
probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1)
reference = torch.einsum("hqk,khd->qhd", probs, full_v)
torch.testing.assert_close(
output[start:end].float(),
reference,
atol=2e-2,
rtol=2e-2,
)
slots = common.slot_mapping[start:end]
written = cache[slots // block_size, :, slots % block_size, :]
torch.testing.assert_close(written[..., :head_size], key[start:end])
torch.testing.assert_close(written[..., head_size:], value[start:end])
start = end
assert torch.count_nonzero(cache[0]) == 0
for query_lens in layouts:
check(query_lens, graph.replay)
if num_tokens == 40:
# A longer prefill forces a separate launch in eager/piecewise mode.
# Its length must not become the decode kernel's query bound, and
# compacted CPU placeholders still need not match the GPU allocation.
common.num_reqs = 5
common.max_query_len = 24
common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40])
common.query_start_loc_cpu[6:] = 40
def mixed_forward():
with set_current_vllm_config(config):
metadata = builder.build(
0,
replace(
common,
query_start_loc=common.query_start_loc[:6],
query_start_loc_cpu=common.query_start_loc_cpu[:6],
seq_lens=common.seq_lens[:5],
seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5],
block_table_tensor=common.block_table_tensor[:5],
),
)
assert metadata.num_decodes == 4
assert metadata.num_prefills == 1
assert metadata.decode.q_len_per_req == 8
forward(metadata)
check([8, 4, 3, 1, 24], mixed_forward)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
@@ -294,6 +294,7 @@ def _make_spec_decode_manager(
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
max_num_seqs: int = 8,
use_kda_recoverssm: bool = False,
varlen_decode: bool = False,
) -> gpu_cudagraph_utils.CudaGraphManager:
monkeypatch.setattr(
gpu_cudagraph_utils,
@@ -316,6 +317,7 @@ def _make_spec_decode_manager(
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=decode_query_len,
varlen_decode=varlen_decode,
)
manager._graphs_captured = True
return manager
@@ -373,6 +375,28 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch):
assert desc.num_tokens == 16
def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch):
"""A varlen decode graph replays any decode batch within its query-length
bound. The runner passes no bound for a batch with a prefill, which must not
replay one however short the prefill is."""
manager = _make_spec_decode_manager(monkeypatch, varlen_decode=True)
def dispatch(max_query_len):
return manager.dispatch(
num_reqs=3,
num_tokens=6,
uniform_token_count=None,
num_active_loras=0,
max_query_len=max_query_len,
)
desc = dispatch(_DECODE_QUERY_LEN)
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.max_query_len == _DECODE_QUERY_LEN
assert dispatch(None).cg_mode == CUDAGraphMode.PIECEWISE
assert dispatch(_DECODE_QUERY_LEN + 1).cg_mode == CUDAGraphMode.PIECEWISE
def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch):
"""A mixed batch must not fall to eager where only decode graphs are staged.
@@ -1,15 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
import numpy as np
import pytest
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backend import AttentionCGSupport, AttentionMetadataBuilder
from vllm.v1.worker.gpu.async_utils import StepTimingSample
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_module
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
AdaptiveVerificationManager,
@@ -58,37 +58,90 @@ def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expe
)
def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch):
class Backend:
@classmethod
def supports_device_cpu_query_lens_mismatch(cls):
return True
@pytest.mark.parametrize(
"target_support,target_bound,device_offsets,additional,error",
[
pytest.param(AttentionCGSupport.ALWAYS, None, True, None, None, id="always"),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH, 8, True, None, None, id="bounded"
),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH,
None,
True,
None,
"TargetBackend allows none",
id="unsupported",
),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH,
7,
True,
None,
"up to 8, but TargetBackend allows at most 7",
id="too-narrow",
),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH,
8,
False,
None,
"trims verification requests",
id="host-query-lens",
),
pytest.param(
AttentionCGSupport.ALWAYS,
None,
True,
(AttentionCGSupport.UNIFORM_BATCH, "EncoderBackend"),
"EncoderBackend allows none",
id="additional-group",
),
],
)
def test_manager_checks_target_varlen_cudagraph_bound(
monkeypatch, target_support, target_bound, device_offsets, additional, error
):
"""Target builders must replay varlen decode graphs of the verification
width; draft-only groups are not checked."""
class Builder:
def __init__(self, support):
self.support = support
def group(backend_name, layer_name, support, bound=None, mismatch=True):
class Builder(AttentionMetadataBuilder):
_cudagraph_support = support
def get_cudagraph_support(self, *_args):
return self.support
def __init__(self):
pass
def group(layer_name, support):
builder = Builder(support)
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
raise NotImplementedError
if bound is not None:
@classmethod
def get_varlen_cudagraph_max_query_len(cls, *_args):
return bound
backend = type(
backend_name,
(),
{"supports_device_cpu_query_lens_mismatch": staticmethod(lambda: mismatch)},
)
builder = Builder()
return SimpleNamespace(
layer_names=[layer_name],
backend=Backend,
backend=backend,
kv_cache_spec=None,
get_metadata_builder=lambda _index: builder,
)
groups = [
[
group("target", AttentionCGSupport.ALWAYS),
group("draft", AttentionCGSupport.UNIFORM_BATCH),
group(
"TargetBackend", "target", target_support, target_bound, device_offsets
),
group("DraftBackend", "draft", AttentionCGSupport.UNIFORM_BATCH),
]
]
runner_support = AttentionCGSupportInfo(
AttentionCGSupport.UNIFORM_BATCH, "DraftBackend"
)
created = object()
monkeypatch.setattr(
adaptive_module,
@@ -96,20 +149,19 @@ def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatc
lambda *_args, **_kwargs: created,
)
manager = maybe_create_adaptive_verification_manager(
enable_adaptive_verification=True,
attn_groups=groups,
attn_cg_support=runner_support,
req_states=object(),
query_start_loc=object(),
num_bonus_tokens=1,
max_total_logits=1,
vllm_config=None,
target_layer_names={"target"},
)
assert manager is created
assert runner_support.min_cg_support == AttentionCGSupport.UNIFORM_BATCH
with pytest.raises(ValueError, match=error) if error else nullcontext():
manager = maybe_create_adaptive_verification_manager(
enable_adaptive_verification=True,
attn_groups=groups,
req_states=SimpleNamespace(num_speculative_steps=7),
query_start_loc=object(),
num_bonus_tokens=1,
max_total_logits=1,
vllm_config=None,
target_layer_names={"target"},
additional_attn_cg_support=additional,
)
assert manager is created
def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
+42 -1
View File
@@ -8,6 +8,7 @@ never addressed by the logical view.
"""
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
@@ -99,12 +100,16 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
class _FakeMetadataBuilder:
def __init__(self, support: AttentionCGSupport):
def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None):
self.support = support
self.varlen_bound = varlen_bound
def get_cudagraph_support(self, *_args):
return self.support
def get_varlen_cudagraph_max_query_len(self, *_args):
return self.varlen_bound
class _TargetBackend:
@classmethod
@@ -183,6 +188,42 @@ def test_attention_checks_preserve_global_and_target_scoped_support():
)
def test_varlen_cudagraph_unsupported_backend_checks_scoped_bounds():
"""ALWAYS passes without a bound, other builders need one at least as wide as
the requested length, and NEVER fails whatever bound a builder reports."""
config: Any = SimpleNamespace()
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
)
def group(
backend: Any,
layer_name: str,
support: AttentionCGSupport,
bound: int | None = None,
):
builder: Any = _FakeMetadataBuilder(support, bound)
attn_group = AttentionGroup(backend, [layer_name], spec, 0)
attn_group.metadata_builders = [builder]
return attn_group
target = group(_TargetBackend, "target", AttentionCGSupport.ALWAYS)
draft = group(_DraftBackend, "draft", AttentionCGSupport.UNIFORM_BATCH, 8)
never = group(_DraftBackend, "never", AttentionCGSupport.NEVER, 8)
unsupported = attn_utils.get_varlen_cudagraph_unsupported_backend
assert unsupported([[target, draft]], config, 8) is None
assert unsupported([[target, draft]], config, 9) == ("_DraftBackend", 8)
assert (
unsupported([[target, draft]], config, 9, checked_layer_names={"target"})
is None
)
assert unsupported([[target, never]], config, 1) == ("_DraftBackend", None)
def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyPatch):
"""Fast prefill applies to the contiguous suffix of KV-sharing layers.
+2
View File
@@ -631,6 +631,8 @@ class SpeculativeConfig:
"dspark",
)
factors.append(uses_aux_hidden_states)
if self.method == "dspark":
factors.append(self.enable_adaptive_verification)
if self.draft_model_config is not None:
factors.append(self.draft_model_config.compute_hash())
+22 -3
View File
@@ -24,7 +24,11 @@ from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from .gemma4_mtp import Gemma4MTPAttention, Gemma4MTPDecoderLayer
from .qwen3_dflash import DFlashQwen3Model, _dflash_layer_causal
from .qwen3_dspark import DSparkMarkovHead, Qwen3DSparkForCausalLM
from .qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
Qwen3DSparkForCausalLM,
)
from .utils import extract_layer_index, maybe_prefix
@@ -187,6 +191,18 @@ class Gemma4DSparkModel(DFlashQwen3Model):
config.markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(config, "enable_confidence_head", False):
with_markov = getattr(config, "confidence_head_with_markov", False)
input_dim = config.hidden_size
if with_markov:
input_dim += config.markov_rank
self.confidence_head = DSparkConfidenceHead(
input_dim,
prefix=maybe_prefix(prefix, "confidence_head"),
bias=True,
with_markov=with_markov,
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids) * self.normalizer
@@ -295,9 +311,8 @@ class Gemma4DSparkForCausalLM(Qwen3DSparkForCausalLM):
params = dict(self.named_parameters())
params.update(dict(self.named_buffers()))
loaded: set[str] = set()
includes_confidence_head = False
for name, w in weights:
if "confidence_head" in name:
continue
if "lm_head" not in name:
name = "model." + name
for pn, wn, shard in stacked:
@@ -310,5 +325,9 @@ class Gemma4DSparkForCausalLM(Qwen3DSparkForCausalLM):
p = params[name]
getattr(p, "weight_loader", default_weight_loader)(p, w)
loaded.add(name)
if "confidence_head" in name:
includes_confidence_head = True
if not includes_confidence_head:
self.model.confidence_head = None
self.model._build_fused_kv_buffers()
return loaded
+6 -5
View File
@@ -52,7 +52,11 @@ from vllm.models.minimax_m3.common.sparse_attention import (
_minimax_m3_aiter_sparse_pa_requested,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backend import AttentionBackend, CommonAttentionMetadata
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
max_decode_query_len,
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.kv_cache_interface import AttentionSpec
@@ -97,10 +101,7 @@ def aiter_indexer_max_decode_query_len(vllm_config: VllmConfig) -> int:
since that is what the builder splits the batch on and therefore what the
decode kernel will actually be handed.
"""
spec = vllm_config.speculative_config
if spec is None or spec.num_speculative_tokens is None:
return 1
return 1 + (2 if spec.parallel_drafting else 1) * spec.num_speculative_tokens
return max_decode_query_len(vllm_config)
@cache
+14 -1
View File
@@ -387,7 +387,20 @@ class HFConfigParser(ConfigParserBase):
if extra_layer_types := _PATCH_HF_ALLOWED_LAYER_TYPES.get(model_type):
_patch_hf_transformers_allowed_layer_types(extra_layer_types)
if model_type in _SPECULATIVE_DECODING_CONFIGS:
rope_parameters = config_dict.get("rope_parameters") or {}
if model_type == "gemma4_text" and "full_attention" in rope_parameters:
from transformers import Gemma4TextConfig
# Published DSpark configs mix redundant scalar entries with per-layer
# RoPE dicts. Remove them before Transformers' constructor validates.
config_dict["rope_parameters"] = {
k: v
for k, v in rope_parameters.items()
if k not in ("rope_type", "rope_theta")
}
kwargs.setdefault("name_or_path", str(model))
config = Gemma4TextConfig.from_dict(config_dict, **kwargs)
elif model_type in _SPECULATIVE_DECODING_CONFIGS:
config_class = _CONFIG_REGISTRY[model_type]
config = config_class.from_pretrained(
model,
+45 -6
View File
@@ -573,6 +573,26 @@ class AttentionCGSupport(Enum):
"""NO cudagraph support"""
def max_decode_query_len(vllm_config: "VllmConfig") -> int:
"""Widest request a spec-as-decode builder treats as a decode.
On model runner V2 this is a verification request, 1 +
num_speculative_tokens: no V2 speculator builds a wider query. The reorder
threshold and the code that mirrors the builders' decode/prefill split all
read this, so they cannot drift apart.
"""
speculative_config = getattr(vllm_config, "speculative_config", None)
if speculative_config is None or speculative_config.num_speculative_tokens is None:
return 1
num_speculative_tokens = speculative_config.num_speculative_tokens
max_query_len = 1 + num_speculative_tokens
if speculative_config.parallel_drafting and not vllm_config.use_v2_model_runner:
# Model runner V1 only; remove with it. Its parallel drafter appends up
# to num_speculative_tokens mask slots to each verified request.
max_query_len += num_speculative_tokens
return max_query_len
class AttentionMetadataBuilder(ABC, Generic[M]):
# Does this backend/builder support CUDA Graphs for attention (default: no).
# Do not access directly. Call get_cudagraph_support() instead.
@@ -616,6 +636,30 @@ class AttentionMetadataBuilder(ABC, Generic[M]):
"""Get the cudagraph support level of this builder class."""
return cls._cudagraph_support
@classmethod
def get_varlen_cudagraph_max_query_len(
cls: type["AttentionMetadataBuilder"],
vllm_config: "VllmConfig",
kv_cache_spec: "KVCacheSpec",
) -> int | None:
"""Get the largest per-request query length L of the variable-length
decode batches a FULL cudagraph of this builder class can replay.
The graph must replay any decode batch in which every real request has
between 1 and L query tokens and every padding request has 0. Lengths
come from the device query_start_loc; host metadata carries only the
token count and an upper bound on the per-request length. Batches with
a prefill never replay these graphs. Whether host metadata may
understate device query lengths at all is a separate backend question;
see supports_device_cpu_query_lens_mismatch().
Returns:
L, or None for builders reporting ALWAYS, which replay any batch,
and for builders that cannot replay variable-length batches.
"""
return None
def _init_reorder_batch_threshold(
self,
reorder_batch_threshold: int | None = 1,
@@ -632,14 +676,9 @@ class AttentionMetadataBuilder(ABC, Generic[M]):
speculative_config is not None
and speculative_config.num_speculative_tokens is not None
):
max_num_queries_for_spec = (
1
+ (2 if speculative_config.parallel_drafting else 1)
* speculative_config.num_speculative_tokens
)
self.reorder_batch_threshold = max(
self.reorder_batch_threshold,
max_num_queries_for_spec,
max_decode_query_len(self.vllm_config),
)
if (
+56 -18
View File
@@ -67,6 +67,7 @@ from vllm.v1.attention.backend import (
AttentionType,
CommonAttentionMetadata,
MultipleOf,
max_decode_query_len,
)
from vllm.v1.attention.backends.utils import (
get_dcp_local_seq_lens,
@@ -856,15 +857,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self.use_xqa = (
self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA
)
# Adaptive verification trims drafts on device, so decode query lengths
# must come from the device qo_indptr; only trtllm-gen supports that
# (the selector already rejects the other configurations).
self.use_trtllm_gen_varlen_decode = (
speculative_config is not None
and speculative_config.enable_adaptive_verification
and self.flashinfer_trtllm_api_decode_kernel
== FlashInferDecodeKernel.TRTLLM_GEN
and not self.use_dcp
self.use_trtllm_gen_varlen_decode = self._uses_trtllm_gen_varlen_decode(
vllm_config, self.flashinfer_trtllm_api_decode_kernel, self.use_dcp
)
self._init_reorder_batch_threshold(
1,
@@ -1011,6 +1005,31 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
else:
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
@override # type: ignore[misc]
@classmethod
def get_varlen_cudagraph_max_query_len(
cls: type["FlashInferMetadataBuilder"],
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
# The uniform gate already covers DCP, head geometry, the XQA head-dim
# limit and causality, and yields None for subclasses that force NEVER.
if (
cls.get_cudagraph_support(vllm_config, kv_cache_spec)
!= AttentionCGSupport.UNIFORM_BATCH
):
return None
# The kernel selector asserts off the trtllm-capable architectures, so
# call it only past the gate, as get_cudagraph_support does.
if not cls._uses_trtllm_gen_varlen_decode(
vllm_config,
cls._get_flashinfer_trtllm_api_decode_kernel(),
use_dcp=vllm_config.parallel_config.decode_context_parallel_size > 1,
):
return None
# Longer requests are split off as prefills.
return max_decode_query_len(vllm_config)
def _get_workspace_buffer(self):
if self._workspace_buffer is None:
buffer_size = envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
@@ -1039,6 +1058,24 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
def set_workspace_buffer(self, workspace_buffer: torch.Tensor):
self._workspace_buffer = workspace_buffer
@staticmethod
def _uses_trtllm_gen_varlen_decode(
vllm_config: VllmConfig,
decode_kernel: FlashInferDecodeKernel | None,
use_dcp: bool,
) -> bool:
"""Whether decode reads per-request query lengths from the device
qo_indptr. Adaptive verification trims drafts on device, and only
trtllm-gen supports that (the selector already rejects the other
configurations)."""
speculative_config = vllm_config.speculative_config
return (
speculative_config is not None
and speculative_config.enable_adaptive_verification
and decode_kernel == FlashInferDecodeKernel.TRTLLM_GEN
and not use_dcp
)
@staticmethod
def _get_flashinfer_trtllm_api_decode_kernel() -> FlashInferDecodeKernel:
if current_platform.is_device_capability(
@@ -1302,7 +1339,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold,
require_uniform=not self.use_xqa,
require_uniform=not (
self.use_xqa or self.use_trtllm_gen_varlen_decode
),
)
)
else:
@@ -1704,15 +1743,14 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
bool(causal),
)
elif self.use_trtllm_gen_varlen_decode:
# CPU lens are only an upper bound, device qo_indptr is
# the truth.
decode_q_lens = (
qo_indptr_cpu[1 : num_decodes + 1] - qo_indptr_cpu[:num_decodes]
# CPU placeholders may underestimate query lengths;
# use a safe upper bound.
q_len_per_req = min(
common_attn_metadata.max_query_len, self.reorder_batch_threshold
)
max_q_upper = int(decode_q_lens.max())
if max_q_upper > 1:
q_len_per_req = max_q_upper
q_cu_seq_lens = qo_indptr[: num_decodes + 1]
# Always pass GPU offsets so a graph captured with uniform
# query lengths can replay batches with variable query lengths.
q_cu_seq_lens = qo_indptr[: num_decodes + 1]
attn_metadata.decode = FlashInferTrtllmAPIDecode(
kernel=self.flashinfer_trtllm_api_decode_kernel,
block_tables=block_table_tensor[:num_decodes],
+2 -10
View File
@@ -12,6 +12,7 @@ from vllm import _custom_ops as ops
from vllm.config import VllmConfig
from vllm.forward_context import in_piecewise_cudagraph
from vllm.utils.torch_utils import current_stream
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.attention.backends.mla.sparse_utils import (
triton_convert_req_index_to_global_index,
)
@@ -496,17 +497,8 @@ class SparseMLAIndexGroupBuilder:
def get_sparse_mla_index_group_max_rows(vllm_config: VllmConfig) -> int:
max_query_len = 1
speculative_config = vllm_config.speculative_config
if (
speculative_config is not None
and speculative_config.num_speculative_tokens is not None
):
max_query_len += speculative_config.num_speculative_tokens * (
2 if speculative_config.parallel_drafting else 1
)
scheduler_config = vllm_config.scheduler_config
return min(
scheduler_config.max_num_batched_tokens,
scheduler_config.max_num_seqs * max_query_len,
scheduler_config.max_num_seqs * max_decode_query_len(vllm_config),
)
+4 -7
View File
@@ -26,6 +26,7 @@ from vllm.v1.attention.backend import (
AttentionMetadataBuilder,
CommonAttentionMetadata,
MultipleOf,
max_decode_query_len,
)
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
@@ -456,14 +457,10 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
self.num_speculative_tokens = (
spec_config.num_speculative_tokens if spec_config else 0
)
# Decode can have query_len up to
# 1 + (2 if parallel drafting else 1) * num_speculative_tokens.
# sparse_swa has no MQA-vs-dense-MHA routing, so multi-token queries take
# the prefill path and the decode/prefill split stays at that width.
spec_mult = (
2 if (spec_config is not None and spec_config.parallel_drafting) else 1
)
self.decode_threshold = 1 + spec_mult * self.num_speculative_tokens
# the prefill path and the decode/prefill split stays at the widest
# decode.
self.decode_threshold = max_decode_query_len(self.vllm_config)
self.reorder_batch_threshold = None
hf_config = self.vllm_config.model_config.hf_config
+4 -10
View File
@@ -27,6 +27,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
triton_scalar_specialization_rep,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.attention.ops.cp_common import (
DirectCPWorkspace,
direct_cp_enabled,
@@ -1011,16 +1012,9 @@ def dcp_a2a_lse_reduce(
def get_dcp_workspace_max_num_tokens(vllm_config: VllmConfig) -> int:
scheduler_config = vllm_config.scheduler_config
speculative_config = vllm_config.speculative_config
speculative_tokens = vllm_config.num_speculative_tokens
tokens_per_seq = (
1
+ (
2
if speculative_config is not None and speculative_config.parallel_drafting
else 1
)
* speculative_tokens
# num_speculative_tokens also counts a diffusion canvas.
tokens_per_seq = max(
1 + vllm_config.num_speculative_tokens, max_decode_query_len(vllm_config)
)
return min(
scheduler_config.max_num_batched_tokens,
+3 -14
View File
@@ -21,6 +21,7 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import current_stream
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
@@ -46,20 +47,8 @@ def is_hisparse_decode_batch(attn_metadata: Any | None) -> bool:
)
def _get_max_decode_query_len(vllm_config: VllmConfig) -> int:
speculative_config = getattr(vllm_config, "speculative_config", None)
if (
speculative_config is not None
and speculative_config.num_speculative_tokens is not None
):
return 1 + speculative_config.num_speculative_tokens * (
2 if speculative_config.parallel_drafting else 1
)
return 1
def _get_max_swap_rows(vllm_config: VllmConfig) -> int:
max_query_len = _get_max_decode_query_len(vllm_config)
max_query_len = max_decode_query_len(vllm_config)
scheduler_config = vllm_config.scheduler_config
return min(
scheduler_config.max_num_batched_tokens,
@@ -120,7 +109,7 @@ class ResolvedHiSparseConfig:
if config is None:
return None
max_query_len = _get_max_decode_query_len(vllm_config)
max_query_len = max_decode_query_len(vllm_config)
min_device_buffer_size = max_query_len * model_top_k
# Retain the worst-case union of every speculative step's top-k and
# leave one additional top-k of LRU slack.
+34
View File
@@ -328,6 +328,40 @@ def get_query_lens_mismatch_unsupported_backend(
return None
def get_varlen_cudagraph_unsupported_backend(
attn_groups: list[list[AttentionGroup]],
vllm_config: VllmConfig,
max_query_len: int,
checked_layer_names: set[str] | None = None,
) -> tuple[str, int | None] | None:
"""Name the first backend whose FULL cudagraphs cannot replay decode batches
of 1 to max_query_len tokens per request, with its bound, if any.
ALWAYS builders replay any batch; the others need a bound of at least
max_query_len. See AttentionMetadataBuilder.get_varlen_cudagraph_max_query_len().
"""
for groups in attn_groups:
for group in groups:
if checked_layer_names is not None and checked_layer_names.isdisjoint(
group.layer_names
):
continue
builder = group.get_metadata_builder(0)
spec = group.kv_cache_spec
support = builder.get_cudagraph_support(vllm_config, spec)
if support == AttentionCGSupport.ALWAYS:
continue
bound = (
# NEVER means never, whatever an inherited override says.
None
if support == AttentionCGSupport.NEVER
else builder.get_varlen_cudagraph_max_query_len(vllm_config, spec)
)
if bound is None or bound < max_query_len:
return group.backend.__name__, bound
return None
def init_kv_cache(
forward_context: dict[str, Any],
kv_cache_config: KVCacheConfig,
+2 -1
View File
@@ -69,7 +69,8 @@ class BatchExecutionDescriptor:
num_reqs: int | None # None means no request padding is needed (PIECEWISE graphs)
uniform_token_count: int | None = None
# Upper bound on per-request query length. Varlen decode graphs leave
# uniform_token_count unset, so this is what keeps a prefill batch out of one.
# uniform_token_count unset, so this is what keeps a prefill batch out of one:
# the runner passes None for any batch with a prefill.
max_query_len: int | None = None
num_active_loras: int = 0
# Number of microbatches the batch is split into (DBO). 1 means no splitting.
+2 -1
View File
@@ -243,7 +243,8 @@ def dispatch_cg_and_sync_dp(
dp_size: Data-parallel world size. 1 skips all cross-rank work.
dp_rank: This rank's index in the DP group.
max_query_len: Upper bound on per-request query length, for selecting
varlen decode graphs. None means the graph must not constrain it.
varlen decode graphs. None, as for a batch with a prefill, keeps
the batch out of graphs that constrain it.
need_eager: Force `CUDAGraphMode.NONE` instead of dispatching.
num_active_loras: Active LoRA count for this rank. Does not need
cross-rank agreement; it never changes a bucket's token count.
+5 -2
View File
@@ -640,7 +640,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculator, "enable_adaptive_verification", False
),
attn_groups=self.attn_groups,
attn_cg_support=attn_cg_support,
req_states=self.req_states,
query_start_loc=self.input_buffers.query_start_loc,
num_bonus_tokens=self.model_state.num_new_sampled_tokens_per_step,
@@ -1646,12 +1645,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Get batch descriptor and sync across DP ranks.
num_reqs = len(scheduler_output.num_scheduled_tokens)
num_toks = scheduler_output.total_num_scheduled_tokens
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
max_query_len: int | None = max(scheduler_output.num_scheduled_tokens.values())
batch_req_state, uniform_tok_count = self.gather_batch_req_state(
scheduler_output, dummy_run
)
if batch_req_state is not None:
num_toks = batch_req_state.num_tokens
if batch_req_state.has_prefill:
# Varlen decode graphs replay decodes only, and their bound
# alone would admit a short prefill.
max_query_len = None
if self.pcp_manager is not None:
num_toks = self.pcp_manager.get_num_tokens_for_dispatch(
batch_req_state.num_scheduled_tokens,
@@ -19,8 +19,8 @@ from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.utils import CpuGpuBuffer
from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream
from vllm.v1.worker.gpu.attn_utils import (
get_attn_cg_support,
get_query_lens_mismatch_unsupported_backend,
get_varlen_cudagraph_unsupported_backend,
)
logger = init_logger(__name__)
@@ -28,7 +28,6 @@ _PROFILE_REPLAYS = 5
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.utils import AttentionGroup
@@ -452,7 +451,6 @@ def maybe_create_adaptive_verification_manager(
*,
enable_adaptive_verification: bool,
attn_groups: list[list["AttentionGroup"]],
attn_cg_support: "AttentionCGSupportInfo",
req_states: "RequestState",
query_start_loc: torch.Tensor,
num_bonus_tokens: int,
@@ -477,25 +475,30 @@ def maybe_create_adaptive_verification_manager(
"use a backend that does."
)
target_attn_cg_support = attn_cg_support
if target_layer_names is not None:
target_attn_cg_support = get_attn_cg_support(
# The runner's decode_query_len, the width varlen decode graphs capture.
max_query_len = req_states.num_speculative_steps + num_bonus_tokens
unsupported: tuple[str | None, int | None] | None = (
get_varlen_cudagraph_unsupported_backend(
attn_groups,
vllm_config,
max_query_len,
checked_layer_names=target_layer_names,
)
if additional_attn_cg_support is not None:
target_attn_cg_support = target_attn_cg_support.narrow(
*additional_attn_cg_support
)
if target_attn_cg_support.min_cg_support != AttentionCGSupport.ALWAYS:
)
if unsupported is None and additional_attn_cg_support is not None:
# Groups built outside init_attn_backend report only their support
# level, and without a bound only ALWAYS replays varlen batches.
additional_support, additional_backend = additional_attn_cg_support
if additional_support != AttentionCGSupport.ALWAYS:
unsupported = additional_backend, None
if unsupported is not None:
backend, bound = unsupported
allowed = "none" if bound is None else f"at most {bound}"
raise ValueError(
"Adaptive verification captures varlen decode cudagraphs, so every"
" target attention builder must report AttentionCGSupport.ALWAYS, but "
f"{target_attn_cg_support.min_cg_attn_backend} reports "
f"{target_attn_cg_support.min_cg_support}. Pass "
"enable_adaptive_verification=false in the speculative config, or "
"use a backend that does."
"Adaptive verification replays decode cudagraphs whose per-request "
f"query lengths vary up to {max_query_len}, but {backend} allows "
f"{allowed}. Pass enable_adaptive_verification=false in the "
"speculative config, or use a backend that does."
)
return AdaptiveVerificationManager(