[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-09-26 17:01:17 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5.5 Nick Hill
parent 3b4566c5cf
commit a4eb3f25d6
24 changed files with 894 additions and 135 deletions
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for v1 attention backends without GPUModelRunner dependency."""
from dataclasses import replace
from functools import partial
from types import SimpleNamespace
import pytest
import torch
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
from transformers import LlamaConfig
from tests.v1.attention.utils import (
BatchSpec,
@@ -28,8 +30,10 @@ from vllm.utils.torch_utils import (
)
from vllm.v1.attention.backend import (
AttentionCGSupport,
AttentionMetadataBuilder,
AttentionType,
CommonAttentionMetadata,
max_decode_query_len,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
@@ -1066,6 +1070,378 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
assert q_cu_seq_lens.tolist() == qo_indptr_values
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
@pytest.mark.parametrize(
"adaptive,decode_kernel,dcp_size,expected_bound",
[
(None, "TRTLLM_GEN", 1, None),
(False, "TRTLLM_GEN", 1, None),
(True, "TRTLLM_GEN", 1, 8),
(True, "XQA", 1, None),
(True, "TRTLLM_GEN", 2, None),
],
)
def test_flashinfer_varlen_cudagraph_capability(
monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound
):
"""Only the trtllm-gen path that adaptive verification enables replays
varlen decode graphs, up to the decode width; the uniform level is fixed."""
from vllm.model_executor.layers.attention.chunked_local_attention import (
create_chunked_local_attention_backend,
)
from vllm.v1.attention.backends import flashinfer as fi
config = SimpleNamespace(
attention_config=SimpleNamespace(use_non_causal=False),
speculative_config=SimpleNamespace(
enable_adaptive_verification=adaptive,
num_speculative_tokens=7,
parallel_drafting=True,
)
if adaptive is not None
else None,
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32),
use_v2_model_runner=True,
)
monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True)
monkeypatch.setattr(
fi.FlashInferMetadataBuilder,
"_get_flashinfer_trtllm_api_decode_kernel",
staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]),
)
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=4,
head_size=64,
dtype=torch.bfloat16,
)
builder_cls = fi.FlashInferMetadataBuilder
assert builder_cls.get_cudagraph_support(config, spec) == (
AttentionCGSupport.UNIFORM_BATCH
if dcp_size == 1
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec)
assert bound == expected_bound
if bound is not None:
# A wrapper forcing NEVER inherits the override but not the bound.
chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16)
wrapped_cls = chunked_local.get_builder_cls()
assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None
@pytest.mark.parametrize(
"use_v2_model_runner,parallel_drafting,expected",
[
(True, None, 1),
(True, False, 8),
(True, True, 8),
(False, False, 8),
(False, True, 15),
],
)
def test_spec_as_decode_width_follows_model_runner(
use_v2_model_runner, parallel_drafting, expected
):
"""Only model runner V1's parallel drafter widens decode requests."""
class Builder(AttentionMetadataBuilder):
def __init__(self, vllm_config):
self.vllm_config = vllm_config
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
raise NotImplementedError
config = SimpleNamespace(
speculative_config=None
if parallel_drafting is None
else SimpleNamespace(
num_speculative_tokens=7, parallel_drafting=parallel_drafting
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
use_v2_model_runner=use_v2_model_runner,
)
assert max_decode_query_len(config) == expected
builder = Builder(config)
builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
assert builder.reorder_batch_threshold == expected
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
or not current_platform.is_device_capability_family(100),
reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs",
)
@pytest.mark.parametrize(
"head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads",
[
pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"),
pytest.param(
256,
1024,
40,
32,
16,
8,
id="gemma4-sliding-gqa",
),
pytest.param(
512,
None,
40,
32,
16,
1,
id="gemma4-global-mqa",
),
],
)
def test_flashinfer_varlen_decode_graph_replays_changed_layout(
tmp_path,
monkeypatch,
head_size,
sliding_window,
num_tokens,
num_reqs,
num_heads,
num_kv_heads,
):
"""Replay changes device offsets, active slots, context, and KV writes.
The graph is captured with evenly spread dummy queries and an explicit
bound of eight. CPU offsets then remain stale while device offsets change.
Compare real output rows with independent FP32 causal/windowed attention,
and verify cache writes including preservation of the null padding block.
"""
from vllm.v1.attention.backends import flashinfer as fi
from vllm.v1.attention.backends.utils import PerLayerParameters
set_random_seed(0)
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
block_size = 16
dtype = torch.bfloat16
LlamaConfig(
architectures=["LlamaForCausalLM"],
hidden_size=num_heads * head_size,
head_dim=head_size,
num_attention_heads=num_heads,
num_key_value_heads=num_kv_heads,
num_hidden_layers=1,
max_position_embeddings=4096,
).save_pretrained(tmp_path)
num_blocks = 1 + num_reqs * cdiv(2112, block_size)
config = create_vllm_config(
model_name=str(tmp_path),
max_model_len=4096,
dtype=dtype,
num_gpu_blocks=num_blocks,
block_size=block_size,
max_num_seqs=32,
max_num_batched_tokens=256,
)
config.cache_config.kv_cache_layout = "BLHNC"
config.speculative_config = SimpleNamespace(
num_speculative_tokens=7,
enable_adaptive_verification=True,
parallel_drafting=False,
)
spec = create_standard_kv_cache_spec(config)
scale = head_size**-0.5
monkeypatch.setattr(
fi,
"get_per_layer_parameters",
lambda *_: {
"layer": PerLayerParameters(
window_left=-1 if sliding_window is None else sliding_window - 1,
logits_soft_cap=0.0,
sm_scale=scale,
has_sinks=False,
),
},
)
device = torch.device("cuda")
layout = KVCacheLayout.BLHNC
query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device)
key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device)
value = torch.empty_like(key)
output = torch.empty_like(query)
dummy_lens = [
num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs)
]
common = create_common_attn_metadata(
BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens),
block_size,
device,
)
common.max_query_len = 8
cache = None
def prepare(query_lens):
nonlocal cache
query_lens = query_lens + [0] * (num_reqs - len(query_lens))
query.normal_()
key.normal_()
value.normal_()
# Alternating short prefills, nonempty decode contexts, and contexts
# beyond the sliding window; inactive requests have zero sequence length.
contexts = [
([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens)
]
ks = [
torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device)
for c in contexts
]
vs = [torch.randn_like(k) for k in ks]
actual = create_common_attn_metadata(
BatchSpec(
seq_lens=[c + q for c, q in zip(contexts, query_lens)],
query_lens=query_lens,
),
block_size,
device,
)
fresh_cache = create_and_prepopulate_kv_cache(
ks,
vs,
block_size,
num_kv_heads,
head_size,
dtype,
device,
num_blocks,
actual,
layout,
randomize_blocks=True,
)
if cache is None:
cache = fresh_cache
else:
cache.copy_(fresh_cache)
common.query_start_loc.copy_(actual.query_start_loc)
common.seq_lens.copy_(actual.seq_lens)
common.block_table_tensor.zero_()
common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_(
actual.block_table_tensor
)
common.slot_mapping.fill_(-1)
common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping)
return contexts, ks, vs
prepare(dummy_lens)
with set_current_vllm_config(config):
builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device)
metadata = builder.build(0, common)
assert metadata.num_prefills == 0
assert metadata.decode.q_len_per_req == 8
impl = fi.FlashInferImpl(
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
scale=scale,
alibi_slopes=None,
sliding_window=sliding_window,
kv_cache_dtype="auto",
attn_type=AttentionType.DECODER,
)
layer = MockAttentionLayer(device)
def forward(attn_metadata=metadata):
impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
impl.forward(layer, query, key, value, cache, attn_metadata, output=output)
for _ in range(3):
forward()
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
forward()
layouts = [[8], [1] * num_reqs]
if num_tokens >= 16:
layouts.append([8, 4, 3, 1])
if num_tokens >= 40:
layouts.append([8] * 5)
def check(query_lens, run):
contexts, ks, vs = prepare(query_lens)
assert cache is not None
output.fill_(float("nan"))
run()
start = 0
for i, q_len in enumerate(query_lens):
if q_len == 0:
continue
end = start + q_len
full_k = (
torch.cat((ks[i], key[start:end]))
.float()
.repeat_interleave(num_heads // num_kv_heads, dim=1)
)
full_v = (
torch.cat((vs[i], value[start:end]))
.float()
.repeat_interleave(num_heads // num_kv_heads, dim=1)
)
scores = (
torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale
)
q_pos = torch.arange(q_len, device=device) + contexts[i]
k_pos = torch.arange(full_k.shape[0], device=device)
allowed = k_pos[None, :] <= q_pos[:, None]
if sliding_window is not None:
allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window
probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1)
reference = torch.einsum("hqk,khd->qhd", probs, full_v)
torch.testing.assert_close(
output[start:end].float(),
reference,
atol=2e-2,
rtol=2e-2,
)
slots = common.slot_mapping[start:end]
written = cache[slots // block_size, :, slots % block_size, :]
torch.testing.assert_close(written[..., :head_size], key[start:end])
torch.testing.assert_close(written[..., head_size:], value[start:end])
start = end
assert torch.count_nonzero(cache[0]) == 0
for query_lens in layouts:
check(query_lens, graph.replay)
if num_tokens == 40:
# A longer prefill forces a separate launch in eager/piecewise mode.
# Its length must not become the decode kernel's query bound, and
# compacted CPU placeholders still need not match the GPU allocation.
common.num_reqs = 5
common.max_query_len = 24
common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40])
common.query_start_loc_cpu[6:] = 40
def mixed_forward():
with set_current_vllm_config(config):
metadata = builder.build(
0,
replace(
common,
query_start_loc=common.query_start_loc[:6],
query_start_loc_cpu=common.query_start_loc_cpu[:6],
seq_lens=common.seq_lens[:5],
seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5],
block_table_tensor=common.block_table_tensor[:5],
),
)
assert metadata.num_decodes == 4
assert metadata.num_prefills == 1
assert metadata.decode.q_len_per_req == 8
forward(metadata)
check([8, 4, 3, 1, 24], mixed_forward)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
@@ -294,6 +294,7 @@ def _make_spec_decode_manager(
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
max_num_seqs: int = 8,
use_kda_recoverssm: bool = False,
varlen_decode: bool = False,
) -> gpu_cudagraph_utils.CudaGraphManager:
monkeypatch.setattr(
gpu_cudagraph_utils,
@@ -316,6 +317,7 @@ def _make_spec_decode_manager(
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=decode_query_len,
varlen_decode=varlen_decode,
)
manager._graphs_captured = True
return manager
@@ -373,6 +375,28 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch):
assert desc.num_tokens == 16
def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch):
"""A varlen decode graph replays any decode batch within its query-length
bound. The runner passes no bound for a batch with a prefill, which must not
replay one however short the prefill is."""
manager = _make_spec_decode_manager(monkeypatch, varlen_decode=True)
def dispatch(max_query_len):
return manager.dispatch(
num_reqs=3,
num_tokens=6,
uniform_token_count=None,
num_active_loras=0,
max_query_len=max_query_len,
)
desc = dispatch(_DECODE_QUERY_LEN)
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.max_query_len == _DECODE_QUERY_LEN
assert dispatch(None).cg_mode == CUDAGraphMode.PIECEWISE
assert dispatch(_DECODE_QUERY_LEN + 1).cg_mode == CUDAGraphMode.PIECEWISE
def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch):
"""A mixed batch must not fall to eager where only decode graphs are staged.
@@ -1,15 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
import numpy as np
import pytest
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backend import AttentionCGSupport, AttentionMetadataBuilder
from vllm.v1.worker.gpu.async_utils import StepTimingSample
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_module
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
AdaptiveVerificationManager,
@@ -58,37 +58,90 @@ def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expe
)
def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch):
class Backend:
@classmethod
def supports_device_cpu_query_lens_mismatch(cls):
return True
@pytest.mark.parametrize(
"target_support,target_bound,device_offsets,additional,error",
[
pytest.param(AttentionCGSupport.ALWAYS, None, True, None, None, id="always"),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH, 8, True, None, None, id="bounded"
),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH,
None,
True,
None,
"TargetBackend allows none",
id="unsupported",
),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH,
7,
True,
None,
"up to 8, but TargetBackend allows at most 7",
id="too-narrow",
),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH,
8,
False,
None,
"trims verification requests",
id="host-query-lens",
),
pytest.param(
AttentionCGSupport.ALWAYS,
None,
True,
(AttentionCGSupport.UNIFORM_BATCH, "EncoderBackend"),
"EncoderBackend allows none",
id="additional-group",
),
],
)
def test_manager_checks_target_varlen_cudagraph_bound(
monkeypatch, target_support, target_bound, device_offsets, additional, error
):
"""Target builders must replay varlen decode graphs of the verification
width; draft-only groups are not checked."""
class Builder:
def __init__(self, support):
self.support = support
def group(backend_name, layer_name, support, bound=None, mismatch=True):
class Builder(AttentionMetadataBuilder):
_cudagraph_support = support
def get_cudagraph_support(self, *_args):
return self.support
def __init__(self):
pass
def group(layer_name, support):
builder = Builder(support)
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
raise NotImplementedError
if bound is not None:
@classmethod
def get_varlen_cudagraph_max_query_len(cls, *_args):
return bound
backend = type(
backend_name,
(),
{"supports_device_cpu_query_lens_mismatch": staticmethod(lambda: mismatch)},
)
builder = Builder()
return SimpleNamespace(
layer_names=[layer_name],
backend=Backend,
backend=backend,
kv_cache_spec=None,
get_metadata_builder=lambda _index: builder,
)
groups = [
[
group("target", AttentionCGSupport.ALWAYS),
group("draft", AttentionCGSupport.UNIFORM_BATCH),
group(
"TargetBackend", "target", target_support, target_bound, device_offsets
),
group("DraftBackend", "draft", AttentionCGSupport.UNIFORM_BATCH),
]
]
runner_support = AttentionCGSupportInfo(
AttentionCGSupport.UNIFORM_BATCH, "DraftBackend"
)
created = object()
monkeypatch.setattr(
adaptive_module,
@@ -96,20 +149,19 @@ def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatc
lambda *_args, **_kwargs: created,
)
manager = maybe_create_adaptive_verification_manager(
enable_adaptive_verification=True,
attn_groups=groups,
attn_cg_support=runner_support,
req_states=object(),
query_start_loc=object(),
num_bonus_tokens=1,
max_total_logits=1,
vllm_config=None,
target_layer_names={"target"},
)
assert manager is created
assert runner_support.min_cg_support == AttentionCGSupport.UNIFORM_BATCH
with pytest.raises(ValueError, match=error) if error else nullcontext():
manager = maybe_create_adaptive_verification_manager(
enable_adaptive_verification=True,
attn_groups=groups,
req_states=SimpleNamespace(num_speculative_steps=7),
query_start_loc=object(),
num_bonus_tokens=1,
max_total_logits=1,
vllm_config=None,
target_layer_names={"target"},
additional_attn_cg_support=additional,
)
assert manager is created
def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
+42 -1
View File
@@ -8,6 +8,7 @@ never addressed by the logical view.
"""
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
@@ -99,12 +100,16 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
class _FakeMetadataBuilder:
def __init__(self, support: AttentionCGSupport):
def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None):
self.support = support
self.varlen_bound = varlen_bound
def get_cudagraph_support(self, *_args):
return self.support
def get_varlen_cudagraph_max_query_len(self, *_args):
return self.varlen_bound
class _TargetBackend:
@classmethod
@@ -183,6 +188,42 @@ def test_attention_checks_preserve_global_and_target_scoped_support():
)
def test_varlen_cudagraph_unsupported_backend_checks_scoped_bounds():
"""ALWAYS passes without a bound, other builders need one at least as wide as
the requested length, and NEVER fails whatever bound a builder reports."""
config: Any = SimpleNamespace()
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
)
def group(
backend: Any,
layer_name: str,
support: AttentionCGSupport,
bound: int | None = None,
):
builder: Any = _FakeMetadataBuilder(support, bound)
attn_group = AttentionGroup(backend, [layer_name], spec, 0)
attn_group.metadata_builders = [builder]
return attn_group
target = group(_TargetBackend, "target", AttentionCGSupport.ALWAYS)
draft = group(_DraftBackend, "draft", AttentionCGSupport.UNIFORM_BATCH, 8)
never = group(_DraftBackend, "never", AttentionCGSupport.NEVER, 8)
unsupported = attn_utils.get_varlen_cudagraph_unsupported_backend
assert unsupported([[target, draft]], config, 8) is None
assert unsupported([[target, draft]], config, 9) == ("_DraftBackend", 8)
assert (
unsupported([[target, draft]], config, 9, checked_layer_names={"target"})
is None
)
assert unsupported([[target, never]], config, 1) == ("_DraftBackend", None)
def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyPatch):
"""Fast prefill applies to the contiguous suffix of KV-sharing layers.