[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-09-26 17:01:17 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5.5 Nick Hill
parent 3b4566c5cf
commit a4eb3f25d6
24 changed files with 894 additions and 135 deletions
@@ -455,13 +455,29 @@ def test_mla_dcp_manager_selects_fallback_backends(monkeypatch):
)
def test_dcp_workspace_covers_parallel_drafting():
@pytest.mark.parametrize(
("parallel_drafting", "use_v2_model_runner", "expected"),
[
# Model runner V1's parallel drafter widens requests to 1 + 2 * 3.
(True, False, 28),
(True, True, 16),
# A diffusion canvas of 3 without a speculative config.
(None, True, 16),
],
)
def test_dcp_workspace_covers_decode_width(
parallel_drafting, use_v2_model_runner, expected
):
config = _manager_config()
config.scheduler_config.max_num_batched_tokens = 128
config.num_speculative_tokens = 3
config.speculative_config = MagicMock(parallel_drafting=True)
config.use_v2_model_runner = use_v2_model_runner
if parallel_drafting is not None:
config.speculative_config = MagicMock(
num_speculative_tokens=3, parallel_drafting=parallel_drafting
)
assert dcp.get_dcp_workspace_max_num_tokens(config) == 28
assert dcp.get_dcp_workspace_max_num_tokens(config) == expected
def test_mla_dcp_manager_selects_pcp_combine(monkeypatch):
+67 -2
View File
@@ -7,7 +7,8 @@ import numpy as np
import pytest
import torch
from vllm.config import VllmConfig
from vllm.config import CompilationConfig, VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models import gemma
from vllm.model_executor.models.gemma3n import (
@@ -18,10 +19,72 @@ from vllm.model_executor.models.gemma4 import (
Gemma4ForCausalLM,
_gemma4_layer_weights_mapper,
)
from vllm.model_executor.models.gemma4_dspark import (
Gemma4DSparkForCausalLM,
Gemma4DSparkModel,
)
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize(
"enabled,has_weights,with_markov",
[
pytest.param(True, True, False, id="hidden-only"),
pytest.param(True, True, True, id="with-markov"),
pytest.param(True, False, True, id="missing-weights"),
pytest.param(False, True, True, id="disabled-head"),
],
)
def test_gemma4_dspark_loads_confidence_head(
monkeypatch, enabled, has_weights, with_markov
) -> None:
"""Use checkpoint confidence parameters, or disable an unavailable head."""
config = SimpleNamespace(
vocab_size=64,
hidden_size=8,
target_layer_ids=[0, 1],
num_hidden_layers=0,
rms_norm_eps=1e-6,
markov_rank=4,
enable_confidence_head=enabled,
confidence_head_with_markov=with_markov,
)
vllm_config = SimpleNamespace(
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
model_config=SimpleNamespace(dtype=torch.bfloat16),
speculative_config=SimpleNamespace(
draft_model_config=SimpleNamespace(hf_config=config),
),
)
monkeypatch.setattr(Gemma4DSparkModel, "_build_fused_kv_buffers", lambda _: None)
model = Gemma4DSparkForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
width = config.hidden_size + (config.markov_rank if with_markov else 0)
weight = torch.arange(width, dtype=torch.bfloat16).reshape(1, -1) / 16
bias = torch.tensor([-0.5], dtype=torch.bfloat16)
loaded = model.load_weights(
[("confidence_head.proj.weight", weight), ("confidence_head.proj.bias", bias)]
if has_weights
else []
)
if not (enabled and has_weights):
assert model.model.confidence_head is None
assert not loaded
return
assert loaded == {
"model.confidence_head.proj.weight",
"model.confidence_head.proj.bias",
}
assert model.model.confidence_head.proj.weight.dtype == torch.float32
hidden = torch.full((3, config.hidden_size), 0.25, dtype=torch.bfloat16)
markov = torch.full((3, config.markov_rank), -0.5, dtype=torch.bfloat16)
inputs = torch.cat([hidden, markov], dim=-1) if with_markov else hidden
expected = (inputs.float() @ weight.float().T + bias.float()).sigmoid().squeeze(-1)
torch.testing.assert_close(model.compute_confidence(hidden, markov), expected)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
@@ -164,7 +227,9 @@ def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
) as llm:
if model == "google/gemma-3-4b-it":
normalizers = llm.llm.collective_rpc(
lambda self: self.model_runner.model.language_model.model.normalizer.cpu().item() # noqa: E501
lambda self: (
self.model_runner.model.language_model.model.normalizer.cpu().item()
) # noqa: E501
)
config = llm.llm.llm_engine.model_config.hf_config.text_config
else:
+12
View File
@@ -92,6 +92,16 @@ def test_nested_rope_validation_patch_preserves_flat_rope_parameters(monkeypatch
assert len(calls) == 2
def test_dspark_adaptive_verification_separates_graph_cache():
config = object.__new__(SpeculativeConfig)
config.method = "dspark"
config.draft_model_config = None
config.enable_adaptive_verification = False
fixed_hash = config.compute_hash()
config.enable_adaptive_verification = True
assert config.compute_hash() != fixed_hash
def _write_json(path: Path, value: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value), encoding="utf-8")
@@ -878,6 +888,8 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
("FULL_AND_PIECEWISE", False, "ALWAYS", "FULL_DECODE_ONLY"),
("FULL_DECODE_ONLY", False, "ALWAYS", "FULL_DECODE_ONLY"),
("FULL_DECODE_ONLY", False, "NEVER", "NONE"),
("FULL_AND_PIECEWISE", True, "UNIFORM_BATCH", "FULL_AND_PIECEWISE"),
("FULL", True, "UNIFORM_BATCH", "FULL_DECODE_ONLY"),
],
)
def test_resolve_cudagraph_mode_uses_loaded_piecewise_provider(
+45
View File
@@ -5,6 +5,7 @@ only get the `eos_token_id` from the tokenizer as defined by
`BaseRenderer.get_eos_token_id`.
"""
import json
import math
from types import SimpleNamespace
from typing import cast
@@ -31,6 +32,50 @@ from vllm.transformers_utils.configs.glm5_next import (
from vllm.transformers_utils.configs.mistral import adapt_config_dict
@pytest.mark.parametrize("layout", ["mixed", "flat"])
def test_gemma4_dspark_rope_config_preserves_parameters(tmp_path, layout):
"""Remove redundant shared entries while preserving per-layer and flat RoPE."""
from transformers import Gemma4TextConfig
per_layer = {
"full_attention": {
"rope_type": "proportional",
"partial_rotary_factor": 0.25,
"rope_theta": 1000000.0,
},
"sliding_attention": {"rope_type": "default", "rope_theta": 10000.0},
}
rope_parameters: dict[str, object] = dict(per_layer)
if layout == "mixed":
rope_parameters.update(rope_type="default", rope_theta=None)
else:
rope_parameters = {"rope_type": "default", "rope_theta": 12345.0}
(tmp_path / "config.json").write_text(
json.dumps(
{
"model_type": "gemma4_text",
"architectures": ["Gemma4DSparkModel"],
"num_hidden_layers": 1,
"layer_types": ["full_attention"],
"rope_parameters": rope_parameters,
}
)
)
_, config = config_module.HFConfigParser().parse(
tmp_path, trust_remote_code=False, max_position_embeddings=8192
)
assert isinstance(config, Gemma4TextConfig)
assert config.name_or_path == str(tmp_path)
assert config.max_position_embeddings == 8192
if layout == "flat":
assert config.rope_parameters["rope_theta"] == 12345.0
else:
for layer_type, expected in per_layer.items():
for key, value in expected.items():
assert config.rope_parameters[layer_type][key] == value
assert set(config.rope_parameters) == set(per_layer)
def test_patch_legacy_rope_type_preserves_nope_layers():
"""NoPE layers stay disabled while later RoPE layers are normalized."""
rope_parameters = {
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for v1 attention backends without GPUModelRunner dependency."""
from dataclasses import replace
from functools import partial
from types import SimpleNamespace
import pytest
import torch
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
from transformers import LlamaConfig
from tests.v1.attention.utils import (
BatchSpec,
@@ -28,8 +30,10 @@ from vllm.utils.torch_utils import (
)
from vllm.v1.attention.backend import (
AttentionCGSupport,
AttentionMetadataBuilder,
AttentionType,
CommonAttentionMetadata,
max_decode_query_len,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
@@ -1066,6 +1070,378 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
assert q_cu_seq_lens.tolist() == qo_indptr_values
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
@pytest.mark.parametrize(
"adaptive,decode_kernel,dcp_size,expected_bound",
[
(None, "TRTLLM_GEN", 1, None),
(False, "TRTLLM_GEN", 1, None),
(True, "TRTLLM_GEN", 1, 8),
(True, "XQA", 1, None),
(True, "TRTLLM_GEN", 2, None),
],
)
def test_flashinfer_varlen_cudagraph_capability(
monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound
):
"""Only the trtllm-gen path that adaptive verification enables replays
varlen decode graphs, up to the decode width; the uniform level is fixed."""
from vllm.model_executor.layers.attention.chunked_local_attention import (
create_chunked_local_attention_backend,
)
from vllm.v1.attention.backends import flashinfer as fi
config = SimpleNamespace(
attention_config=SimpleNamespace(use_non_causal=False),
speculative_config=SimpleNamespace(
enable_adaptive_verification=adaptive,
num_speculative_tokens=7,
parallel_drafting=True,
)
if adaptive is not None
else None,
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32),
use_v2_model_runner=True,
)
monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True)
monkeypatch.setattr(
fi.FlashInferMetadataBuilder,
"_get_flashinfer_trtllm_api_decode_kernel",
staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]),
)
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=4,
head_size=64,
dtype=torch.bfloat16,
)
builder_cls = fi.FlashInferMetadataBuilder
assert builder_cls.get_cudagraph_support(config, spec) == (
AttentionCGSupport.UNIFORM_BATCH
if dcp_size == 1
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec)
assert bound == expected_bound
if bound is not None:
# A wrapper forcing NEVER inherits the override but not the bound.
chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16)
wrapped_cls = chunked_local.get_builder_cls()
assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None
@pytest.mark.parametrize(
"use_v2_model_runner,parallel_drafting,expected",
[
(True, None, 1),
(True, False, 8),
(True, True, 8),
(False, False, 8),
(False, True, 15),
],
)
def test_spec_as_decode_width_follows_model_runner(
use_v2_model_runner, parallel_drafting, expected
):
"""Only model runner V1's parallel drafter widens decode requests."""
class Builder(AttentionMetadataBuilder):
def __init__(self, vllm_config):
self.vllm_config = vllm_config
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
raise NotImplementedError
config = SimpleNamespace(
speculative_config=None
if parallel_drafting is None
else SimpleNamespace(
num_speculative_tokens=7, parallel_drafting=parallel_drafting
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
use_v2_model_runner=use_v2_model_runner,
)
assert max_decode_query_len(config) == expected
builder = Builder(config)
builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
assert builder.reorder_batch_threshold == expected
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
or not current_platform.is_device_capability_family(100),
reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs",
)
@pytest.mark.parametrize(
"head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads",
[
pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"),
pytest.param(
256,
1024,
40,
32,
16,
8,
id="gemma4-sliding-gqa",
),
pytest.param(
512,
None,
40,
32,
16,
1,
id="gemma4-global-mqa",
),
],
)
def test_flashinfer_varlen_decode_graph_replays_changed_layout(
tmp_path,
monkeypatch,
head_size,
sliding_window,
num_tokens,
num_reqs,
num_heads,
num_kv_heads,
):
"""Replay changes device offsets, active slots, context, and KV writes.
The graph is captured with evenly spread dummy queries and an explicit
bound of eight. CPU offsets then remain stale while device offsets change.
Compare real output rows with independent FP32 causal/windowed attention,
and verify cache writes including preservation of the null padding block.
"""
from vllm.v1.attention.backends import flashinfer as fi
from vllm.v1.attention.backends.utils import PerLayerParameters
set_random_seed(0)
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
block_size = 16
dtype = torch.bfloat16
LlamaConfig(
architectures=["LlamaForCausalLM"],
hidden_size=num_heads * head_size,
head_dim=head_size,
num_attention_heads=num_heads,
num_key_value_heads=num_kv_heads,
num_hidden_layers=1,
max_position_embeddings=4096,
).save_pretrained(tmp_path)
num_blocks = 1 + num_reqs * cdiv(2112, block_size)
config = create_vllm_config(
model_name=str(tmp_path),
max_model_len=4096,
dtype=dtype,
num_gpu_blocks=num_blocks,
block_size=block_size,
max_num_seqs=32,
max_num_batched_tokens=256,
)
config.cache_config.kv_cache_layout = "BLHNC"
config.speculative_config = SimpleNamespace(
num_speculative_tokens=7,
enable_adaptive_verification=True,
parallel_drafting=False,
)
spec = create_standard_kv_cache_spec(config)
scale = head_size**-0.5
monkeypatch.setattr(
fi,
"get_per_layer_parameters",
lambda *_: {
"layer": PerLayerParameters(
window_left=-1 if sliding_window is None else sliding_window - 1,
logits_soft_cap=0.0,
sm_scale=scale,
has_sinks=False,
),
},
)
device = torch.device("cuda")
layout = KVCacheLayout.BLHNC
query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device)
key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device)
value = torch.empty_like(key)
output = torch.empty_like(query)
dummy_lens = [
num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs)
]
common = create_common_attn_metadata(
BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens),
block_size,
device,
)
common.max_query_len = 8
cache = None
def prepare(query_lens):
nonlocal cache
query_lens = query_lens + [0] * (num_reqs - len(query_lens))
query.normal_()
key.normal_()
value.normal_()
# Alternating short prefills, nonempty decode contexts, and contexts
# beyond the sliding window; inactive requests have zero sequence length.
contexts = [
([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens)
]
ks = [
torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device)
for c in contexts
]
vs = [torch.randn_like(k) for k in ks]
actual = create_common_attn_metadata(
BatchSpec(
seq_lens=[c + q for c, q in zip(contexts, query_lens)],
query_lens=query_lens,
),
block_size,
device,
)
fresh_cache = create_and_prepopulate_kv_cache(
ks,
vs,
block_size,
num_kv_heads,
head_size,
dtype,
device,
num_blocks,
actual,
layout,
randomize_blocks=True,
)
if cache is None:
cache = fresh_cache
else:
cache.copy_(fresh_cache)
common.query_start_loc.copy_(actual.query_start_loc)
common.seq_lens.copy_(actual.seq_lens)
common.block_table_tensor.zero_()
common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_(
actual.block_table_tensor
)
common.slot_mapping.fill_(-1)
common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping)
return contexts, ks, vs
prepare(dummy_lens)
with set_current_vllm_config(config):
builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device)
metadata = builder.build(0, common)
assert metadata.num_prefills == 0
assert metadata.decode.q_len_per_req == 8
impl = fi.FlashInferImpl(
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
scale=scale,
alibi_slopes=None,
sliding_window=sliding_window,
kv_cache_dtype="auto",
attn_type=AttentionType.DECODER,
)
layer = MockAttentionLayer(device)
def forward(attn_metadata=metadata):
impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
impl.forward(layer, query, key, value, cache, attn_metadata, output=output)
for _ in range(3):
forward()
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
forward()
layouts = [[8], [1] * num_reqs]
if num_tokens >= 16:
layouts.append([8, 4, 3, 1])
if num_tokens >= 40:
layouts.append([8] * 5)
def check(query_lens, run):
contexts, ks, vs = prepare(query_lens)
assert cache is not None
output.fill_(float("nan"))
run()
start = 0
for i, q_len in enumerate(query_lens):
if q_len == 0:
continue
end = start + q_len
full_k = (
torch.cat((ks[i], key[start:end]))
.float()
.repeat_interleave(num_heads // num_kv_heads, dim=1)
)
full_v = (
torch.cat((vs[i], value[start:end]))
.float()
.repeat_interleave(num_heads // num_kv_heads, dim=1)
)
scores = (
torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale
)
q_pos = torch.arange(q_len, device=device) + contexts[i]
k_pos = torch.arange(full_k.shape[0], device=device)
allowed = k_pos[None, :] <= q_pos[:, None]
if sliding_window is not None:
allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window
probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1)
reference = torch.einsum("hqk,khd->qhd", probs, full_v)
torch.testing.assert_close(
output[start:end].float(),
reference,
atol=2e-2,
rtol=2e-2,
)
slots = common.slot_mapping[start:end]
written = cache[slots // block_size, :, slots % block_size, :]
torch.testing.assert_close(written[..., :head_size], key[start:end])
torch.testing.assert_close(written[..., head_size:], value[start:end])
start = end
assert torch.count_nonzero(cache[0]) == 0
for query_lens in layouts:
check(query_lens, graph.replay)
if num_tokens == 40:
# A longer prefill forces a separate launch in eager/piecewise mode.
# Its length must not become the decode kernel's query bound, and
# compacted CPU placeholders still need not match the GPU allocation.
common.num_reqs = 5
common.max_query_len = 24
common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40])
common.query_start_loc_cpu[6:] = 40
def mixed_forward():
with set_current_vllm_config(config):
metadata = builder.build(
0,
replace(
common,
query_start_loc=common.query_start_loc[:6],
query_start_loc_cpu=common.query_start_loc_cpu[:6],
seq_lens=common.seq_lens[:5],
seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5],
block_table_tensor=common.block_table_tensor[:5],
),
)
assert metadata.num_decodes == 4
assert metadata.num_prefills == 1
assert metadata.decode.q_len_per_req == 8
forward(metadata)
check([8, 4, 3, 1, 24], mixed_forward)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
@@ -294,6 +294,7 @@ def _make_spec_decode_manager(
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
max_num_seqs: int = 8,
use_kda_recoverssm: bool = False,
varlen_decode: bool = False,
) -> gpu_cudagraph_utils.CudaGraphManager:
monkeypatch.setattr(
gpu_cudagraph_utils,
@@ -316,6 +317,7 @@ def _make_spec_decode_manager(
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=decode_query_len,
varlen_decode=varlen_decode,
)
manager._graphs_captured = True
return manager
@@ -373,6 +375,28 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch):
assert desc.num_tokens == 16
def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch):
"""A varlen decode graph replays any decode batch within its query-length
bound. The runner passes no bound for a batch with a prefill, which must not
replay one however short the prefill is."""
manager = _make_spec_decode_manager(monkeypatch, varlen_decode=True)
def dispatch(max_query_len):
return manager.dispatch(
num_reqs=3,
num_tokens=6,
uniform_token_count=None,
num_active_loras=0,
max_query_len=max_query_len,
)
desc = dispatch(_DECODE_QUERY_LEN)
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.max_query_len == _DECODE_QUERY_LEN
assert dispatch(None).cg_mode == CUDAGraphMode.PIECEWISE
assert dispatch(_DECODE_QUERY_LEN + 1).cg_mode == CUDAGraphMode.PIECEWISE
def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch):
"""A mixed batch must not fall to eager where only decode graphs are staged.
@@ -1,15 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
import numpy as np
import pytest
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backend import AttentionCGSupport, AttentionMetadataBuilder
from vllm.v1.worker.gpu.async_utils import StepTimingSample
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_module
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
AdaptiveVerificationManager,
@@ -58,37 +58,90 @@ def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expe
)
def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch):
class Backend:
@classmethod
def supports_device_cpu_query_lens_mismatch(cls):
return True
@pytest.mark.parametrize(
"target_support,target_bound,device_offsets,additional,error",
[
pytest.param(AttentionCGSupport.ALWAYS, None, True, None, None, id="always"),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH, 8, True, None, None, id="bounded"
),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH,
None,
True,
None,
"TargetBackend allows none",
id="unsupported",
),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH,
7,
True,
None,
"up to 8, but TargetBackend allows at most 7",
id="too-narrow",
),
pytest.param(
AttentionCGSupport.UNIFORM_BATCH,
8,
False,
None,
"trims verification requests",
id="host-query-lens",
),
pytest.param(
AttentionCGSupport.ALWAYS,
None,
True,
(AttentionCGSupport.UNIFORM_BATCH, "EncoderBackend"),
"EncoderBackend allows none",
id="additional-group",
),
],
)
def test_manager_checks_target_varlen_cudagraph_bound(
monkeypatch, target_support, target_bound, device_offsets, additional, error
):
"""Target builders must replay varlen decode graphs of the verification
width; draft-only groups are not checked."""
class Builder:
def __init__(self, support):
self.support = support
def group(backend_name, layer_name, support, bound=None, mismatch=True):
class Builder(AttentionMetadataBuilder):
_cudagraph_support = support
def get_cudagraph_support(self, *_args):
return self.support
def __init__(self):
pass
def group(layer_name, support):
builder = Builder(support)
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
raise NotImplementedError
if bound is not None:
@classmethod
def get_varlen_cudagraph_max_query_len(cls, *_args):
return bound
backend = type(
backend_name,
(),
{"supports_device_cpu_query_lens_mismatch": staticmethod(lambda: mismatch)},
)
builder = Builder()
return SimpleNamespace(
layer_names=[layer_name],
backend=Backend,
backend=backend,
kv_cache_spec=None,
get_metadata_builder=lambda _index: builder,
)
groups = [
[
group("target", AttentionCGSupport.ALWAYS),
group("draft", AttentionCGSupport.UNIFORM_BATCH),
group(
"TargetBackend", "target", target_support, target_bound, device_offsets
),
group("DraftBackend", "draft", AttentionCGSupport.UNIFORM_BATCH),
]
]
runner_support = AttentionCGSupportInfo(
AttentionCGSupport.UNIFORM_BATCH, "DraftBackend"
)
created = object()
monkeypatch.setattr(
adaptive_module,
@@ -96,20 +149,19 @@ def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatc
lambda *_args, **_kwargs: created,
)
manager = maybe_create_adaptive_verification_manager(
enable_adaptive_verification=True,
attn_groups=groups,
attn_cg_support=runner_support,
req_states=object(),
query_start_loc=object(),
num_bonus_tokens=1,
max_total_logits=1,
vllm_config=None,
target_layer_names={"target"},
)
assert manager is created
assert runner_support.min_cg_support == AttentionCGSupport.UNIFORM_BATCH
with pytest.raises(ValueError, match=error) if error else nullcontext():
manager = maybe_create_adaptive_verification_manager(
enable_adaptive_verification=True,
attn_groups=groups,
req_states=SimpleNamespace(num_speculative_steps=7),
query_start_loc=object(),
num_bonus_tokens=1,
max_total_logits=1,
vllm_config=None,
target_layer_names={"target"},
additional_attn_cg_support=additional,
)
assert manager is created
def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
+42 -1
View File
@@ -8,6 +8,7 @@ never addressed by the logical view.
"""
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
@@ -99,12 +100,16 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
class _FakeMetadataBuilder:
def __init__(self, support: AttentionCGSupport):
def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None):
self.support = support
self.varlen_bound = varlen_bound
def get_cudagraph_support(self, *_args):
return self.support
def get_varlen_cudagraph_max_query_len(self, *_args):
return self.varlen_bound
class _TargetBackend:
@classmethod
@@ -183,6 +188,42 @@ def test_attention_checks_preserve_global_and_target_scoped_support():
)
def test_varlen_cudagraph_unsupported_backend_checks_scoped_bounds():
"""ALWAYS passes without a bound, other builders need one at least as wide as
the requested length, and NEVER fails whatever bound a builder reports."""
config: Any = SimpleNamespace()
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
)
def group(
backend: Any,
layer_name: str,
support: AttentionCGSupport,
bound: int | None = None,
):
builder: Any = _FakeMetadataBuilder(support, bound)
attn_group = AttentionGroup(backend, [layer_name], spec, 0)
attn_group.metadata_builders = [builder]
return attn_group
target = group(_TargetBackend, "target", AttentionCGSupport.ALWAYS)
draft = group(_DraftBackend, "draft", AttentionCGSupport.UNIFORM_BATCH, 8)
never = group(_DraftBackend, "never", AttentionCGSupport.NEVER, 8)
unsupported = attn_utils.get_varlen_cudagraph_unsupported_backend
assert unsupported([[target, draft]], config, 8) is None
assert unsupported([[target, draft]], config, 9) == ("_DraftBackend", 8)
assert (
unsupported([[target, draft]], config, 9, checked_layer_names={"target"})
is None
)
assert unsupported([[target, never]], config, 1) == ("_DraftBackend", None)
def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyPatch):
"""Fast prefill applies to the contiguous suffix of KV-sharing layers.