[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5.5
Nick Hill
parent
3b4566c5cf
commit
a4eb3f25d6
@@ -455,13 +455,29 @@ def test_mla_dcp_manager_selects_fallback_backends(monkeypatch):
|
||||
)
|
||||
|
||||
|
||||
def test_dcp_workspace_covers_parallel_drafting():
|
||||
@pytest.mark.parametrize(
|
||||
("parallel_drafting", "use_v2_model_runner", "expected"),
|
||||
[
|
||||
# Model runner V1's parallel drafter widens requests to 1 + 2 * 3.
|
||||
(True, False, 28),
|
||||
(True, True, 16),
|
||||
# A diffusion canvas of 3 without a speculative config.
|
||||
(None, True, 16),
|
||||
],
|
||||
)
|
||||
def test_dcp_workspace_covers_decode_width(
|
||||
parallel_drafting, use_v2_model_runner, expected
|
||||
):
|
||||
config = _manager_config()
|
||||
config.scheduler_config.max_num_batched_tokens = 128
|
||||
config.num_speculative_tokens = 3
|
||||
config.speculative_config = MagicMock(parallel_drafting=True)
|
||||
config.use_v2_model_runner = use_v2_model_runner
|
||||
if parallel_drafting is not None:
|
||||
config.speculative_config = MagicMock(
|
||||
num_speculative_tokens=3, parallel_drafting=parallel_drafting
|
||||
)
|
||||
|
||||
assert dcp.get_dcp_workspace_max_num_tokens(config) == 28
|
||||
assert dcp.get_dcp_workspace_max_num_tokens(config) == expected
|
||||
|
||||
|
||||
def test_mla_dcp_manager_selects_pcp_combine(monkeypatch):
|
||||
|
||||
@@ -7,7 +7,8 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import CompilationConfig, VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models import gemma
|
||||
from vllm.model_executor.models.gemma3n import (
|
||||
@@ -18,10 +19,72 @@ from vllm.model_executor.models.gemma4 import (
|
||||
Gemma4ForCausalLM,
|
||||
_gemma4_layer_weights_mapper,
|
||||
)
|
||||
from vllm.model_executor.models.gemma4_dspark import (
|
||||
Gemma4DSparkForCausalLM,
|
||||
Gemma4DSparkModel,
|
||||
)
|
||||
|
||||
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,has_weights,with_markov",
|
||||
[
|
||||
pytest.param(True, True, False, id="hidden-only"),
|
||||
pytest.param(True, True, True, id="with-markov"),
|
||||
pytest.param(True, False, True, id="missing-weights"),
|
||||
pytest.param(False, True, True, id="disabled-head"),
|
||||
],
|
||||
)
|
||||
def test_gemma4_dspark_loads_confidence_head(
|
||||
monkeypatch, enabled, has_weights, with_markov
|
||||
) -> None:
|
||||
"""Use checkpoint confidence parameters, or disable an unavailable head."""
|
||||
config = SimpleNamespace(
|
||||
vocab_size=64,
|
||||
hidden_size=8,
|
||||
target_layer_ids=[0, 1],
|
||||
num_hidden_layers=0,
|
||||
rms_norm_eps=1e-6,
|
||||
markov_rank=4,
|
||||
enable_confidence_head=enabled,
|
||||
confidence_head_with_markov=with_markov,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(hf_config=config),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(Gemma4DSparkModel, "_build_fused_kv_buffers", lambda _: None)
|
||||
model = Gemma4DSparkForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
|
||||
width = config.hidden_size + (config.markov_rank if with_markov else 0)
|
||||
weight = torch.arange(width, dtype=torch.bfloat16).reshape(1, -1) / 16
|
||||
bias = torch.tensor([-0.5], dtype=torch.bfloat16)
|
||||
loaded = model.load_weights(
|
||||
[("confidence_head.proj.weight", weight), ("confidence_head.proj.bias", bias)]
|
||||
if has_weights
|
||||
else []
|
||||
)
|
||||
if not (enabled and has_weights):
|
||||
assert model.model.confidence_head is None
|
||||
assert not loaded
|
||||
return
|
||||
|
||||
assert loaded == {
|
||||
"model.confidence_head.proj.weight",
|
||||
"model.confidence_head.proj.bias",
|
||||
}
|
||||
assert model.model.confidence_head.proj.weight.dtype == torch.float32
|
||||
hidden = torch.full((3, config.hidden_size), 0.25, dtype=torch.bfloat16)
|
||||
markov = torch.full((3, config.markov_rank), -0.5, dtype=torch.bfloat16)
|
||||
inputs = torch.cat([hidden, markov], dim=-1) if with_markov else hidden
|
||||
expected = (inputs.float() @ weight.float().T + bias.float()).sigmoid().squeeze(-1)
|
||||
torch.testing.assert_close(model.compute_confidence(hidden, markov), expected)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
|
||||
@@ -164,7 +227,9 @@ def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
|
||||
) as llm:
|
||||
if model == "google/gemma-3-4b-it":
|
||||
normalizers = llm.llm.collective_rpc(
|
||||
lambda self: self.model_runner.model.language_model.model.normalizer.cpu().item() # noqa: E501
|
||||
lambda self: (
|
||||
self.model_runner.model.language_model.model.normalizer.cpu().item()
|
||||
) # noqa: E501
|
||||
)
|
||||
config = llm.llm.llm_engine.model_config.hf_config.text_config
|
||||
else:
|
||||
|
||||
@@ -92,6 +92,16 @@ def test_nested_rope_validation_patch_preserves_flat_rope_parameters(monkeypatch
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
def test_dspark_adaptive_verification_separates_graph_cache():
|
||||
config = object.__new__(SpeculativeConfig)
|
||||
config.method = "dspark"
|
||||
config.draft_model_config = None
|
||||
config.enable_adaptive_verification = False
|
||||
fixed_hash = config.compute_hash()
|
||||
config.enable_adaptive_verification = True
|
||||
assert config.compute_hash() != fixed_hash
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(value), encoding="utf-8")
|
||||
@@ -878,6 +888,8 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
|
||||
("FULL_AND_PIECEWISE", False, "ALWAYS", "FULL_DECODE_ONLY"),
|
||||
("FULL_DECODE_ONLY", False, "ALWAYS", "FULL_DECODE_ONLY"),
|
||||
("FULL_DECODE_ONLY", False, "NEVER", "NONE"),
|
||||
("FULL_AND_PIECEWISE", True, "UNIFORM_BATCH", "FULL_AND_PIECEWISE"),
|
||||
("FULL", True, "UNIFORM_BATCH", "FULL_DECODE_ONLY"),
|
||||
],
|
||||
)
|
||||
def test_resolve_cudagraph_mode_uses_loaded_piecewise_provider(
|
||||
|
||||
@@ -5,6 +5,7 @@ only get the `eos_token_id` from the tokenizer as defined by
|
||||
`BaseRenderer.get_eos_token_id`.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
@@ -31,6 +32,50 @@ from vllm.transformers_utils.configs.glm5_next import (
|
||||
from vllm.transformers_utils.configs.mistral import adapt_config_dict
|
||||
|
||||
|
||||
@pytest.mark.parametrize("layout", ["mixed", "flat"])
|
||||
def test_gemma4_dspark_rope_config_preserves_parameters(tmp_path, layout):
|
||||
"""Remove redundant shared entries while preserving per-layer and flat RoPE."""
|
||||
from transformers import Gemma4TextConfig
|
||||
|
||||
per_layer = {
|
||||
"full_attention": {
|
||||
"rope_type": "proportional",
|
||||
"partial_rotary_factor": 0.25,
|
||||
"rope_theta": 1000000.0,
|
||||
},
|
||||
"sliding_attention": {"rope_type": "default", "rope_theta": 10000.0},
|
||||
}
|
||||
rope_parameters: dict[str, object] = dict(per_layer)
|
||||
if layout == "mixed":
|
||||
rope_parameters.update(rope_type="default", rope_theta=None)
|
||||
else:
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": 12345.0}
|
||||
(tmp_path / "config.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_type": "gemma4_text",
|
||||
"architectures": ["Gemma4DSparkModel"],
|
||||
"num_hidden_layers": 1,
|
||||
"layer_types": ["full_attention"],
|
||||
"rope_parameters": rope_parameters,
|
||||
}
|
||||
)
|
||||
)
|
||||
_, config = config_module.HFConfigParser().parse(
|
||||
tmp_path, trust_remote_code=False, max_position_embeddings=8192
|
||||
)
|
||||
assert isinstance(config, Gemma4TextConfig)
|
||||
assert config.name_or_path == str(tmp_path)
|
||||
assert config.max_position_embeddings == 8192
|
||||
if layout == "flat":
|
||||
assert config.rope_parameters["rope_theta"] == 12345.0
|
||||
else:
|
||||
for layer_type, expected in per_layer.items():
|
||||
for key, value in expected.items():
|
||||
assert config.rope_parameters[layer_type][key] == value
|
||||
assert set(config.rope_parameters) == set(per_layer)
|
||||
|
||||
|
||||
def test_patch_legacy_rope_type_preserves_nope_layers():
|
||||
"""NoPE layers stay disabled while later RoPE layers are normalized."""
|
||||
rope_parameters = {
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for v1 attention backends without GPUModelRunner dependency."""
|
||||
|
||||
from dataclasses import replace
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
|
||||
from transformers import LlamaConfig
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
@@ -28,8 +30,10 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionCGSupport,
|
||||
AttentionMetadataBuilder,
|
||||
AttentionType,
|
||||
CommonAttentionMetadata,
|
||||
max_decode_query_len,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
|
||||
@@ -1066,6 +1070,378 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
|
||||
assert q_cu_seq_lens.tolist() == qo_indptr_values
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"adaptive,decode_kernel,dcp_size,expected_bound",
|
||||
[
|
||||
(None, "TRTLLM_GEN", 1, None),
|
||||
(False, "TRTLLM_GEN", 1, None),
|
||||
(True, "TRTLLM_GEN", 1, 8),
|
||||
(True, "XQA", 1, None),
|
||||
(True, "TRTLLM_GEN", 2, None),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_varlen_cudagraph_capability(
|
||||
monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound
|
||||
):
|
||||
"""Only the trtllm-gen path that adaptive verification enables replays
|
||||
varlen decode graphs, up to the decode width; the uniform level is fixed."""
|
||||
from vllm.model_executor.layers.attention.chunked_local_attention import (
|
||||
create_chunked_local_attention_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends import flashinfer as fi
|
||||
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(use_non_causal=False),
|
||||
speculative_config=SimpleNamespace(
|
||||
enable_adaptive_verification=adaptive,
|
||||
num_speculative_tokens=7,
|
||||
parallel_drafting=True,
|
||||
)
|
||||
if adaptive is not None
|
||||
else None,
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
|
||||
model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32),
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True)
|
||||
monkeypatch.setattr(
|
||||
fi.FlashInferMetadataBuilder,
|
||||
"_get_flashinfer_trtllm_api_decode_kernel",
|
||||
staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
builder_cls = fi.FlashInferMetadataBuilder
|
||||
assert builder_cls.get_cudagraph_support(config, spec) == (
|
||||
AttentionCGSupport.UNIFORM_BATCH
|
||||
if dcp_size == 1
|
||||
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec)
|
||||
assert bound == expected_bound
|
||||
if bound is not None:
|
||||
# A wrapper forcing NEVER inherits the override but not the bound.
|
||||
chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16)
|
||||
wrapped_cls = chunked_local.get_builder_cls()
|
||||
assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_v2_model_runner,parallel_drafting,expected",
|
||||
[
|
||||
(True, None, 1),
|
||||
(True, False, 8),
|
||||
(True, True, 8),
|
||||
(False, False, 8),
|
||||
(False, True, 15),
|
||||
],
|
||||
)
|
||||
def test_spec_as_decode_width_follows_model_runner(
|
||||
use_v2_model_runner, parallel_drafting, expected
|
||||
):
|
||||
"""Only model runner V1's parallel drafter widens decode requests."""
|
||||
|
||||
class Builder(AttentionMetadataBuilder):
|
||||
def __init__(self, vllm_config):
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
|
||||
raise NotImplementedError
|
||||
|
||||
config = SimpleNamespace(
|
||||
speculative_config=None
|
||||
if parallel_drafting is None
|
||||
else SimpleNamespace(
|
||||
num_speculative_tokens=7, parallel_drafting=parallel_drafting
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
use_v2_model_runner=use_v2_model_runner,
|
||||
)
|
||||
assert max_decode_query_len(config) == expected
|
||||
builder = Builder(config)
|
||||
builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
||||
assert builder.reorder_batch_threshold == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
|
||||
or not current_platform.is_device_capability_family(100),
|
||||
reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads",
|
||||
[
|
||||
pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"),
|
||||
pytest.param(
|
||||
256,
|
||||
1024,
|
||||
40,
|
||||
32,
|
||||
16,
|
||||
8,
|
||||
id="gemma4-sliding-gqa",
|
||||
),
|
||||
pytest.param(
|
||||
512,
|
||||
None,
|
||||
40,
|
||||
32,
|
||||
16,
|
||||
1,
|
||||
id="gemma4-global-mqa",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_varlen_decode_graph_replays_changed_layout(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
head_size,
|
||||
sliding_window,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
):
|
||||
"""Replay changes device offsets, active slots, context, and KV writes.
|
||||
|
||||
The graph is captured with evenly spread dummy queries and an explicit
|
||||
bound of eight. CPU offsets then remain stale while device offsets change.
|
||||
Compare real output rows with independent FP32 causal/windowed attention,
|
||||
and verify cache writes including preservation of the null padding block.
|
||||
"""
|
||||
from vllm.v1.attention.backends import flashinfer as fi
|
||||
from vllm.v1.attention.backends.utils import PerLayerParameters
|
||||
|
||||
set_random_seed(0)
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
block_size = 16
|
||||
dtype = torch.bfloat16
|
||||
LlamaConfig(
|
||||
architectures=["LlamaForCausalLM"],
|
||||
hidden_size=num_heads * head_size,
|
||||
head_dim=head_size,
|
||||
num_attention_heads=num_heads,
|
||||
num_key_value_heads=num_kv_heads,
|
||||
num_hidden_layers=1,
|
||||
max_position_embeddings=4096,
|
||||
).save_pretrained(tmp_path)
|
||||
num_blocks = 1 + num_reqs * cdiv(2112, block_size)
|
||||
config = create_vllm_config(
|
||||
model_name=str(tmp_path),
|
||||
max_model_len=4096,
|
||||
dtype=dtype,
|
||||
num_gpu_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
max_num_seqs=32,
|
||||
max_num_batched_tokens=256,
|
||||
)
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
config.speculative_config = SimpleNamespace(
|
||||
num_speculative_tokens=7,
|
||||
enable_adaptive_verification=True,
|
||||
parallel_drafting=False,
|
||||
)
|
||||
spec = create_standard_kv_cache_spec(config)
|
||||
scale = head_size**-0.5
|
||||
monkeypatch.setattr(
|
||||
fi,
|
||||
"get_per_layer_parameters",
|
||||
lambda *_: {
|
||||
"layer": PerLayerParameters(
|
||||
window_left=-1 if sliding_window is None else sliding_window - 1,
|
||||
logits_soft_cap=0.0,
|
||||
sm_scale=scale,
|
||||
has_sinks=False,
|
||||
),
|
||||
},
|
||||
)
|
||||
device = torch.device("cuda")
|
||||
layout = KVCacheLayout.BLHNC
|
||||
query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device)
|
||||
key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device)
|
||||
value = torch.empty_like(key)
|
||||
output = torch.empty_like(query)
|
||||
dummy_lens = [
|
||||
num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs)
|
||||
]
|
||||
common = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens),
|
||||
block_size,
|
||||
device,
|
||||
)
|
||||
common.max_query_len = 8
|
||||
cache = None
|
||||
|
||||
def prepare(query_lens):
|
||||
nonlocal cache
|
||||
query_lens = query_lens + [0] * (num_reqs - len(query_lens))
|
||||
query.normal_()
|
||||
key.normal_()
|
||||
value.normal_()
|
||||
# Alternating short prefills, nonempty decode contexts, and contexts
|
||||
# beyond the sliding window; inactive requests have zero sequence length.
|
||||
contexts = [
|
||||
([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens)
|
||||
]
|
||||
ks = [
|
||||
torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device)
|
||||
for c in contexts
|
||||
]
|
||||
vs = [torch.randn_like(k) for k in ks]
|
||||
actual = create_common_attn_metadata(
|
||||
BatchSpec(
|
||||
seq_lens=[c + q for c, q in zip(contexts, query_lens)],
|
||||
query_lens=query_lens,
|
||||
),
|
||||
block_size,
|
||||
device,
|
||||
)
|
||||
fresh_cache = create_and_prepopulate_kv_cache(
|
||||
ks,
|
||||
vs,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype,
|
||||
device,
|
||||
num_blocks,
|
||||
actual,
|
||||
layout,
|
||||
randomize_blocks=True,
|
||||
)
|
||||
if cache is None:
|
||||
cache = fresh_cache
|
||||
else:
|
||||
cache.copy_(fresh_cache)
|
||||
common.query_start_loc.copy_(actual.query_start_loc)
|
||||
common.seq_lens.copy_(actual.seq_lens)
|
||||
common.block_table_tensor.zero_()
|
||||
common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_(
|
||||
actual.block_table_tensor
|
||||
)
|
||||
common.slot_mapping.fill_(-1)
|
||||
common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping)
|
||||
return contexts, ks, vs
|
||||
|
||||
prepare(dummy_lens)
|
||||
with set_current_vllm_config(config):
|
||||
builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device)
|
||||
metadata = builder.build(0, common)
|
||||
assert metadata.num_prefills == 0
|
||||
assert metadata.decode.q_len_per_req == 8
|
||||
impl = fi.FlashInferImpl(
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
scale=scale,
|
||||
alibi_slopes=None,
|
||||
sliding_window=sliding_window,
|
||||
kv_cache_dtype="auto",
|
||||
attn_type=AttentionType.DECODER,
|
||||
)
|
||||
layer = MockAttentionLayer(device)
|
||||
|
||||
def forward(attn_metadata=metadata):
|
||||
impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
|
||||
impl.forward(layer, query, key, value, cache, attn_metadata, output=output)
|
||||
|
||||
for _ in range(3):
|
||||
forward()
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
forward()
|
||||
|
||||
layouts = [[8], [1] * num_reqs]
|
||||
if num_tokens >= 16:
|
||||
layouts.append([8, 4, 3, 1])
|
||||
if num_tokens >= 40:
|
||||
layouts.append([8] * 5)
|
||||
|
||||
def check(query_lens, run):
|
||||
contexts, ks, vs = prepare(query_lens)
|
||||
assert cache is not None
|
||||
output.fill_(float("nan"))
|
||||
run()
|
||||
start = 0
|
||||
for i, q_len in enumerate(query_lens):
|
||||
if q_len == 0:
|
||||
continue
|
||||
end = start + q_len
|
||||
full_k = (
|
||||
torch.cat((ks[i], key[start:end]))
|
||||
.float()
|
||||
.repeat_interleave(num_heads // num_kv_heads, dim=1)
|
||||
)
|
||||
full_v = (
|
||||
torch.cat((vs[i], value[start:end]))
|
||||
.float()
|
||||
.repeat_interleave(num_heads // num_kv_heads, dim=1)
|
||||
)
|
||||
scores = (
|
||||
torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale
|
||||
)
|
||||
q_pos = torch.arange(q_len, device=device) + contexts[i]
|
||||
k_pos = torch.arange(full_k.shape[0], device=device)
|
||||
allowed = k_pos[None, :] <= q_pos[:, None]
|
||||
if sliding_window is not None:
|
||||
allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window
|
||||
probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1)
|
||||
reference = torch.einsum("hqk,khd->qhd", probs, full_v)
|
||||
torch.testing.assert_close(
|
||||
output[start:end].float(),
|
||||
reference,
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
)
|
||||
slots = common.slot_mapping[start:end]
|
||||
written = cache[slots // block_size, :, slots % block_size, :]
|
||||
torch.testing.assert_close(written[..., :head_size], key[start:end])
|
||||
torch.testing.assert_close(written[..., head_size:], value[start:end])
|
||||
start = end
|
||||
assert torch.count_nonzero(cache[0]) == 0
|
||||
|
||||
for query_lens in layouts:
|
||||
check(query_lens, graph.replay)
|
||||
|
||||
if num_tokens == 40:
|
||||
# A longer prefill forces a separate launch in eager/piecewise mode.
|
||||
# Its length must not become the decode kernel's query bound, and
|
||||
# compacted CPU placeholders still need not match the GPU allocation.
|
||||
common.num_reqs = 5
|
||||
common.max_query_len = 24
|
||||
common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40])
|
||||
common.query_start_loc_cpu[6:] = 40
|
||||
|
||||
def mixed_forward():
|
||||
with set_current_vllm_config(config):
|
||||
metadata = builder.build(
|
||||
0,
|
||||
replace(
|
||||
common,
|
||||
query_start_loc=common.query_start_loc[:6],
|
||||
query_start_loc_cpu=common.query_start_loc_cpu[:6],
|
||||
seq_lens=common.seq_lens[:5],
|
||||
seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5],
|
||||
block_table_tensor=common.block_table_tensor[:5],
|
||||
),
|
||||
)
|
||||
assert metadata.num_decodes == 4
|
||||
assert metadata.num_prefills == 1
|
||||
assert metadata.decode.q_len_per_req == 8
|
||||
forward(metadata)
|
||||
|
||||
check([8, 4, 3, 1, 24], mixed_forward)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
|
||||
@@ -294,6 +294,7 @@ def _make_spec_decode_manager(
|
||||
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
|
||||
max_num_seqs: int = 8,
|
||||
use_kda_recoverssm: bool = False,
|
||||
varlen_decode: bool = False,
|
||||
) -> gpu_cudagraph_utils.CudaGraphManager:
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
@@ -316,6 +317,7 @@ def _make_spec_decode_manager(
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
decode_query_len=decode_query_len,
|
||||
varlen_decode=varlen_decode,
|
||||
)
|
||||
manager._graphs_captured = True
|
||||
return manager
|
||||
@@ -373,6 +375,28 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch):
|
||||
assert desc.num_tokens == 16
|
||||
|
||||
|
||||
def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch):
|
||||
"""A varlen decode graph replays any decode batch within its query-length
|
||||
bound. The runner passes no bound for a batch with a prefill, which must not
|
||||
replay one however short the prefill is."""
|
||||
manager = _make_spec_decode_manager(monkeypatch, varlen_decode=True)
|
||||
|
||||
def dispatch(max_query_len):
|
||||
return manager.dispatch(
|
||||
num_reqs=3,
|
||||
num_tokens=6,
|
||||
uniform_token_count=None,
|
||||
num_active_loras=0,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
|
||||
desc = dispatch(_DECODE_QUERY_LEN)
|
||||
assert desc.cg_mode == CUDAGraphMode.FULL
|
||||
assert desc.max_query_len == _DECODE_QUERY_LEN
|
||||
assert dispatch(None).cg_mode == CUDAGraphMode.PIECEWISE
|
||||
assert dispatch(_DECODE_QUERY_LEN + 1).cg_mode == CUDAGraphMode.PIECEWISE
|
||||
|
||||
|
||||
def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch):
|
||||
"""A mixed batch must not fall to eager where only decode graphs are staged.
|
||||
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, AttentionMetadataBuilder
|
||||
from vllm.v1.worker.gpu.async_utils import StepTimingSample
|
||||
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
|
||||
from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_module
|
||||
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
|
||||
AdaptiveVerificationManager,
|
||||
@@ -58,37 +58,90 @@ def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expe
|
||||
)
|
||||
|
||||
|
||||
def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch):
|
||||
class Backend:
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls):
|
||||
return True
|
||||
@pytest.mark.parametrize(
|
||||
"target_support,target_bound,device_offsets,additional,error",
|
||||
[
|
||||
pytest.param(AttentionCGSupport.ALWAYS, None, True, None, None, id="always"),
|
||||
pytest.param(
|
||||
AttentionCGSupport.UNIFORM_BATCH, 8, True, None, None, id="bounded"
|
||||
),
|
||||
pytest.param(
|
||||
AttentionCGSupport.UNIFORM_BATCH,
|
||||
None,
|
||||
True,
|
||||
None,
|
||||
"TargetBackend allows none",
|
||||
id="unsupported",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionCGSupport.UNIFORM_BATCH,
|
||||
7,
|
||||
True,
|
||||
None,
|
||||
"up to 8, but TargetBackend allows at most 7",
|
||||
id="too-narrow",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionCGSupport.UNIFORM_BATCH,
|
||||
8,
|
||||
False,
|
||||
None,
|
||||
"trims verification requests",
|
||||
id="host-query-lens",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionCGSupport.ALWAYS,
|
||||
None,
|
||||
True,
|
||||
(AttentionCGSupport.UNIFORM_BATCH, "EncoderBackend"),
|
||||
"EncoderBackend allows none",
|
||||
id="additional-group",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_manager_checks_target_varlen_cudagraph_bound(
|
||||
monkeypatch, target_support, target_bound, device_offsets, additional, error
|
||||
):
|
||||
"""Target builders must replay varlen decode graphs of the verification
|
||||
width; draft-only groups are not checked."""
|
||||
|
||||
class Builder:
|
||||
def __init__(self, support):
|
||||
self.support = support
|
||||
def group(backend_name, layer_name, support, bound=None, mismatch=True):
|
||||
class Builder(AttentionMetadataBuilder):
|
||||
_cudagraph_support = support
|
||||
|
||||
def get_cudagraph_support(self, *_args):
|
||||
return self.support
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def group(layer_name, support):
|
||||
builder = Builder(support)
|
||||
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
|
||||
raise NotImplementedError
|
||||
|
||||
if bound is not None:
|
||||
|
||||
@classmethod
|
||||
def get_varlen_cudagraph_max_query_len(cls, *_args):
|
||||
return bound
|
||||
|
||||
backend = type(
|
||||
backend_name,
|
||||
(),
|
||||
{"supports_device_cpu_query_lens_mismatch": staticmethod(lambda: mismatch)},
|
||||
)
|
||||
builder = Builder()
|
||||
return SimpleNamespace(
|
||||
layer_names=[layer_name],
|
||||
backend=Backend,
|
||||
backend=backend,
|
||||
kv_cache_spec=None,
|
||||
get_metadata_builder=lambda _index: builder,
|
||||
)
|
||||
|
||||
groups = [
|
||||
[
|
||||
group("target", AttentionCGSupport.ALWAYS),
|
||||
group("draft", AttentionCGSupport.UNIFORM_BATCH),
|
||||
group(
|
||||
"TargetBackend", "target", target_support, target_bound, device_offsets
|
||||
),
|
||||
group("DraftBackend", "draft", AttentionCGSupport.UNIFORM_BATCH),
|
||||
]
|
||||
]
|
||||
runner_support = AttentionCGSupportInfo(
|
||||
AttentionCGSupport.UNIFORM_BATCH, "DraftBackend"
|
||||
)
|
||||
created = object()
|
||||
monkeypatch.setattr(
|
||||
adaptive_module,
|
||||
@@ -96,20 +149,19 @@ def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatc
|
||||
lambda *_args, **_kwargs: created,
|
||||
)
|
||||
|
||||
manager = maybe_create_adaptive_verification_manager(
|
||||
enable_adaptive_verification=True,
|
||||
attn_groups=groups,
|
||||
attn_cg_support=runner_support,
|
||||
req_states=object(),
|
||||
query_start_loc=object(),
|
||||
num_bonus_tokens=1,
|
||||
max_total_logits=1,
|
||||
vllm_config=None,
|
||||
target_layer_names={"target"},
|
||||
)
|
||||
|
||||
assert manager is created
|
||||
assert runner_support.min_cg_support == AttentionCGSupport.UNIFORM_BATCH
|
||||
with pytest.raises(ValueError, match=error) if error else nullcontext():
|
||||
manager = maybe_create_adaptive_verification_manager(
|
||||
enable_adaptive_verification=True,
|
||||
attn_groups=groups,
|
||||
req_states=SimpleNamespace(num_speculative_steps=7),
|
||||
query_start_loc=object(),
|
||||
num_bonus_tokens=1,
|
||||
max_total_logits=1,
|
||||
vllm_config=None,
|
||||
target_layer_names={"target"},
|
||||
additional_attn_cg_support=additional,
|
||||
)
|
||||
assert manager is created
|
||||
|
||||
|
||||
def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
|
||||
|
||||
@@ -8,6 +8,7 @@ never addressed by the logical view.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -99,12 +100,16 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
|
||||
|
||||
class _FakeMetadataBuilder:
|
||||
def __init__(self, support: AttentionCGSupport):
|
||||
def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None):
|
||||
self.support = support
|
||||
self.varlen_bound = varlen_bound
|
||||
|
||||
def get_cudagraph_support(self, *_args):
|
||||
return self.support
|
||||
|
||||
def get_varlen_cudagraph_max_query_len(self, *_args):
|
||||
return self.varlen_bound
|
||||
|
||||
|
||||
class _TargetBackend:
|
||||
@classmethod
|
||||
@@ -183,6 +188,42 @@ def test_attention_checks_preserve_global_and_target_scoped_support():
|
||||
)
|
||||
|
||||
|
||||
def test_varlen_cudagraph_unsupported_backend_checks_scoped_bounds():
|
||||
"""ALWAYS passes without a bound, other builders need one at least as wide as
|
||||
the requested length, and NEVER fails whatever bound a builder reports."""
|
||||
config: Any = SimpleNamespace()
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
def group(
|
||||
backend: Any,
|
||||
layer_name: str,
|
||||
support: AttentionCGSupport,
|
||||
bound: int | None = None,
|
||||
):
|
||||
builder: Any = _FakeMetadataBuilder(support, bound)
|
||||
attn_group = AttentionGroup(backend, [layer_name], spec, 0)
|
||||
attn_group.metadata_builders = [builder]
|
||||
return attn_group
|
||||
|
||||
target = group(_TargetBackend, "target", AttentionCGSupport.ALWAYS)
|
||||
draft = group(_DraftBackend, "draft", AttentionCGSupport.UNIFORM_BATCH, 8)
|
||||
never = group(_DraftBackend, "never", AttentionCGSupport.NEVER, 8)
|
||||
|
||||
unsupported = attn_utils.get_varlen_cudagraph_unsupported_backend
|
||||
assert unsupported([[target, draft]], config, 8) is None
|
||||
assert unsupported([[target, draft]], config, 9) == ("_DraftBackend", 8)
|
||||
assert (
|
||||
unsupported([[target, draft]], config, 9, checked_layer_names={"target"})
|
||||
is None
|
||||
)
|
||||
assert unsupported([[target, never]], config, 1) == ("_DraftBackend", None)
|
||||
|
||||
|
||||
def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Fast prefill applies to the contiguous suffix of KV-sharing layers.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user