[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-09-26 17:01:17 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5.5 Nick Hill
parent 3b4566c5cf
commit a4eb3f25d6
24 changed files with 894 additions and 135 deletions
@@ -19,8 +19,8 @@ from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.utils import CpuGpuBuffer
from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream
from vllm.v1.worker.gpu.attn_utils import (
get_attn_cg_support,
get_query_lens_mismatch_unsupported_backend,
get_varlen_cudagraph_unsupported_backend,
)
logger = init_logger(__name__)
@@ -28,7 +28,6 @@ _PROFILE_REPLAYS = 5
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.utils import AttentionGroup
@@ -452,7 +451,6 @@ def maybe_create_adaptive_verification_manager(
*,
enable_adaptive_verification: bool,
attn_groups: list[list["AttentionGroup"]],
attn_cg_support: "AttentionCGSupportInfo",
req_states: "RequestState",
query_start_loc: torch.Tensor,
num_bonus_tokens: int,
@@ -477,25 +475,30 @@ def maybe_create_adaptive_verification_manager(
"use a backend that does."
)
target_attn_cg_support = attn_cg_support
if target_layer_names is not None:
target_attn_cg_support = get_attn_cg_support(
# The runner's decode_query_len, the width varlen decode graphs capture.
max_query_len = req_states.num_speculative_steps + num_bonus_tokens
unsupported: tuple[str | None, int | None] | None = (
get_varlen_cudagraph_unsupported_backend(
attn_groups,
vllm_config,
max_query_len,
checked_layer_names=target_layer_names,
)
if additional_attn_cg_support is not None:
target_attn_cg_support = target_attn_cg_support.narrow(
*additional_attn_cg_support
)
if target_attn_cg_support.min_cg_support != AttentionCGSupport.ALWAYS:
)
if unsupported is None and additional_attn_cg_support is not None:
# Groups built outside init_attn_backend report only their support
# level, and without a bound only ALWAYS replays varlen batches.
additional_support, additional_backend = additional_attn_cg_support
if additional_support != AttentionCGSupport.ALWAYS:
unsupported = additional_backend, None
if unsupported is not None:
backend, bound = unsupported
allowed = "none" if bound is None else f"at most {bound}"
raise ValueError(
"Adaptive verification captures varlen decode cudagraphs, so every"
" target attention builder must report AttentionCGSupport.ALWAYS, but "
f"{target_attn_cg_support.min_cg_attn_backend} reports "
f"{target_attn_cg_support.min_cg_support}. Pass "
"enable_adaptive_verification=false in the speculative config, or "
"use a backend that does."
"Adaptive verification replays decode cudagraphs whose per-request "
f"query lengths vary up to {max_query_len}, but {backend} allows "
f"{allowed}. Pass enable_adaptive_verification=false in the "
"speculative config, or use a backend that does."
)
return AdaptiveVerificationManager(