[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5.5
Nick Hill
parent
3b4566c5cf
commit
a4eb3f25d6
@@ -19,8 +19,8 @@ from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.utils import CpuGpuBuffer
|
||||
from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
get_attn_cg_support,
|
||||
get_query_lens_mismatch_unsupported_backend,
|
||||
get_varlen_cudagraph_unsupported_backend,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -28,7 +28,6 @@ _PROFILE_REPLAYS = 5
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
@@ -452,7 +451,6 @@ def maybe_create_adaptive_verification_manager(
|
||||
*,
|
||||
enable_adaptive_verification: bool,
|
||||
attn_groups: list[list["AttentionGroup"]],
|
||||
attn_cg_support: "AttentionCGSupportInfo",
|
||||
req_states: "RequestState",
|
||||
query_start_loc: torch.Tensor,
|
||||
num_bonus_tokens: int,
|
||||
@@ -477,25 +475,30 @@ def maybe_create_adaptive_verification_manager(
|
||||
"use a backend that does."
|
||||
)
|
||||
|
||||
target_attn_cg_support = attn_cg_support
|
||||
if target_layer_names is not None:
|
||||
target_attn_cg_support = get_attn_cg_support(
|
||||
# The runner's decode_query_len, the width varlen decode graphs capture.
|
||||
max_query_len = req_states.num_speculative_steps + num_bonus_tokens
|
||||
unsupported: tuple[str | None, int | None] | None = (
|
||||
get_varlen_cudagraph_unsupported_backend(
|
||||
attn_groups,
|
||||
vllm_config,
|
||||
max_query_len,
|
||||
checked_layer_names=target_layer_names,
|
||||
)
|
||||
if additional_attn_cg_support is not None:
|
||||
target_attn_cg_support = target_attn_cg_support.narrow(
|
||||
*additional_attn_cg_support
|
||||
)
|
||||
if target_attn_cg_support.min_cg_support != AttentionCGSupport.ALWAYS:
|
||||
)
|
||||
if unsupported is None and additional_attn_cg_support is not None:
|
||||
# Groups built outside init_attn_backend report only their support
|
||||
# level, and without a bound only ALWAYS replays varlen batches.
|
||||
additional_support, additional_backend = additional_attn_cg_support
|
||||
if additional_support != AttentionCGSupport.ALWAYS:
|
||||
unsupported = additional_backend, None
|
||||
if unsupported is not None:
|
||||
backend, bound = unsupported
|
||||
allowed = "none" if bound is None else f"at most {bound}"
|
||||
raise ValueError(
|
||||
"Adaptive verification captures varlen decode cudagraphs, so every"
|
||||
" target attention builder must report AttentionCGSupport.ALWAYS, but "
|
||||
f"{target_attn_cg_support.min_cg_attn_backend} reports "
|
||||
f"{target_attn_cg_support.min_cg_support}. Pass "
|
||||
"enable_adaptive_verification=false in the speculative config, or "
|
||||
"use a backend that does."
|
||||
"Adaptive verification replays decode cudagraphs whose per-request "
|
||||
f"query lengths vary up to {max_query_len}, but {backend} allows "
|
||||
f"{allowed}. Pass enable_adaptive_verification=false in the "
|
||||
"speculative config, or use a backend that does."
|
||||
)
|
||||
|
||||
return AdaptiveVerificationManager(
|
||||
|
||||
Reference in New Issue
Block a user