[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-09-26 17:01:17 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5.5 Nick Hill
parent 3b4566c5cf
commit a4eb3f25d6
24 changed files with 894 additions and 135 deletions
+3 -14
View File
@@ -21,6 +21,7 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import current_stream
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
@@ -46,20 +47,8 @@ def is_hisparse_decode_batch(attn_metadata: Any | None) -> bool:
)
def _get_max_decode_query_len(vllm_config: VllmConfig) -> int:
speculative_config = getattr(vllm_config, "speculative_config", None)
if (
speculative_config is not None
and speculative_config.num_speculative_tokens is not None
):
return 1 + speculative_config.num_speculative_tokens * (
2 if speculative_config.parallel_drafting else 1
)
return 1
def _get_max_swap_rows(vllm_config: VllmConfig) -> int:
max_query_len = _get_max_decode_query_len(vllm_config)
max_query_len = max_decode_query_len(vllm_config)
scheduler_config = vllm_config.scheduler_config
return min(
scheduler_config.max_num_batched_tokens,
@@ -120,7 +109,7 @@ class ResolvedHiSparseConfig:
if config is None:
return None
max_query_len = _get_max_decode_query_len(vllm_config)
max_query_len = max_decode_query_len(vllm_config)
min_device_buffer_size = max_query_len * model_top_k
# Retain the worst-case union of every speculative step's top-k and
# leave one additional top-k of LRU slack.