[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5.5
Nick Hill
parent
3b4566c5cf
commit
a4eb3f25d6
@@ -21,6 +21,7 @@ from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.attention.backend import max_decode_query_len
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
|
||||
|
||||
@@ -46,20 +47,8 @@ def is_hisparse_decode_batch(attn_metadata: Any | None) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _get_max_decode_query_len(vllm_config: VllmConfig) -> int:
|
||||
speculative_config = getattr(vllm_config, "speculative_config", None)
|
||||
if (
|
||||
speculative_config is not None
|
||||
and speculative_config.num_speculative_tokens is not None
|
||||
):
|
||||
return 1 + speculative_config.num_speculative_tokens * (
|
||||
2 if speculative_config.parallel_drafting else 1
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
def _get_max_swap_rows(vllm_config: VllmConfig) -> int:
|
||||
max_query_len = _get_max_decode_query_len(vllm_config)
|
||||
max_query_len = max_decode_query_len(vllm_config)
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
return min(
|
||||
scheduler_config.max_num_batched_tokens,
|
||||
@@ -120,7 +109,7 @@ class ResolvedHiSparseConfig:
|
||||
if config is None:
|
||||
return None
|
||||
|
||||
max_query_len = _get_max_decode_query_len(vllm_config)
|
||||
max_query_len = max_decode_query_len(vllm_config)
|
||||
min_device_buffer_size = max_query_len * model_top_k
|
||||
# Retain the worst-case union of every speculative step's top-k and
|
||||
# leave one additional top-k of LRU slack.
|
||||
|
||||
Reference in New Issue
Block a user