[Spec Decode] Enable adaptive verification on DSv4 + sm90 (#52795)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
df3b3422b4
commit
5df31ea52d
@@ -157,8 +157,11 @@ class DeepseekV32IndexerBackend(AttentionBackend):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
||||||
# Only the varlen paged MQA logits kernel takes per-request query
|
# Only the varlen paged MQA logits kernel takes per-request query
|
||||||
# lengths from device tensors; otherwise the indexer needs uniform ones.
|
# lengths from device tensors natively. Hopper can instead flatten each
|
||||||
return _supports_varlen_paged_mqa_logits()
|
# query into a single-token row using device-built metadata.
|
||||||
|
return _supports_varlen_paged_mqa_logits() or (
|
||||||
|
_supports_flattened_device_query_lens()
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_name() -> str:
|
def get_name() -> str:
|
||||||
@@ -493,6 +496,14 @@ def _supports_varlen_paged_mqa_logits() -> bool:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _supports_flattened_device_query_lens() -> bool:
|
||||||
|
return (
|
||||||
|
current_platform.is_cuda()
|
||||||
|
and current_platform.is_device_capability_family(90)
|
||||||
|
and has_deep_gemm()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _supports_native_decode(next_n: int) -> bool:
|
def _supports_native_decode(next_n: int) -> bool:
|
||||||
"""Whether decode can pass `next_n` Q rows per request to the kernel
|
"""Whether decode can pass `next_n` Q rows per request to the kernel
|
||||||
instead of flattening to one single-token row per query, which re-reads
|
instead of flattening to one single-token row per query, which re-reads
|
||||||
@@ -507,6 +518,16 @@ def _supports_native_decode(next_n: int) -> bool:
|
|||||||
return next_n in (1, 2)
|
return next_n in (1, 2)
|
||||||
|
|
||||||
|
|
||||||
|
def _use_flattening(vllm_config: VllmConfig) -> bool:
|
||||||
|
speculative_config = vllm_config.speculative_config
|
||||||
|
next_n = 1 + vllm_config.num_speculative_tokens
|
||||||
|
return not _supports_native_decode(next_n) or (
|
||||||
|
speculative_config is not None
|
||||||
|
and speculative_config.enable_adaptive_verification
|
||||||
|
and _supports_flattened_device_query_lens()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||||
# The indexer opts out of the shared reorder-threshold vote (see __init__),
|
# The indexer opts out of the shared reorder-threshold vote (see __init__),
|
||||||
# so this is None; its own split uses self.decode_threshold.
|
# so this is None; its own split uses self.decode_threshold.
|
||||||
@@ -519,7 +540,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
kv_cache_spec: KVCacheSpec,
|
kv_cache_spec: KVCacheSpec,
|
||||||
) -> AttentionCGSupport:
|
) -> AttentionCGSupport:
|
||||||
if _supports_varlen_paged_mqa_logits():
|
if _supports_varlen_paged_mqa_logits() or _use_flattening(vllm_config):
|
||||||
return AttentionCGSupport.ALWAYS
|
return AttentionCGSupport.ALWAYS
|
||||||
return AttentionCGSupport.UNIFORM_BATCH
|
return AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
@@ -553,7 +574,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
next_n = self.num_speculative_tokens + 1
|
next_n = self.num_speculative_tokens + 1
|
||||||
self.decode_threshold = next_n
|
self.decode_threshold = next_n
|
||||||
self.reorder_batch_threshold = None
|
self.reorder_batch_threshold = None
|
||||||
self.use_flattening = not _supports_native_decode(next_n)
|
self.use_flattening = _use_flattening(self.vllm_config)
|
||||||
self.supports_varlen = _supports_varlen_paged_mqa_logits()
|
self.supports_varlen = _supports_varlen_paged_mqa_logits()
|
||||||
logger.info_once(
|
logger.info_once(
|
||||||
"DSA indexer decode path: use_flattening=%s supports_varlen=%s "
|
"DSA indexer decode path: use_flattening=%s supports_varlen=%s "
|
||||||
@@ -674,11 +695,14 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
max_decode_len: int,
|
max_decode_len: int,
|
||||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, bool]:
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, bool]:
|
||||||
"""Prepare native or per-token flattened decode tensors."""
|
"""Prepare native or per-token flattened decode tensors."""
|
||||||
|
spec_config = self.vllm_config.speculative_config
|
||||||
|
adaptive = bool(spec_config and spec_config.enable_adaptive_verification)
|
||||||
min_decode_len = int(decode_lens_cpu.min().item())
|
min_decode_len = int(decode_lens_cpu.min().item())
|
||||||
if not use_native:
|
if not use_native:
|
||||||
assert self.decode_seq_lens_buffer.dim() == 1
|
assert self.decode_seq_lens_buffer.dim() == 1
|
||||||
if (
|
if (
|
||||||
not self.supports_varlen
|
not self.supports_varlen
|
||||||
|
and (num_decodes == 1 or not adaptive)
|
||||||
and min_decode_len == max_decode_len
|
and min_decode_len == max_decode_len
|
||||||
and num_decodes * max_decode_len == num_decode_tokens
|
and num_decodes * max_decode_len == num_decode_tokens
|
||||||
):
|
):
|
||||||
|
|||||||
Reference in New Issue
Block a user