[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -32,8 +32,8 @@ from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
_C128A_TOPK_ALIGNMENT = 128
|
||||
|
||||
|
||||
class DeepseekV4FlashMLABackend(AttentionBackend):
|
||||
"""DeepSeek-V4 sparse-MLA backend.
|
||||
class DeepseekV4SparseMLABackend(AttentionBackend):
|
||||
"""DeepSeek-V4 sparse-MLA backend base.
|
||||
|
||||
Subclasses ``AttentionBackend`` directly (not the V3.2
|
||||
``FlashMLASparseBackend``): DeepSeek-V4 runs its own attention layer
|
||||
@@ -54,12 +54,8 @@ class DeepseekV4FlashMLABackend(AttentionBackend):
|
||||
return [256]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHMLA_SPARSE_DSV4"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["DeepseekV4FlashMLAMetadataBuilder"]:
|
||||
return DeepseekV4FlashMLAMetadataBuilder
|
||||
def get_builder_cls() -> type["DeepseekV4SparseMLAMetadataBuilder"]:
|
||||
return DeepseekV4SparseMLAMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type[Any]:
|
||||
@@ -67,7 +63,7 @@ class DeepseekV4FlashMLABackend(AttentionBackend):
|
||||
# not the generic ``Attention``/``MLAAttention`` layer, so the backend's
|
||||
# impl class is never instantiated.
|
||||
raise NotImplementedError(
|
||||
"DeepseekV4FlashMLABackend has no separate impl class; DeepSeek-V4 "
|
||||
"DeepseekV4SparseMLABackend has no separate impl class; DeepSeek-V4 "
|
||||
"attention runs through DeepseekV4Attention."
|
||||
)
|
||||
|
||||
@@ -131,7 +127,7 @@ class DeepseekV4FlashMLAMetadata(AttentionMetadata):
|
||||
c128a_prefill_topk_indices: torch.Tensor | None = None
|
||||
|
||||
|
||||
class DeepseekV4FlashMLAMetadataBuilder(
|
||||
class DeepseekV4SparseMLAMetadataBuilder(
|
||||
AttentionMetadataBuilder[DeepseekV4FlashMLAMetadata]
|
||||
):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
@@ -294,6 +290,20 @@ class DeepseekV4FlashMLAMetadataBuilder(
|
||||
return result
|
||||
|
||||
|
||||
class DeepseekV4FlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
|
||||
|
||||
class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHMLA_SPARSE_DSV4"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]:
|
||||
return DeepseekV4FlashMLAMetadataBuilder
|
||||
|
||||
|
||||
def build_c128a_topk_metadata(
|
||||
positions: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
|
||||
Reference in New Issue
Block a user