[Spec Decode] Allow causal DFlash (#43445)

This commit is contained in:
Benjamin Chislett
2026-05-28 21:18:44 +00:00
committed by GitHub
parent 69b8956dcd
commit 9202ea6fda
+17 -15
View File
@@ -68,6 +68,8 @@ class DFlashProposer(SpecDecodeBaseProposer):
# For DFlash we use the input embeddings to embed the mask token
self.parallel_drafting_hidden_state_tensor = None
self.dflash_causal = self.dflash_config.get("causal", False)
@override
def _create_draft_vllm_config(self) -> VllmConfig:
base = super()._create_draft_vllm_config()
@@ -75,7 +77,7 @@ class DFlashProposer(SpecDecodeBaseProposer):
base,
attention_config=replace(
base.attention_config,
use_non_causal=True,
use_non_causal=not self.dflash_causal,
),
)
@@ -184,7 +186,7 @@ class DFlashProposer(SpecDecodeBaseProposer):
max_seq_len=cad.max_seq_len + num_query_per_req,
block_table_tensor=cad.block_table_tensor,
slot_mapping=query_slot_mapping,
causal=False, # Non-causal attention is required for DFlash
causal=self.dflash_causal,
)
return num_query_total, token_indices_to_sample, new_cad
@@ -281,20 +283,20 @@ class DFlashProposer(SpecDecodeBaseProposer):
per_group, per_layer = super().build_per_group_and_layer_attn_metadata(
cad, draft_index
)
for layer_name, attn_metadata in per_layer.items():
assert getattr(attn_metadata, "causal", None) is False, (
f"Attention metadata for layer {layer_name} does not have"
" non-causal support, which is required for DFlash."
" Consider using a different attention backend, such as FlashAttention."
)
if not self.dflash_causal:
# Require all layers to support non-causal attention when required by DFlash
for layer_name, attn_metadata in per_layer.items():
assert getattr(attn_metadata, "causal", None) is False, (
f"Attention metadata for layer {layer_name} does not have"
" non-causal support, which is required for DFlash."
" Consider using a different attention backend, e.g FlashAttention."
)
return per_group, per_layer
@override
def _get_eagle3_use_aux_hidden_state_from_config(self):
use_aux_hidden_state = True
dflash_config = getattr(
self.draft_model_config.hf_config, "dflash_config", None
)
if dflash_config is not None:
use_aux_hidden_state = dflash_config.get("use_aux_hidden_state", True)
return use_aux_hidden_state
return self.dflash_config.get("use_aux_hidden_state", True)
@property
def dflash_config(self):
return getattr(self.draft_model_config.hf_config, "dflash_config", None) or {}