[Spec Decode] Allow causal DFlash (#43445)
This commit is contained in:
@@ -68,6 +68,8 @@ class DFlashProposer(SpecDecodeBaseProposer):
|
||||
# For DFlash we use the input embeddings to embed the mask token
|
||||
self.parallel_drafting_hidden_state_tensor = None
|
||||
|
||||
self.dflash_causal = self.dflash_config.get("causal", False)
|
||||
|
||||
@override
|
||||
def _create_draft_vllm_config(self) -> VllmConfig:
|
||||
base = super()._create_draft_vllm_config()
|
||||
@@ -75,7 +77,7 @@ class DFlashProposer(SpecDecodeBaseProposer):
|
||||
base,
|
||||
attention_config=replace(
|
||||
base.attention_config,
|
||||
use_non_causal=True,
|
||||
use_non_causal=not self.dflash_causal,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -184,7 +186,7 @@ class DFlashProposer(SpecDecodeBaseProposer):
|
||||
max_seq_len=cad.max_seq_len + num_query_per_req,
|
||||
block_table_tensor=cad.block_table_tensor,
|
||||
slot_mapping=query_slot_mapping,
|
||||
causal=False, # Non-causal attention is required for DFlash
|
||||
causal=self.dflash_causal,
|
||||
)
|
||||
|
||||
return num_query_total, token_indices_to_sample, new_cad
|
||||
@@ -281,20 +283,20 @@ class DFlashProposer(SpecDecodeBaseProposer):
|
||||
per_group, per_layer = super().build_per_group_and_layer_attn_metadata(
|
||||
cad, draft_index
|
||||
)
|
||||
for layer_name, attn_metadata in per_layer.items():
|
||||
assert getattr(attn_metadata, "causal", None) is False, (
|
||||
f"Attention metadata for layer {layer_name} does not have"
|
||||
" non-causal support, which is required for DFlash."
|
||||
" Consider using a different attention backend, such as FlashAttention."
|
||||
)
|
||||
if not self.dflash_causal:
|
||||
# Require all layers to support non-causal attention when required by DFlash
|
||||
for layer_name, attn_metadata in per_layer.items():
|
||||
assert getattr(attn_metadata, "causal", None) is False, (
|
||||
f"Attention metadata for layer {layer_name} does not have"
|
||||
" non-causal support, which is required for DFlash."
|
||||
" Consider using a different attention backend, e.g FlashAttention."
|
||||
)
|
||||
return per_group, per_layer
|
||||
|
||||
@override
|
||||
def _get_eagle3_use_aux_hidden_state_from_config(self):
|
||||
use_aux_hidden_state = True
|
||||
dflash_config = getattr(
|
||||
self.draft_model_config.hf_config, "dflash_config", None
|
||||
)
|
||||
if dflash_config is not None:
|
||||
use_aux_hidden_state = dflash_config.get("use_aux_hidden_state", True)
|
||||
return use_aux_hidden_state
|
||||
return self.dflash_config.get("use_aux_hidden_state", True)
|
||||
|
||||
@property
|
||||
def dflash_config(self):
|
||||
return getattr(self.draft_model_config.hf_config, "dflash_config", None) or {}
|
||||
|
||||
Reference in New Issue
Block a user