[Bugfix][Spec Decode] Use group geometry for FlashAttention metadata (#53002)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Misha Goin
2026-08-21 17:07:19 -07:00
committed by GitHub
parent a34f2ab70a
commit 9ff7041b55
4 changed files with 70 additions and 18 deletions
+53 -6
View File
@@ -1,10 +1,6 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler metadata sizes a scratchpad from the query head count, so it must """Attention metadata geometry must come from the builder's own group."""
come from the builder's own group: the model-wide ``get_num_attention_heads()``
is wrong for models that vary it per layer (e.g. Laguna), and too small a
scratchpad is indexed past its end.
"""
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
@@ -17,8 +13,9 @@ from vllm.v1.attention.backends.cpu_attn import (
CPUAttentionBackendImpl, CPUAttentionBackendImpl,
CPUAttentionMetadataBuilder, CPUAttentionMetadataBuilder,
) )
from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadataBuilder
pytestmark = pytest.mark.skipif( requires_cpu = pytest.mark.skipif(
not current_platform.is_cpu(), reason="CPU attention backend" not current_platform.is_cpu(), reason="CPU attention backend"
) )
@@ -69,6 +66,7 @@ def _build(layer_num_heads: list[int]) -> CPUAttentionMetadataBuilder:
) )
@requires_cpu
@pytest.mark.parametrize("group_num_heads", [MODEL_WIDE_NUM_HEADS, 64, 16]) @pytest.mark.parametrize("group_num_heads", [MODEL_WIDE_NUM_HEADS, 64, 16])
def test_num_heads_comes_from_the_group(group_num_heads): def test_num_heads_comes_from_the_group(group_num_heads):
"""The group's own count wins, even when it is not the model-wide one.""" """The group's own count wins, even when it is not the model-wide one."""
@@ -76,7 +74,56 @@ def test_num_heads_comes_from_the_group(group_num_heads):
assert builder.num_heads == group_num_heads assert builder.num_heads == group_num_heads
@requires_cpu
def test_mixed_head_counts_in_one_group_are_rejected(): def test_mixed_head_counts_in_one_group_are_rejected():
"""Grouping guarantees uniformity; a mixed group means that broke.""" """Grouping guarantees uniformity; a mixed group means that broke."""
with pytest.raises(AssertionError, match="share num_heads"): with pytest.raises(AssertionError, match="share num_heads"):
_build([MODEL_WIDE_NUM_HEADS, 64]) _build([MODEL_WIDE_NUM_HEADS, 64])
def test_flash_attention_geometry_comes_from_the_group():
"""FA3 must use group geometry for layers without an ``impl`` wrapper."""
layers = {f"layer_{i}": SimpleNamespace(num_heads=16) for i in range(2)}
vllm_config = MagicMock()
vllm_config.model_config.get_num_attention_heads.return_value = MODEL_WIDE_NUM_HEADS
vllm_config.model_config.get_num_kv_heads.return_value = NUM_KV_HEADS
vllm_config.model_config.get_head_size.return_value = 128
vllm_config.model_config.rswa_window = None
vllm_config.model_config.is_mm_prefix_lm = False
vllm_config.parallel_config.cp_kv_cache_interleave_size = 1
vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs.return_value = (
False
)
vllm_config.compilation_config.max_cudagraph_capture_size = None
kv_cache_spec = SimpleNamespace(
block_size=16,
num_kv_heads=2,
head_size=64,
dtype=torch.bfloat16,
)
with (
patch(
"vllm.v1.attention.backends.utils.get_layers_from_vllm_config",
return_value=layers,
),
patch(
"vllm.distributed.parallel_state.get_dcp_group",
side_effect=AssertionError,
),
patch(
"vllm.v1.attention.backends.flash_attn.get_flash_attn_version",
return_value=3,
),
):
builder = FlashAttentionMetadataBuilder(
kv_cache_spec=kv_cache_spec,
layer_names=list(layers),
vllm_config=vllm_config,
device=torch.device("cpu"),
)
assert builder.aot_schedule
assert builder.num_heads_q == 16
assert builder.num_heads_kv == 2
assert builder.headdim == 64
+6 -5
View File
@@ -33,6 +33,7 @@ from vllm.v1.attention.backends.fa_utils import (
from vllm.v1.attention.backends.utils import ( from vllm.v1.attention.backends.utils import (
fill_mm_prefix_query_ranges, fill_mm_prefix_query_ranges,
get_dcp_local_seq_lens, get_dcp_local_seq_lens,
get_num_attention_heads_from_layers,
) )
from vllm.v1.attention.ops.dcp import ( from vllm.v1.attention.ops.dcp import (
cp_lse_ag_out_rs, cp_lse_ag_out_rs,
@@ -435,12 +436,12 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
self.compilation_config = vllm_config.compilation_config self.compilation_config = vllm_config.compilation_config
self.attention_config = vllm_config.attention_config self.attention_config = vllm_config.attention_config
self.num_heads_q = self.model_config.get_num_attention_heads( self.num_heads_q = get_num_attention_heads_from_layers(
self.parallel_config vllm_config, layer_names
) ) or self.model_config.get_num_attention_heads(self.parallel_config)
self.num_heads_kv = self.model_config.get_num_kv_heads(self.parallel_config) self.num_heads_kv = kv_cache_spec.num_kv_heads
self.kv_cache_dtype = kv_cache_spec.dtype self.kv_cache_dtype = kv_cache_spec.dtype
self.headdim = self.model_config.get_head_size() self.headdim = kv_cache_spec.head_size
self.block_size = kv_cache_spec.block_size self.block_size = kv_cache_spec.block_size
self.max_num_splits = 0 # No upper bound on the number of splits. self.max_num_splits = 0 # No upper bound on the number of splits.
+5 -1
View File
@@ -9,6 +9,7 @@ from typing import (
Any, Any,
Literal, Literal,
Protocol, Protocol,
cast,
get_args, get_args,
) )
@@ -263,7 +264,10 @@ def get_num_attention_heads_from_layers(
) )
if not attn_layers: if not attn_layers:
return None return None
heads = {layer.impl.num_heads for layer in attn_layers.values()} heads = {
cast(Any, getattr(layer, "impl", layer)).num_heads
for layer in attn_layers.values()
}
assert len(heads) == 1, ( assert len(heads) == 1, (
f"All layers in one attention group must share num_heads; " f"All layers in one attention group must share num_heads; "
f"got {heads} for {layer_names}." f"got {heads} for {layer_names}."
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from collections.abc import Mapping from collections.abc import Mapping
from typing import Any from typing import Any
@@ -103,13 +104,12 @@ class DFlashSpeculator(DraftModelSpeculator):
@property @property
def attn_vllm_config(self) -> VllmConfig: def attn_vllm_config(self) -> VllmConfig:
# The draft's attention differs from the target's in causality. # The draft's attention differs from the target's in causality.
return replace( config = copy.copy(super().attn_vllm_config)
self.vllm_config, config.attention_config = replace(
attention_config=replace( self.vllm_config.attention_config,
self.vllm_config.attention_config, use_non_causal=self.requires_non_causal,
use_non_causal=self.requires_non_causal,
),
) )
return config
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
wants_full = cudagraph_mode.decode_mode() == CUDAGraphMode.FULL wants_full = cudagraph_mode.decode_mode() == CUDAGraphMode.FULL