[Bugfix] Restore part of bugfix #42650 after accidental deletion in #43241 (#47039)

Signed-off-by: zhanda <zhandazhu@gmail.com>
Signed-off-by: Nikita Shapovalov <nikita@poolside.ai>
Co-authored-by: Zhanda Zhu <49645678+zhandaz@users.noreply.github.com>
Co-authored-by: Shang Wang <shangw@nvidia.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Nikita Shapovalov
2026-06-30 11:07:59 -07:00
committed by GitHub
co-authored by Zhanda Zhu Shang Wang Michael Goin
parent 727971f1c1
commit a7732537f4
3 changed files with 36 additions and 6 deletions
+5 -3
View File
@@ -62,6 +62,7 @@ from vllm.v1.attention.backends.utils import (
KVCacheLayoutType,
get_dcp_local_seq_lens,
get_kv_cache_layout,
get_num_attention_heads_from_layers,
get_per_layer_parameters,
infer_global_hyperparameters,
split_decodes_and_prefills,
@@ -635,9 +636,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self.use_dcp and vllm_config.parallel_config.dcp_comm_backend == "a2a"
)
self.num_qo_heads = self.model_config.get_num_attention_heads(
self.vllm_config.parallel_config
)
# Compatible with models with non-uniform per-layer head counts.
self.num_qo_heads = get_num_attention_heads_from_layers(
vllm_config, layer_names
) or self.model_config.get_num_attention_heads(self.vllm_config.parallel_config)
self.num_kv_heads = self.kv_cache_spec.num_kv_heads
self.head_dim = self.kv_cache_spec.head_size
+5 -3
View File
@@ -33,6 +33,7 @@ from vllm.v1.attention.backend import (
from vllm.v1.attention.backends.utils import (
compute_mm_prefix_range_tensor,
get_kv_cache_layout,
get_num_attention_heads_from_layers,
)
from vllm.v1.attention.ops.triton_prefill_attention import context_attention_fwd
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
@@ -110,9 +111,10 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
self.block_size = kv_cache_spec.block_size
model_config = vllm_config.model_config
self.num_heads_q = model_config.get_num_attention_heads(
vllm_config.parallel_config
)
# Compatible with models with non-uniform per-layer head counts.
self.num_heads_q = get_num_attention_heads_from_layers(
vllm_config, layer_names
) or model_config.get_num_attention_heads(vllm_config.parallel_config)
self.num_heads_kv = model_config.get_num_kv_heads(vllm_config.parallel_config)
self.headdim = model_config.get_head_size()
+26
View File
@@ -166,6 +166,32 @@ def get_per_layer_parameters(
return per_layer_params
def get_num_attention_heads_from_layers(
vllm_config: VllmConfig, layer_names: list[str]
) -> int | None:
"""Per-TP-rank ``num_heads`` shared by the named Attention layers.
Use in metadata builders whose plan-time allocations depend on the
head count: the model-wide ``get_num_attention_heads()`` is wrong
for models with non-uniform per-layer head counts. All layers in
one attention group must agree on ``num_heads``; this is asserted.
Returns ``None`` when no matching Attention layer is found.
"""
attn_layers = get_layers_from_vllm_config(
vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
layer_names,
)
if not attn_layers:
return None
heads = {layer.impl.num_heads for layer in attn_layers.values()}
assert len(heads) == 1, (
f"All layers in one attention group must share num_heads; "
f"got {heads} for {layer_names}."
)
return heads.pop()
def infer_global_hyperparameters(
per_layer_params: dict[str, PerLayerParameters],
) -> PerLayerParameters: