[Bugfix][Spec Decode] Qualify Transformers backend attention layer names with the model prefix (#59293)

Signed-off-by: Farhad Al-Amin <99006654+Legend-2727@users.noreply.github.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Farhad Al-Amin
2026-10-02 11:21:20 +01:00
committed by GitHub
co-authored by Claude Opus 5.5 Harry Mellor
parent 8e58ac22af
commit af2a582d5c
2 changed files with 24 additions and 8 deletions
@@ -150,6 +150,21 @@ def test_draft_model_quantization(
assert_draft_model_correctness(sd_case, vllm_runner)
@single_gpu_only
def test_draft_model_transformers_backend(vllm_runner):
"""Ensure that Attention prefixes don't collide between the target and
draft models when both models use the Transformers modelling backend."""
sd_case = ArgsTest(
target_model="allenai/OLMo-2-0425-1B-Instruct",
draft_model="allenai/OLMo-2-0425-1B-Instruct",
sampling_config=greedy_sampling(),
num_speculative_tokens=3,
expected_acceptance_len=0.98 * (3 + 1), # epsilon discount of K + 1
expected_acceptance_rate=0.98, # slight epsilon
)
assert_draft_model_correctness(sd_case, vllm_runner)
@multi_gpu_only(num_gpus=2)
def test_draft_model_tensor_parallelism(vllm_runner):
"""Ensure spec decode works when running with TP > 1."""
@@ -184,7 +184,7 @@ class Base(
# Substitute remaining layers with vLLM's layers as needed
self.recursive_replace()
# Create attention instances for KV cache allocation
self._create_attention_instances()
self._create_attention_instances(prefix)
# Initialize any parameters that have not had their modules replaced
self.init_parameters(self.model)
@@ -579,7 +579,7 @@ class Base(
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
def _create_attention_instances(self):
def _create_attention_instances(self, prefix: str = ""):
"""Create `Attention` instances to inform KV cache allocation."""
text_config = self.text_config
attn_cls = self._get_attn_cls()
@@ -613,8 +613,8 @@ class Base(
f"Layer {layer} does not dispatch through the Transformers "
"attention interface and vLLM has no other way to handle it."
)
prefix, attn_fuser = self.attention_fusers[i]
attn_module = self.get_submodule(prefix)
attn_prefix, attn_fuser = self.attention_fusers[i]
attn_module = self.get_submodule(attn_prefix)
# `[i]` is the whole-model config unless the checkpoint is
# heterogeneous, in which case it is this layer's own geometry.
@@ -632,17 +632,18 @@ class Base(
scale=scale,
cache_config=self.cache_config,
quant_config=self.quant_config,
prefix=f"{i}.attn",
prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"),
)
if attn_cls is MLAAttention:
mla_fuser = next(
(f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None
(f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)),
None,
)
if mla_fuser is None:
raise ValueError(
f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of "
"the model did, so its latent KV cache cannot be allocated."
f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest "
"of the model did, so its latent KV cache cannot be allocated."
)
dims = get_mla_dims(self.model_config)
kwargs.update(