[Bugfix][Spec Decode] Qualify Transformers backend attention layer names with the model prefix (#59293)
Signed-off-by: Farhad Al-Amin <99006654+Legend-2727@users.noreply.github.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Harry Mellor
parent
8e58ac22af
commit
af2a582d5c
@@ -150,6 +150,21 @@ def test_draft_model_quantization(
|
||||
assert_draft_model_correctness(sd_case, vllm_runner)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
def test_draft_model_transformers_backend(vllm_runner):
|
||||
"""Ensure that Attention prefixes don't collide between the target and
|
||||
draft models when both models use the Transformers modelling backend."""
|
||||
sd_case = ArgsTest(
|
||||
target_model="allenai/OLMo-2-0425-1B-Instruct",
|
||||
draft_model="allenai/OLMo-2-0425-1B-Instruct",
|
||||
sampling_config=greedy_sampling(),
|
||||
num_speculative_tokens=3,
|
||||
expected_acceptance_len=0.98 * (3 + 1), # epsilon discount of K + 1
|
||||
expected_acceptance_rate=0.98, # slight epsilon
|
||||
)
|
||||
assert_draft_model_correctness(sd_case, vllm_runner)
|
||||
|
||||
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
def test_draft_model_tensor_parallelism(vllm_runner):
|
||||
"""Ensure spec decode works when running with TP > 1."""
|
||||
|
||||
@@ -184,7 +184,7 @@ class Base(
|
||||
# Substitute remaining layers with vLLM's layers as needed
|
||||
self.recursive_replace()
|
||||
# Create attention instances for KV cache allocation
|
||||
self._create_attention_instances()
|
||||
self._create_attention_instances(prefix)
|
||||
|
||||
# Initialize any parameters that have not had their modules replaced
|
||||
self.init_parameters(self.model)
|
||||
@@ -579,7 +579,7 @@ class Base(
|
||||
|
||||
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
|
||||
|
||||
def _create_attention_instances(self):
|
||||
def _create_attention_instances(self, prefix: str = ""):
|
||||
"""Create `Attention` instances to inform KV cache allocation."""
|
||||
text_config = self.text_config
|
||||
attn_cls = self._get_attn_cls()
|
||||
@@ -613,8 +613,8 @@ class Base(
|
||||
f"Layer {layer} does not dispatch through the Transformers "
|
||||
"attention interface and vLLM has no other way to handle it."
|
||||
)
|
||||
prefix, attn_fuser = self.attention_fusers[i]
|
||||
attn_module = self.get_submodule(prefix)
|
||||
attn_prefix, attn_fuser = self.attention_fusers[i]
|
||||
attn_module = self.get_submodule(attn_prefix)
|
||||
|
||||
# `[i]` is the whole-model config unless the checkpoint is
|
||||
# heterogeneous, in which case it is this layer's own geometry.
|
||||
@@ -632,17 +632,18 @@ class Base(
|
||||
scale=scale,
|
||||
cache_config=self.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{i}.attn",
|
||||
prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"),
|
||||
)
|
||||
|
||||
if attn_cls is MLAAttention:
|
||||
mla_fuser = next(
|
||||
(f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None
|
||||
(f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)),
|
||||
None,
|
||||
)
|
||||
if mla_fuser is None:
|
||||
raise ValueError(
|
||||
f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of "
|
||||
"the model did, so its latent KV cache cannot be allocated."
|
||||
f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest "
|
||||
"of the model did, so its latent KV cache cannot be allocated."
|
||||
)
|
||||
dims = get_mla_dims(self.model_config)
|
||||
kwargs.update(
|
||||
|
||||
Reference in New Issue
Block a user