[Bugfix][Spec Decode] Qualify Transformers backend attention layer names with the model prefix (#59293)
Signed-off-by: Farhad Al-Amin <99006654+Legend-2727@users.noreply.github.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Harry Mellor
parent
8e58ac22af
commit
af2a582d5c
@@ -150,6 +150,21 @@ def test_draft_model_quantization(
|
|||||||
assert_draft_model_correctness(sd_case, vllm_runner)
|
assert_draft_model_correctness(sd_case, vllm_runner)
|
||||||
|
|
||||||
|
|
||||||
|
@single_gpu_only
|
||||||
|
def test_draft_model_transformers_backend(vllm_runner):
|
||||||
|
"""Ensure that Attention prefixes don't collide between the target and
|
||||||
|
draft models when both models use the Transformers modelling backend."""
|
||||||
|
sd_case = ArgsTest(
|
||||||
|
target_model="allenai/OLMo-2-0425-1B-Instruct",
|
||||||
|
draft_model="allenai/OLMo-2-0425-1B-Instruct",
|
||||||
|
sampling_config=greedy_sampling(),
|
||||||
|
num_speculative_tokens=3,
|
||||||
|
expected_acceptance_len=0.98 * (3 + 1), # epsilon discount of K + 1
|
||||||
|
expected_acceptance_rate=0.98, # slight epsilon
|
||||||
|
)
|
||||||
|
assert_draft_model_correctness(sd_case, vllm_runner)
|
||||||
|
|
||||||
|
|
||||||
@multi_gpu_only(num_gpus=2)
|
@multi_gpu_only(num_gpus=2)
|
||||||
def test_draft_model_tensor_parallelism(vllm_runner):
|
def test_draft_model_tensor_parallelism(vllm_runner):
|
||||||
"""Ensure spec decode works when running with TP > 1."""
|
"""Ensure spec decode works when running with TP > 1."""
|
||||||
|
|||||||
@@ -184,7 +184,7 @@ class Base(
|
|||||||
# Substitute remaining layers with vLLM's layers as needed
|
# Substitute remaining layers with vLLM's layers as needed
|
||||||
self.recursive_replace()
|
self.recursive_replace()
|
||||||
# Create attention instances for KV cache allocation
|
# Create attention instances for KV cache allocation
|
||||||
self._create_attention_instances()
|
self._create_attention_instances(prefix)
|
||||||
|
|
||||||
# Initialize any parameters that have not had their modules replaced
|
# Initialize any parameters that have not had their modules replaced
|
||||||
self.init_parameters(self.model)
|
self.init_parameters(self.model)
|
||||||
@@ -579,7 +579,7 @@ class Base(
|
|||||||
|
|
||||||
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
|
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
|
||||||
|
|
||||||
def _create_attention_instances(self):
|
def _create_attention_instances(self, prefix: str = ""):
|
||||||
"""Create `Attention` instances to inform KV cache allocation."""
|
"""Create `Attention` instances to inform KV cache allocation."""
|
||||||
text_config = self.text_config
|
text_config = self.text_config
|
||||||
attn_cls = self._get_attn_cls()
|
attn_cls = self._get_attn_cls()
|
||||||
@@ -613,8 +613,8 @@ class Base(
|
|||||||
f"Layer {layer} does not dispatch through the Transformers "
|
f"Layer {layer} does not dispatch through the Transformers "
|
||||||
"attention interface and vLLM has no other way to handle it."
|
"attention interface and vLLM has no other way to handle it."
|
||||||
)
|
)
|
||||||
prefix, attn_fuser = self.attention_fusers[i]
|
attn_prefix, attn_fuser = self.attention_fusers[i]
|
||||||
attn_module = self.get_submodule(prefix)
|
attn_module = self.get_submodule(attn_prefix)
|
||||||
|
|
||||||
# `[i]` is the whole-model config unless the checkpoint is
|
# `[i]` is the whole-model config unless the checkpoint is
|
||||||
# heterogeneous, in which case it is this layer's own geometry.
|
# heterogeneous, in which case it is this layer's own geometry.
|
||||||
@@ -632,17 +632,18 @@ class Base(
|
|||||||
scale=scale,
|
scale=scale,
|
||||||
cache_config=self.cache_config,
|
cache_config=self.cache_config,
|
||||||
quant_config=self.quant_config,
|
quant_config=self.quant_config,
|
||||||
prefix=f"{i}.attn",
|
prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"),
|
||||||
)
|
)
|
||||||
|
|
||||||
if attn_cls is MLAAttention:
|
if attn_cls is MLAAttention:
|
||||||
mla_fuser = next(
|
mla_fuser = next(
|
||||||
(f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None
|
(f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)),
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
if mla_fuser is None:
|
if mla_fuser is None:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of "
|
f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest "
|
||||||
"the model did, so its latent KV cache cannot be allocated."
|
"of the model did, so its latent KV cache cannot be allocated."
|
||||||
)
|
)
|
||||||
dims = get_mla_dims(self.model_config)
|
dims = get_mla_dims(self.model_config)
|
||||||
kwargs.update(
|
kwargs.update(
|
||||||
|
|||||||
Reference in New Issue
Block a user