diff --git a/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py b/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py index d2c2ac87b7..10d449c86f 100644 --- a/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py +++ b/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py @@ -150,6 +150,21 @@ def test_draft_model_quantization( assert_draft_model_correctness(sd_case, vllm_runner) +@single_gpu_only +def test_draft_model_transformers_backend(vllm_runner): + """Ensure that Attention prefixes don't collide between the target and + draft models when both models use the Transformers modelling backend.""" + sd_case = ArgsTest( + target_model="allenai/OLMo-2-0425-1B-Instruct", + draft_model="allenai/OLMo-2-0425-1B-Instruct", + sampling_config=greedy_sampling(), + num_speculative_tokens=3, + expected_acceptance_len=0.98 * (3 + 1), # epsilon discount of K + 1 + expected_acceptance_rate=0.98, # slight epsilon + ) + assert_draft_model_correctness(sd_case, vllm_runner) + + @multi_gpu_only(num_gpus=2) def test_draft_model_tensor_parallelism(vllm_runner): """Ensure spec decode works when running with TP > 1.""" diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index 4e53f1fe18..239ef821e7 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -184,7 +184,7 @@ class Base( # Substitute remaining layers with vLLM's layers as needed self.recursive_replace() # Create attention instances for KV cache allocation - self._create_attention_instances() + self._create_attention_instances(prefix) # Initialize any parameters that have not had their modules replaced self.init_parameters(self.model) @@ -579,7 +579,7 @@ class Base( self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked) - def _create_attention_instances(self): + def _create_attention_instances(self, prefix: str = ""): """Create `Attention` instances to inform KV cache allocation.""" text_config = self.text_config attn_cls = self._get_attn_cls() @@ -613,8 +613,8 @@ class Base( f"Layer {layer} does not dispatch through the Transformers " "attention interface and vLLM has no other way to handle it." ) - prefix, attn_fuser = self.attention_fusers[i] - attn_module = self.get_submodule(prefix) + attn_prefix, attn_fuser = self.attention_fusers[i] + attn_module = self.get_submodule(attn_prefix) # `[i]` is the whole-model config unless the checkpoint is # heterogeneous, in which case it is this layer's own geometry. @@ -632,17 +632,18 @@ class Base( scale=scale, cache_config=self.cache_config, quant_config=self.quant_config, - prefix=f"{i}.attn", + prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"), ) if attn_cls is MLAAttention: mla_fuser = next( - (f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None + (f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)), + None, ) if mla_fuser is None: raise ValueError( - f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of " - "the model did, so its latent KV cache cannot be allocated." + f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest " + "of the model did, so its latent KV cache cannot be allocated." ) dims = get_mla_dims(self.model_config) kwargs.update(