diff --git a/tests/v1/spec_decode/test_draft_attention_backend_override.py b/tests/v1/spec_decode/test_draft_attention_backend_override.py new file mode 100644 index 0000000000..5984d33769 --- /dev/null +++ b/tests/v1/spec_decode/test_draft_attention_backend_override.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The draft must honour ``attention_backend`` from --speculative-config. + +``init_attn_backend`` reads the backend off each constructed layer, so these +assert on the config ``load_eagle_model`` hands to ``get_model``. +""" + +from dataclasses import dataclass +from unittest.mock import patch + +import pytest + +from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model + + +@dataclass +class _AttentionConfig: + backend: str | None = None + + +@dataclass +class _KernelConfig: + moe_backend: str | None = None + + +@dataclass +class _CacheConfig: + cache_dtype: str = "auto" + + +@dataclass +class _SpeculativeConfig: + attention_backend: str | None = None + moe_backend: str | None = None + kv_cache_dtype: str | None = None + draft_model_config: object = None + + +@dataclass +class _VllmConfig: + attention_config: _AttentionConfig + kernel_config: _KernelConfig + cache_config: _CacheConfig + speculative_config: _SpeculativeConfig + + +def _config(target_backend: str, draft_backend: str | None) -> _VllmConfig: + return _VllmConfig( + attention_config=_AttentionConfig(backend=target_backend), + kernel_config=_KernelConfig(), + cache_config=_CacheConfig(), + speculative_config=_SpeculativeConfig(attention_backend=draft_backend), + ) + + +class _Captured(Exception): + def __init__(self, vllm_config): + self.vllm_config = vllm_config + + +def _capture_draft_config(cfg): + def _fake_get_model(*, vllm_config, model_config): + raise _Captured(vllm_config) + + with ( + patch("vllm.v1.worker.gpu.spec_decode.eagle.utils.get_model", _fake_get_model), + pytest.raises(_Captured) as exc, + ): + load_eagle_model(object(), cfg) + return exc.value.vllm_config + + +def test_draft_attention_backend_overrides_the_target(): + used = _capture_draft_config(_config("FLASHINFER", "TRITON_ATTN")) + assert used.attention_config.backend == "TRITON_ATTN" + + +def test_unset_leaves_the_target_backend_in_place(): + """Clearing it lets the draft autoselect a KV layout the target lacks.""" + cfg = _config("FLEX_ATTENTION", None) + used = _capture_draft_config(cfg) + assert used is cfg + assert used.attention_config.backend == "FLEX_ATTENTION" + + +def test_override_does_not_mutate_the_target_config(): + cfg = _config("FLASHINFER", "TRITON_ATTN") + _capture_draft_config(cfg) + assert cfg.attention_config.backend == "FLASHINFER" diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py index 579652bc7d..f1d8bd42e1 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py @@ -47,6 +47,16 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod cache_dtype=speculative_config.kv_cache_dtype, ), ) + if speculative_config.attention_backend is not None: + # Before get_model(): the backend is read off the constructed layers. + # Only when set, so the draft keeps a KV cache layout the target shares. + vllm_config = replace( + vllm_config, + attention_config=replace( + vllm_config.attention_config, + backend=speculative_config.attention_backend, + ), + ) with set_model_tag("eagle_head"): eagle_model = get_model( vllm_config=vllm_config, model_config=draft_model_config