[Bugfix][Spec Decode] Honour the draft's attention_backend on Model Runner V2 (#54826)

Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com>
This commit is contained in:
Stefano Castagnetta
2026-09-04 12:23:19 -07:00
committed by GitHub
parent 5690b02c03
commit 2524051385
2 changed files with 100 additions and 0 deletions
@@ -0,0 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The draft must honour ``attention_backend`` from --speculative-config.
``init_attn_backend`` reads the backend off each constructed layer, so these
assert on the config ``load_eagle_model`` hands to ``get_model``.
"""
from dataclasses import dataclass
from unittest.mock import patch
import pytest
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
@dataclass
class _AttentionConfig:
backend: str | None = None
@dataclass
class _KernelConfig:
moe_backend: str | None = None
@dataclass
class _CacheConfig:
cache_dtype: str = "auto"
@dataclass
class _SpeculativeConfig:
attention_backend: str | None = None
moe_backend: str | None = None
kv_cache_dtype: str | None = None
draft_model_config: object = None
@dataclass
class _VllmConfig:
attention_config: _AttentionConfig
kernel_config: _KernelConfig
cache_config: _CacheConfig
speculative_config: _SpeculativeConfig
def _config(target_backend: str, draft_backend: str | None) -> _VllmConfig:
return _VllmConfig(
attention_config=_AttentionConfig(backend=target_backend),
kernel_config=_KernelConfig(),
cache_config=_CacheConfig(),
speculative_config=_SpeculativeConfig(attention_backend=draft_backend),
)
class _Captured(Exception):
def __init__(self, vllm_config):
self.vllm_config = vllm_config
def _capture_draft_config(cfg):
def _fake_get_model(*, vllm_config, model_config):
raise _Captured(vllm_config)
with (
patch("vllm.v1.worker.gpu.spec_decode.eagle.utils.get_model", _fake_get_model),
pytest.raises(_Captured) as exc,
):
load_eagle_model(object(), cfg)
return exc.value.vllm_config
def test_draft_attention_backend_overrides_the_target():
used = _capture_draft_config(_config("FLASHINFER", "TRITON_ATTN"))
assert used.attention_config.backend == "TRITON_ATTN"
def test_unset_leaves_the_target_backend_in_place():
"""Clearing it lets the draft autoselect a KV layout the target lacks."""
cfg = _config("FLEX_ATTENTION", None)
used = _capture_draft_config(cfg)
assert used is cfg
assert used.attention_config.backend == "FLEX_ATTENTION"
def test_override_does_not_mutate_the_target_config():
cfg = _config("FLASHINFER", "TRITON_ATTN")
_capture_draft_config(cfg)
assert cfg.attention_config.backend == "FLASHINFER"
@@ -47,6 +47,16 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
cache_dtype=speculative_config.kv_cache_dtype,
),
)
if speculative_config.attention_backend is not None:
# Before get_model(): the backend is read off the constructed layers.
# Only when set, so the draft keeps a KV cache layout the target shares.
vllm_config = replace(
vllm_config,
attention_config=replace(
vllm_config.attention_config,
backend=speculative_config.attention_backend,
),
)
with set_model_tag("eagle_head"):
eagle_model = get_model(
vllm_config=vllm_config, model_config=draft_model_config