[Bugfix][Spec Decode] Honour the draft's attention_backend on Model Runner V2 (#54826)
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com>
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The draft must honour ``attention_backend`` from --speculative-config.
|
||||
|
||||
``init_attn_backend`` reads the backend off each constructed layer, so these
|
||||
assert on the config ``load_eagle_model`` hands to ``get_model``.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
|
||||
|
||||
@dataclass
|
||||
class _AttentionConfig:
|
||||
backend: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _KernelConfig:
|
||||
moe_backend: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _CacheConfig:
|
||||
cache_dtype: str = "auto"
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SpeculativeConfig:
|
||||
attention_backend: str | None = None
|
||||
moe_backend: str | None = None
|
||||
kv_cache_dtype: str | None = None
|
||||
draft_model_config: object = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _VllmConfig:
|
||||
attention_config: _AttentionConfig
|
||||
kernel_config: _KernelConfig
|
||||
cache_config: _CacheConfig
|
||||
speculative_config: _SpeculativeConfig
|
||||
|
||||
|
||||
def _config(target_backend: str, draft_backend: str | None) -> _VllmConfig:
|
||||
return _VllmConfig(
|
||||
attention_config=_AttentionConfig(backend=target_backend),
|
||||
kernel_config=_KernelConfig(),
|
||||
cache_config=_CacheConfig(),
|
||||
speculative_config=_SpeculativeConfig(attention_backend=draft_backend),
|
||||
)
|
||||
|
||||
|
||||
class _Captured(Exception):
|
||||
def __init__(self, vllm_config):
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
|
||||
def _capture_draft_config(cfg):
|
||||
def _fake_get_model(*, vllm_config, model_config):
|
||||
raise _Captured(vllm_config)
|
||||
|
||||
with (
|
||||
patch("vllm.v1.worker.gpu.spec_decode.eagle.utils.get_model", _fake_get_model),
|
||||
pytest.raises(_Captured) as exc,
|
||||
):
|
||||
load_eagle_model(object(), cfg)
|
||||
return exc.value.vllm_config
|
||||
|
||||
|
||||
def test_draft_attention_backend_overrides_the_target():
|
||||
used = _capture_draft_config(_config("FLASHINFER", "TRITON_ATTN"))
|
||||
assert used.attention_config.backend == "TRITON_ATTN"
|
||||
|
||||
|
||||
def test_unset_leaves_the_target_backend_in_place():
|
||||
"""Clearing it lets the draft autoselect a KV layout the target lacks."""
|
||||
cfg = _config("FLEX_ATTENTION", None)
|
||||
used = _capture_draft_config(cfg)
|
||||
assert used is cfg
|
||||
assert used.attention_config.backend == "FLEX_ATTENTION"
|
||||
|
||||
|
||||
def test_override_does_not_mutate_the_target_config():
|
||||
cfg = _config("FLASHINFER", "TRITON_ATTN")
|
||||
_capture_draft_config(cfg)
|
||||
assert cfg.attention_config.backend == "FLASHINFER"
|
||||
@@ -47,6 +47,16 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
cache_dtype=speculative_config.kv_cache_dtype,
|
||||
),
|
||||
)
|
||||
if speculative_config.attention_backend is not None:
|
||||
# Before get_model(): the backend is read off the constructed layers.
|
||||
# Only when set, so the draft keeps a KV cache layout the target shares.
|
||||
vllm_config = replace(
|
||||
vllm_config,
|
||||
attention_config=replace(
|
||||
vllm_config.attention_config,
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
)
|
||||
with set_model_tag("eagle_head"):
|
||||
eagle_model = get_model(
|
||||
vllm_config=vllm_config, model_config=draft_model_config
|
||||
|
||||
Reference in New Issue
Block a user