[Attention] Allow selecting a different attention backend per KV-cache group (#48012)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
bf578e1abd
commit
5784507da4
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for per-KV-group attention backend selection (backend_per_kind)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.attention import AttentionConfig
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import get_attn_spec_kind
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpecKind
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signals,expected",
|
||||
[
|
||||
(dict(use_mla=False, has_sliding_window=False), "full"),
|
||||
(dict(use_mla=True, has_sliding_window=False), "mla"),
|
||||
(dict(use_mla=True, has_sliding_window=True), "sw_mla"),
|
||||
(dict(use_mla=False, has_sliding_window=True), "sw"),
|
||||
],
|
||||
)
|
||||
def test_get_attn_spec_kind_decoder(signals, expected):
|
||||
kind_by_name = {
|
||||
"full": KVCacheSpecKind.FULL_ATTENTION,
|
||||
"mla": KVCacheSpecKind.MLA_ATTENTION,
|
||||
"sw_mla": KVCacheSpecKind.SLIDING_WINDOW_MLA,
|
||||
"sw": KVCacheSpecKind.SLIDING_WINDOW,
|
||||
}
|
||||
kind = get_attn_spec_kind(attn_type=AttentionType.DECODER, **signals)
|
||||
assert kind is kind_by_name[expected]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attn_type,expected",
|
||||
[
|
||||
(AttentionType.ENCODER_ONLY, KVCacheSpecKind.ENCODER_ONLY_ATTENTION),
|
||||
(AttentionType.ENCODER_DECODER, KVCacheSpecKind.CROSS_ATTENTION),
|
||||
],
|
||||
)
|
||||
def test_get_attn_spec_kind_attn_type(attn_type, expected):
|
||||
kind = get_attn_spec_kind(
|
||||
use_mla=False,
|
||||
has_sliding_window=False,
|
||||
attn_type=attn_type,
|
||||
)
|
||||
assert kind is expected
|
||||
|
||||
|
||||
def test_backend_per_kind_parses_strings():
|
||||
cfg = AttentionConfig(
|
||||
backend_per_kind={
|
||||
"mla_attention": "FLASHINFER_MLA",
|
||||
"sliding_window_mla": "triton_mla", # case-insensitive
|
||||
}
|
||||
)
|
||||
assert cfg.backend_per_kind["mla_attention"] is AttentionBackendEnum.FLASHINFER_MLA
|
||||
assert cfg.backend_per_kind["sliding_window_mla"] is AttentionBackendEnum.TRITON_MLA
|
||||
|
||||
|
||||
def test_backend_per_kind_rejects_unknown_kind():
|
||||
with pytest.raises(ValueError, match="Unknown KV cache group kind"):
|
||||
AttentionConfig(backend_per_kind={"not_a_kind": "TRITON_MLA"})
|
||||
|
||||
|
||||
def test_backend_per_kind_defaults_empty():
|
||||
assert AttentionConfig().backend_per_kind == {}
|
||||
@@ -0,0 +1,94 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end checks that ``backend_per_kind`` selects the requested attention
|
||||
backend for each KV-cache group at runtime.
|
||||
|
||||
Uses ``google/gemma-3-1b-it``, which interleaves full-attention and
|
||||
sliding-window layers, so the model produces separate ``full_attention`` and
|
||||
``sliding_window`` KV-cache groups.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.config.attention import AttentionConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL = "google/gemma-3-1b-it"
|
||||
|
||||
|
||||
def _collect_group_backends(worker) -> list[tuple[str, str]]:
|
||||
"""Runs on the worker: returns (spec_kind, backend_name) per attn group."""
|
||||
from vllm.v1.kv_cache_interface import get_kv_cache_spec_kind
|
||||
|
||||
out: list[tuple[str, str]] = []
|
||||
for kv_group in worker.model_runner.attn_groups:
|
||||
for attn_group in kv_group:
|
||||
kind = get_kv_cache_spec_kind(attn_group.kv_cache_spec)
|
||||
out.append((kind.value, attn_group.backend.get_name()))
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="backend names are CUDA-specific"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"backend_per_kind",
|
||||
[
|
||||
{"full_attention": "FLASH_ATTN", "sliding_window": "TRITON_ATTN"},
|
||||
# Swapped, to prove the mapping is causal rather than the default.
|
||||
{"full_attention": "TRITON_ATTN", "sliding_window": "FLASH_ATTN"},
|
||||
],
|
||||
)
|
||||
def test_backend_per_kind_splits_groups(backend_per_kind, monkeypatch):
|
||||
# collective_rpc ships the callable to the EngineCore subprocess; the
|
||||
# secure msgpack encoder can't serialize functions, so opt into the
|
||||
# pickle fallback (same pattern as test_pooling_chunked_prefill).
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
attention_config=AttentionConfig(backend_per_kind=backend_per_kind),
|
||||
enforce_eager=True,
|
||||
max_model_len=2048,
|
||||
gpu_memory_utilization=0.4,
|
||||
)
|
||||
|
||||
group_backends = llm.llm_engine.collective_rpc(_collect_group_backends)[0]
|
||||
kinds = {kind for kind, _ in group_backends}
|
||||
|
||||
# gemma3 must actually split into both kinds for this test to be meaningful.
|
||||
assert "full_attention" in kinds
|
||||
assert "sliding_window" in kinds
|
||||
|
||||
for kind, backend_name in group_backends:
|
||||
if kind in backend_per_kind:
|
||||
assert backend_name == backend_per_kind[kind], (
|
||||
f"{kind} group used {backend_name}, expected {backend_per_kind[kind]}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="backend names are CUDA-specific"
|
||||
)
|
||||
def test_backend_per_kind_overrides_global_backend(monkeypatch):
|
||||
"""A per-kind entry wins over the global ``backend`` for its kind; other
|
||||
kinds fall back to the global backend."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
attention_config=AttentionConfig(
|
||||
backend="FLASH_ATTN",
|
||||
backend_per_kind={"sliding_window": "TRITON_ATTN"},
|
||||
),
|
||||
enforce_eager=True,
|
||||
max_model_len=2048,
|
||||
gpu_memory_utilization=0.4,
|
||||
)
|
||||
|
||||
group_backends = llm.llm_engine.collective_rpc(_collect_group_backends)[0]
|
||||
|
||||
for kind, backend_name in group_backends:
|
||||
if kind == "sliding_window":
|
||||
assert backend_name == "TRITON_ATTN"
|
||||
elif kind == "full_attention":
|
||||
assert backend_name == "FLASH_ATTN"
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import field
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import field_validator
|
||||
@@ -19,6 +20,17 @@ class AttentionConfig:
|
||||
backend: AttentionBackendEnum | None = None
|
||||
"""Attention backend to use. Use "auto" or None for automatic selection."""
|
||||
|
||||
backend_per_kind: dict[str, AttentionBackendEnum] = field(default_factory=dict)
|
||||
"""Per-KV-cache-group attention backend overrides, keyed by
|
||||
`KVCacheSpecKind` (e.g. `{"mla_attention": "FLASHINFER_MLA",
|
||||
"sliding_window_mla": "TRITON_MLA"}`). This lets a model that splits its
|
||||
layers across multiple KV-cache groups (e.g. interleaved full and
|
||||
sliding-window attention) use a different backend per group.
|
||||
|
||||
An entry overrides `backend` for layers of the matching kind; kinds not
|
||||
listed fall back to `backend` (or automatic selection). A selected backend
|
||||
that is invalid for that kind raises at startup."""
|
||||
|
||||
flash_attn_version: Literal[2, 3, 4] | None = None
|
||||
"""Force vllm to use a specific flash-attention version (2, 3, or 4).
|
||||
Only valid when using the flash-attention backend."""
|
||||
@@ -120,3 +132,29 @@ class AttentionConfig:
|
||||
if isinstance(value, str):
|
||||
return MLAPrefillBackendEnum[value.upper()]
|
||||
return value
|
||||
|
||||
@field_validator("backend_per_kind", mode="before")
|
||||
@classmethod
|
||||
def validate_backend_per_kind_before(cls, value: Any) -> Any:
|
||||
"""Parse the `backend_per_kind` map from strings.
|
||||
|
||||
Keys must be valid `KVCacheSpecKind` values; values are parsed like
|
||||
`backend` (enum name, case-insensitive).
|
||||
"""
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpecKind
|
||||
|
||||
if not isinstance(value, dict):
|
||||
return value
|
||||
valid_kinds = {kind.value for kind in KVCacheSpecKind}
|
||||
parsed: dict[str, AttentionBackendEnum] = {}
|
||||
for kind, backend in value.items():
|
||||
if kind not in valid_kinds:
|
||||
raise ValueError(
|
||||
f"Unknown KV cache group kind '{kind}' in "
|
||||
f"backend_per_kind. Valid kinds are: "
|
||||
f"{', '.join(sorted(valid_kinds))}."
|
||||
)
|
||||
if isinstance(backend, str):
|
||||
backend = AttentionBackendEnum[backend.upper()]
|
||||
parsed[kind] = backend
|
||||
return parsed
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import cache
|
||||
from typing import NamedTuple, cast, get_args
|
||||
from typing import TYPE_CHECKING, NamedTuple, cast, get_args
|
||||
|
||||
import torch
|
||||
|
||||
@@ -15,6 +15,9 @@ from vllm.v1.attention.backends.registry import (
|
||||
MambaAttentionBackendEnum,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpecKind
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -53,6 +56,46 @@ class AttentionSelectorConfig(NamedTuple):
|
||||
)
|
||||
|
||||
|
||||
def get_attn_spec_kind(
|
||||
use_mla: bool,
|
||||
has_sliding_window: bool,
|
||||
attn_type: str,
|
||||
) -> "KVCacheSpecKind":
|
||||
"""Derive the KV-cache group kind a layer belongs to from its signals.
|
||||
|
||||
Mirrors ``get_kv_cache_spec_kind`` (which derives the kind from the
|
||||
produced ``KVCacheSpec``) so users can target groups by kind when
|
||||
setting ``AttentionConfig.backend_per_kind``.
|
||||
|
||||
``SINK_FULL_ATTENTION`` is intentionally not derived here: it is produced
|
||||
only by the ``StaticSinkAttention`` layer, whereas a plain ``Attention``
|
||||
layer with attention sinks (e.g. gpt-oss) still yields a
|
||||
``FullAttentionSpec``/``SlidingWindowSpec``. Sinks therefore do not change
|
||||
the kind.
|
||||
|
||||
Args:
|
||||
use_mla: Whether the layer uses multi-head latent attention.
|
||||
has_sliding_window: Whether the layer applies a sliding window.
|
||||
attn_type: The layer's ``AttentionType``.
|
||||
|
||||
Returns:
|
||||
The ``KVCacheSpecKind`` the layer maps to.
|
||||
"""
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpecKind
|
||||
|
||||
if attn_type == AttentionType.ENCODER_ONLY:
|
||||
return KVCacheSpecKind.ENCODER_ONLY_ATTENTION
|
||||
if attn_type == AttentionType.ENCODER_DECODER:
|
||||
return KVCacheSpecKind.CROSS_ATTENTION
|
||||
if use_mla:
|
||||
if has_sliding_window:
|
||||
return KVCacheSpecKind.SLIDING_WINDOW_MLA
|
||||
return KVCacheSpecKind.MLA_ATTENTION
|
||||
if has_sliding_window:
|
||||
return KVCacheSpecKind.SLIDING_WINDOW
|
||||
return KVCacheSpecKind.FULL_ATTENTION
|
||||
|
||||
|
||||
def get_attn_backend(
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
@@ -91,6 +134,7 @@ def get_attn_backend(
|
||||
kv_transfer_config is not None and kv_transfer_config.is_kv_transfer_instance
|
||||
)
|
||||
|
||||
attn_type = attn_type or AttentionType.DECODER
|
||||
attn_selector_config = AttentionSelectorConfig(
|
||||
head_size=head_size,
|
||||
dtype=dtype,
|
||||
@@ -101,15 +145,27 @@ def get_attn_backend(
|
||||
use_sparse=use_sparse,
|
||||
use_mm_prefix=use_mm_prefix,
|
||||
use_per_head_quant_scales=use_per_head_quant_scales,
|
||||
attn_type=attn_type or AttentionType.DECODER,
|
||||
attn_type=attn_type,
|
||||
has_sliding_window=has_sliding_window,
|
||||
use_non_causal=vllm_config.attention_config.use_non_causal,
|
||||
use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
|
||||
use_kv_connector=use_kv_connector,
|
||||
)
|
||||
|
||||
# A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over
|
||||
# the global backend; kinds not present in the map fall back to it.
|
||||
attention_config = vllm_config.attention_config
|
||||
backend = attention_config.backend
|
||||
if attention_config.backend_per_kind:
|
||||
kind = get_attn_spec_kind(
|
||||
use_mla=use_mla,
|
||||
has_sliding_window=has_sliding_window,
|
||||
attn_type=attn_type,
|
||||
)
|
||||
backend = attention_config.backend_per_kind.get(kind.value, backend)
|
||||
|
||||
return _cached_get_attn_backend(
|
||||
backend=vllm_config.attention_config.backend,
|
||||
backend=backend,
|
||||
attn_selector_config=attn_selector_config,
|
||||
num_heads=num_heads,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user