diff --git a/tests/v1/attention/test_backend_per_kind.py b/tests/v1/attention/test_backend_per_kind.py new file mode 100644 index 0000000000..5b197f7d15 --- /dev/null +++ b/tests/v1/attention/test_backend_per_kind.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for per-KV-group attention backend selection (backend_per_kind).""" + +import pytest + +from vllm.config.attention import AttentionConfig +from vllm.v1.attention.backend import AttentionType +from vllm.v1.attention.backends.registry import AttentionBackendEnum +from vllm.v1.attention.selector import get_attn_spec_kind +from vllm.v1.kv_cache_interface import KVCacheSpecKind + + +@pytest.mark.parametrize( + "signals,expected", + [ + (dict(use_mla=False, has_sliding_window=False), "full"), + (dict(use_mla=True, has_sliding_window=False), "mla"), + (dict(use_mla=True, has_sliding_window=True), "sw_mla"), + (dict(use_mla=False, has_sliding_window=True), "sw"), + ], +) +def test_get_attn_spec_kind_decoder(signals, expected): + kind_by_name = { + "full": KVCacheSpecKind.FULL_ATTENTION, + "mla": KVCacheSpecKind.MLA_ATTENTION, + "sw_mla": KVCacheSpecKind.SLIDING_WINDOW_MLA, + "sw": KVCacheSpecKind.SLIDING_WINDOW, + } + kind = get_attn_spec_kind(attn_type=AttentionType.DECODER, **signals) + assert kind is kind_by_name[expected] + + +@pytest.mark.parametrize( + "attn_type,expected", + [ + (AttentionType.ENCODER_ONLY, KVCacheSpecKind.ENCODER_ONLY_ATTENTION), + (AttentionType.ENCODER_DECODER, KVCacheSpecKind.CROSS_ATTENTION), + ], +) +def test_get_attn_spec_kind_attn_type(attn_type, expected): + kind = get_attn_spec_kind( + use_mla=False, + has_sliding_window=False, + attn_type=attn_type, + ) + assert kind is expected + + +def test_backend_per_kind_parses_strings(): + cfg = AttentionConfig( + backend_per_kind={ + "mla_attention": "FLASHINFER_MLA", + "sliding_window_mla": "triton_mla", # case-insensitive + } + ) + assert cfg.backend_per_kind["mla_attention"] is AttentionBackendEnum.FLASHINFER_MLA + assert cfg.backend_per_kind["sliding_window_mla"] is AttentionBackendEnum.TRITON_MLA + + +def test_backend_per_kind_rejects_unknown_kind(): + with pytest.raises(ValueError, match="Unknown KV cache group kind"): + AttentionConfig(backend_per_kind={"not_a_kind": "TRITON_MLA"}) + + +def test_backend_per_kind_defaults_empty(): + assert AttentionConfig().backend_per_kind == {} diff --git a/tests/v1/e2e/general/test_attention_backend_per_kind.py b/tests/v1/e2e/general/test_attention_backend_per_kind.py new file mode 100644 index 0000000000..3281247b1c --- /dev/null +++ b/tests/v1/e2e/general/test_attention_backend_per_kind.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""End-to-end checks that ``backend_per_kind`` selects the requested attention +backend for each KV-cache group at runtime. + +Uses ``google/gemma-3-1b-it``, which interleaves full-attention and +sliding-window layers, so the model produces separate ``full_attention`` and +``sliding_window`` KV-cache groups. +""" + +import pytest + +from vllm import LLM +from vllm.config.attention import AttentionConfig +from vllm.platforms import current_platform + +MODEL = "google/gemma-3-1b-it" + + +def _collect_group_backends(worker) -> list[tuple[str, str]]: + """Runs on the worker: returns (spec_kind, backend_name) per attn group.""" + from vllm.v1.kv_cache_interface import get_kv_cache_spec_kind + + out: list[tuple[str, str]] = [] + for kv_group in worker.model_runner.attn_groups: + for attn_group in kv_group: + kind = get_kv_cache_spec_kind(attn_group.kv_cache_spec) + out.append((kind.value, attn_group.backend.get_name())) + return out + + +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="backend names are CUDA-specific" +) +@pytest.mark.parametrize( + "backend_per_kind", + [ + {"full_attention": "FLASH_ATTN", "sliding_window": "TRITON_ATTN"}, + # Swapped, to prove the mapping is causal rather than the default. + {"full_attention": "TRITON_ATTN", "sliding_window": "FLASH_ATTN"}, + ], +) +def test_backend_per_kind_splits_groups(backend_per_kind, monkeypatch): + # collective_rpc ships the callable to the EngineCore subprocess; the + # secure msgpack encoder can't serialize functions, so opt into the + # pickle fallback (same pattern as test_pooling_chunked_prefill). + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + llm = LLM( + model=MODEL, + attention_config=AttentionConfig(backend_per_kind=backend_per_kind), + enforce_eager=True, + max_model_len=2048, + gpu_memory_utilization=0.4, + ) + + group_backends = llm.llm_engine.collective_rpc(_collect_group_backends)[0] + kinds = {kind for kind, _ in group_backends} + + # gemma3 must actually split into both kinds for this test to be meaningful. + assert "full_attention" in kinds + assert "sliding_window" in kinds + + for kind, backend_name in group_backends: + if kind in backend_per_kind: + assert backend_name == backend_per_kind[kind], ( + f"{kind} group used {backend_name}, expected {backend_per_kind[kind]}" + ) + + +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="backend names are CUDA-specific" +) +def test_backend_per_kind_overrides_global_backend(monkeypatch): + """A per-kind entry wins over the global ``backend`` for its kind; other + kinds fall back to the global backend.""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + llm = LLM( + model=MODEL, + attention_config=AttentionConfig( + backend="FLASH_ATTN", + backend_per_kind={"sliding_window": "TRITON_ATTN"}, + ), + enforce_eager=True, + max_model_len=2048, + gpu_memory_utilization=0.4, + ) + + group_backends = llm.llm_engine.collective_rpc(_collect_group_backends)[0] + + for kind, backend_name in group_backends: + if kind == "sliding_window": + assert backend_name == "TRITON_ATTN" + elif kind == "full_attention": + assert backend_name == "FLASH_ATTN" diff --git a/vllm/config/attention.py b/vllm/config/attention.py index e2ad4729a1..78bc1f9537 100644 --- a/vllm/config/attention.py +++ b/vllm/config/attention.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import field from typing import Any, Literal from pydantic import field_validator @@ -19,6 +20,17 @@ class AttentionConfig: backend: AttentionBackendEnum | None = None """Attention backend to use. Use "auto" or None for automatic selection.""" + backend_per_kind: dict[str, AttentionBackendEnum] = field(default_factory=dict) + """Per-KV-cache-group attention backend overrides, keyed by + `KVCacheSpecKind` (e.g. `{"mla_attention": "FLASHINFER_MLA", + "sliding_window_mla": "TRITON_MLA"}`). This lets a model that splits its + layers across multiple KV-cache groups (e.g. interleaved full and + sliding-window attention) use a different backend per group. + + An entry overrides `backend` for layers of the matching kind; kinds not + listed fall back to `backend` (or automatic selection). A selected backend + that is invalid for that kind raises at startup.""" + flash_attn_version: Literal[2, 3, 4] | None = None """Force vllm to use a specific flash-attention version (2, 3, or 4). Only valid when using the flash-attention backend.""" @@ -120,3 +132,29 @@ class AttentionConfig: if isinstance(value, str): return MLAPrefillBackendEnum[value.upper()] return value + + @field_validator("backend_per_kind", mode="before") + @classmethod + def validate_backend_per_kind_before(cls, value: Any) -> Any: + """Parse the `backend_per_kind` map from strings. + + Keys must be valid `KVCacheSpecKind` values; values are parsed like + `backend` (enum name, case-insensitive). + """ + from vllm.v1.kv_cache_interface import KVCacheSpecKind + + if not isinstance(value, dict): + return value + valid_kinds = {kind.value for kind in KVCacheSpecKind} + parsed: dict[str, AttentionBackendEnum] = {} + for kind, backend in value.items(): + if kind not in valid_kinds: + raise ValueError( + f"Unknown KV cache group kind '{kind}' in " + f"backend_per_kind. Valid kinds are: " + f"{', '.join(sorted(valid_kinds))}." + ) + if isinstance(backend, str): + backend = AttentionBackendEnum[backend.upper()] + parsed[kind] = backend + return parsed diff --git a/vllm/v1/attention/selector.py b/vllm/v1/attention/selector.py index 917f2f6f70..387eac34c9 100644 --- a/vllm/v1/attention/selector.py +++ b/vllm/v1/attention/selector.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from functools import cache -from typing import NamedTuple, cast, get_args +from typing import TYPE_CHECKING, NamedTuple, cast, get_args import torch @@ -15,6 +15,9 @@ from vllm.v1.attention.backends.registry import ( MambaAttentionBackendEnum, ) +if TYPE_CHECKING: + from vllm.v1.kv_cache_interface import KVCacheSpecKind + logger = init_logger(__name__) @@ -53,6 +56,46 @@ class AttentionSelectorConfig(NamedTuple): ) +def get_attn_spec_kind( + use_mla: bool, + has_sliding_window: bool, + attn_type: str, +) -> "KVCacheSpecKind": + """Derive the KV-cache group kind a layer belongs to from its signals. + + Mirrors ``get_kv_cache_spec_kind`` (which derives the kind from the + produced ``KVCacheSpec``) so users can target groups by kind when + setting ``AttentionConfig.backend_per_kind``. + + ``SINK_FULL_ATTENTION`` is intentionally not derived here: it is produced + only by the ``StaticSinkAttention`` layer, whereas a plain ``Attention`` + layer with attention sinks (e.g. gpt-oss) still yields a + ``FullAttentionSpec``/``SlidingWindowSpec``. Sinks therefore do not change + the kind. + + Args: + use_mla: Whether the layer uses multi-head latent attention. + has_sliding_window: Whether the layer applies a sliding window. + attn_type: The layer's ``AttentionType``. + + Returns: + The ``KVCacheSpecKind`` the layer maps to. + """ + from vllm.v1.kv_cache_interface import KVCacheSpecKind + + if attn_type == AttentionType.ENCODER_ONLY: + return KVCacheSpecKind.ENCODER_ONLY_ATTENTION + if attn_type == AttentionType.ENCODER_DECODER: + return KVCacheSpecKind.CROSS_ATTENTION + if use_mla: + if has_sliding_window: + return KVCacheSpecKind.SLIDING_WINDOW_MLA + return KVCacheSpecKind.MLA_ATTENTION + if has_sliding_window: + return KVCacheSpecKind.SLIDING_WINDOW + return KVCacheSpecKind.FULL_ATTENTION + + def get_attn_backend( head_size: int, dtype: torch.dtype, @@ -91,6 +134,7 @@ def get_attn_backend( kv_transfer_config is not None and kv_transfer_config.is_kv_transfer_instance ) + attn_type = attn_type or AttentionType.DECODER attn_selector_config = AttentionSelectorConfig( head_size=head_size, dtype=dtype, @@ -101,15 +145,27 @@ def get_attn_backend( use_sparse=use_sparse, use_mm_prefix=use_mm_prefix, use_per_head_quant_scales=use_per_head_quant_scales, - attn_type=attn_type or AttentionType.DECODER, + attn_type=attn_type, has_sliding_window=has_sliding_window, use_non_causal=vllm_config.attention_config.use_non_causal, use_batch_invariant=envs.VLLM_BATCH_INVARIANT, use_kv_connector=use_kv_connector, ) + # A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over + # the global backend; kinds not present in the map fall back to it. + attention_config = vllm_config.attention_config + backend = attention_config.backend + if attention_config.backend_per_kind: + kind = get_attn_spec_kind( + use_mla=use_mla, + has_sliding_window=has_sliding_window, + attn_type=attn_type, + ) + backend = attention_config.backend_per_kind.get(kind.value, backend) + return _cached_get_attn_backend( - backend=vllm_config.attention_config.backend, + backend=backend, attn_selector_config=attn_selector_config, num_heads=num_heads, )