[Bugfix] Pick a KV block size supported by every attention backend (#49845)

Signed-off-by: Divy <divy@coralbricks.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Divy
2026-09-23 16:36:35 -04:00
committed by GitHub
co-authored by Claude Opus 5
parent 34b069080d
commit e26547adfb
3 changed files with 132 additions and 21 deletions
+57
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.platforms import current_platform
from vllm.platforms.interface import Platform
from vllm.sampling_params import SamplingParams
from vllm.utils.mem_constants import GiB_bytes
from vllm.utils.system_utils import update_environment_variables
@@ -425,6 +426,62 @@ def test_select_common_block_size_no_valid_option():
select_common_block_size(48, [backend_a, backend_b])
def _mock_backend(supported: list, *, exact: bool = False):
"""Backend accepting multiples of ``supported``, or only those exact sizes
when ``exact`` (as CPU_MLA does)."""
from vllm.v1.attention.backend import AttentionBackend
class _MockBackendCls(AttentionBackend):
@staticmethod
def get_name() -> str:
return "MOCK_EXACT" if exact else "MOCK"
@staticmethod
def get_supported_kernel_block_sizes():
return list(supported)
if exact:
@classmethod
def supports_block_size(cls, block_size: int | None) -> bool:
return block_size is None or block_size in supported
return _MockBackendCls
@pytest.mark.parametrize(
"backends,expected",
[
# A lone backend keeps the default; MultipleOf(8) makes a
# minimum-based regression return 8 instead of 16.
([[MultipleOf(8)]], 16),
# Sparse-MLA main backend (32 or 64) beside an indexer supporting only
# 64: the main backend alone picks 32, which the indexer rejects at
# select_common_block_size time.
([[32, 64], [64]], 64),
([[32, 64]], 32),
# Neither divides the other, so only the LCM satisfies both.
([[32], [48]], 96),
],
)
def test_preferred_block_size_satisfies_every_backend(backends, expected):
classes = [_mock_backend(s) for s in backends]
assert Platform._preferred_block_size_for_backends(classes, 16, None) == expected
def test_preferred_block_size_searches_past_an_exact_size_backend():
# Extending greedily picks lcm(16, 32) = 32, which the exact backend
# rejects; 96 is accepted by both.
classes = [_mock_backend([16, 96], exact=True), _mock_backend([MultipleOf(32)])]
assert Platform._preferred_block_size_for_backends(classes, 16, None) == 96
def test_preferred_block_size_rejects_backends_with_no_common_size():
classes = [_mock_backend([16], exact=True), _mock_backend([MultipleOf(64)])]
with pytest.raises(ValueError, match="share no supported KV cache block size"):
Platform._preferred_block_size_for_backends(classes, 16, None)
def test_set_active_mm_loras_builds_tower_and_connector_mappings():
model = Mock()
model.get_mm_lora_token_counts.side_effect = (
+3 -3
View File
@@ -498,11 +498,11 @@ class CpuPlatform(Platform):
return
# reconcile attention and mamba page sizes
backend_cls = cls._find_non_ssm_backend(vllm_config)
if backend_cls is None:
backend_classes = cls._find_non_ssm_backends(vllm_config)
if not backend_classes:
return
cls._align_hybrid_block_size(vllm_config, backend_cls)
cls._align_hybrid_block_size(vllm_config, backend_classes[0])
@classmethod
def discover_numa_topology(cls) -> list[list[int]]:
+72 -18
View File
@@ -3,6 +3,8 @@
import contextlib
import enum
import functools
import itertools
import math
import os
import platform
import sys
@@ -573,10 +575,10 @@ class Platform:
pass
@classmethod
def _find_non_ssm_backend(
def _find_non_ssm_backends(
cls, vllm_config: "VllmConfig"
) -> "type[AttentionBackend] | None":
"""Find the first non-SSM attention backend from model layers."""
) -> "list[type[AttentionBackend]]":
"""Distinct non-SSM attention backends, in layer order."""
from vllm.config.vllm import get_layers_from_vllm_config
from vllm.model_executor.layers.attention_layer_base import (
AttentionLayerBase,
@@ -586,11 +588,62 @@ class Platform:
vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
)
backends: list[type[AttentionBackend]] = []
for layer in attn_layers.values():
b = layer.get_attn_backend()
if not b.is_ssm():
return b
return None
if not b.is_ssm() and b not in backends:
backends.append(b)
return backends
@classmethod
def _preferred_block_size_for_backends(
cls,
backend_classes: "list[type[AttentionBackend]]",
default_block_size: int,
vllm_config: "VllmConfig",
) -> int:
"""Smallest block size every backend accepts.
``supports_block_size`` may be overridden to accept exact sizes only
(CPU_MLA takes 16 and no multiple of it), so candidates are the LCMs of
one supported size per backend, each checked against every backend.
Raises:
ValueError: If the backends share no supported block size.
"""
from vllm.config.vllm import set_current_vllm_config
from vllm.v1.attention.backend import MultipleOf
# Backends may read the current config to decide a preference.
with set_current_vllm_config(vllm_config):
if len(backend_classes) == 1:
return backend_classes[0].get_preferred_block_size(default_block_size)
if all(b.supports_block_size(default_block_size) for b in backend_classes):
return default_block_size
# A backend declaring no sizes accepts any, so it contributes 1.
per_backend_sizes = [
[
s.base if isinstance(s, MultipleOf) else s
for s in b.get_supported_kernel_block_sizes()
]
or [1]
for b in backend_classes
]
candidates = sorted(
{math.lcm(*sizes) for sizes in itertools.product(*per_backend_sizes)}
)
for candidate in candidates:
if all(b.supports_block_size(candidate) for b in backend_classes):
return candidate
raise ValueError(
"The attention backends share no supported KV cache block size ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
for b in backend_classes
)
+ ")."
)
@classmethod
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
@@ -598,7 +651,6 @@ class Platform:
For hybrid models, also aligns block_size with mamba page sizes.
"""
from vllm.config.cache import CacheConfig
from vllm.config.vllm import set_current_vllm_config
cache_config = vllm_config.cache_config
model_config = vllm_config.model_config
@@ -607,34 +659,36 @@ class Platform:
if not model_config:
return
backend_cls = cls._find_non_ssm_backend(vllm_config)
if backend_cls is None:
backend_classes = cls._find_non_ssm_backends(vllm_config)
if not backend_classes:
return
# Phase 1: Pick block size from backend (skip if user set --block-size)
# Phase 1: Pick a block size every attention backend supports (skip if
# user set --block-size). Models can mix backends with disjoint
# preferences, and a size from the first alone later fails
# select_common_block_size().
if not cache_config.user_specified_block_size:
with set_current_vllm_config(vllm_config):
preferred = backend_cls.get_preferred_block_size(
CacheConfig.DEFAULT_BLOCK_SIZE
)
preferred = cls._preferred_block_size_for_backends(
backend_classes, CacheConfig.DEFAULT_BLOCK_SIZE, vllm_config
)
if preferred != CacheConfig.DEFAULT_BLOCK_SIZE:
logger.info(
"Setting kv cache block size to %d for %s backend.",
"Setting kv cache block size to %d for %s backend(s).",
preferred,
backend_cls.get_name(),
"/".join(b.get_name() for b in backend_classes),
)
cache_config.block_size = preferred
# Phase 2: Align block/mamba sizes for hybrid models
# (may override user settings).
if model_config.is_hybrid:
cls._align_hybrid_block_size(vllm_config, backend_cls)
cls._align_hybrid_block_size(vllm_config, backend_classes[0])
# Phase 3: Align block/page sizes when multiple KV dtypes share the
# block pool (e.g. nvfp4 primary + unquantized skip layers).
# May override the user's --block-size.
if cache_config.kv_cache_dtype_skip_layers:
cls._align_heterogeneous_kv_block_size(vllm_config, backend_cls)
cls._align_heterogeneous_kv_block_size(vllm_config, backend_classes[0])
@classmethod
def _align_heterogeneous_kv_block_size(