From e26547adfb4970f3e3958695256a84566683a683 Mon Sep 17 00:00:00 2001 From: Divy Date: Wed, 23 Sep 2026 13:36:35 -0700 Subject: [PATCH] [Bugfix] Pick a KV block size supported by every attention backend (#49845) Signed-off-by: Divy Co-authored-by: Claude Opus 5 (1M context) --- tests/v1/worker/test_gpu_model_runner.py | 57 +++++++++++++++ vllm/platforms/cpu.py | 6 +- vllm/platforms/interface.py | 90 +++++++++++++++++++----- 3 files changed, 132 insertions(+), 21 deletions(-) diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 32341ce944..e77d278116 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -31,6 +31,7 @@ from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange from vllm.platforms import current_platform +from vllm.platforms.interface import Platform from vllm.sampling_params import SamplingParams from vllm.utils.mem_constants import GiB_bytes from vllm.utils.system_utils import update_environment_variables @@ -425,6 +426,62 @@ def test_select_common_block_size_no_valid_option(): select_common_block_size(48, [backend_a, backend_b]) +def _mock_backend(supported: list, *, exact: bool = False): + """Backend accepting multiples of ``supported``, or only those exact sizes + when ``exact`` (as CPU_MLA does).""" + from vllm.v1.attention.backend import AttentionBackend + + class _MockBackendCls(AttentionBackend): + @staticmethod + def get_name() -> str: + return "MOCK_EXACT" if exact else "MOCK" + + @staticmethod + def get_supported_kernel_block_sizes(): + return list(supported) + + if exact: + + @classmethod + def supports_block_size(cls, block_size: int | None) -> bool: + return block_size is None or block_size in supported + + return _MockBackendCls + + +@pytest.mark.parametrize( + "backends,expected", + [ + # A lone backend keeps the default; MultipleOf(8) makes a + # minimum-based regression return 8 instead of 16. + ([[MultipleOf(8)]], 16), + # Sparse-MLA main backend (32 or 64) beside an indexer supporting only + # 64: the main backend alone picks 32, which the indexer rejects at + # select_common_block_size time. + ([[32, 64], [64]], 64), + ([[32, 64]], 32), + # Neither divides the other, so only the LCM satisfies both. + ([[32], [48]], 96), + ], +) +def test_preferred_block_size_satisfies_every_backend(backends, expected): + classes = [_mock_backend(s) for s in backends] + assert Platform._preferred_block_size_for_backends(classes, 16, None) == expected + + +def test_preferred_block_size_searches_past_an_exact_size_backend(): + # Extending greedily picks lcm(16, 32) = 32, which the exact backend + # rejects; 96 is accepted by both. + classes = [_mock_backend([16, 96], exact=True), _mock_backend([MultipleOf(32)])] + assert Platform._preferred_block_size_for_backends(classes, 16, None) == 96 + + +def test_preferred_block_size_rejects_backends_with_no_common_size(): + classes = [_mock_backend([16], exact=True), _mock_backend([MultipleOf(64)])] + with pytest.raises(ValueError, match="share no supported KV cache block size"): + Platform._preferred_block_size_for_backends(classes, 16, None) + + def test_set_active_mm_loras_builds_tower_and_connector_mappings(): model = Mock() model.get_mm_lora_token_counts.side_effect = ( diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 390c0dbfed..359195d4c2 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -498,11 +498,11 @@ class CpuPlatform(Platform): return # reconcile attention and mamba page sizes - backend_cls = cls._find_non_ssm_backend(vllm_config) - if backend_cls is None: + backend_classes = cls._find_non_ssm_backends(vllm_config) + if not backend_classes: return - cls._align_hybrid_block_size(vllm_config, backend_cls) + cls._align_hybrid_block_size(vllm_config, backend_classes[0]) @classmethod def discover_numa_topology(cls) -> list[list[int]]: diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 13e5b55402..76d96846e1 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -3,6 +3,8 @@ import contextlib import enum import functools +import itertools +import math import os import platform import sys @@ -573,10 +575,10 @@ class Platform: pass @classmethod - def _find_non_ssm_backend( + def _find_non_ssm_backends( cls, vllm_config: "VllmConfig" - ) -> "type[AttentionBackend] | None": - """Find the first non-SSM attention backend from model layers.""" + ) -> "list[type[AttentionBackend]]": + """Distinct non-SSM attention backends, in layer order.""" from vllm.config.vllm import get_layers_from_vllm_config from vllm.model_executor.layers.attention_layer_base import ( AttentionLayerBase, @@ -586,11 +588,62 @@ class Platform: vllm_config, AttentionLayerBase, # type: ignore[type-abstract] ) + backends: list[type[AttentionBackend]] = [] for layer in attn_layers.values(): b = layer.get_attn_backend() - if not b.is_ssm(): - return b - return None + if not b.is_ssm() and b not in backends: + backends.append(b) + return backends + + @classmethod + def _preferred_block_size_for_backends( + cls, + backend_classes: "list[type[AttentionBackend]]", + default_block_size: int, + vllm_config: "VllmConfig", + ) -> int: + """Smallest block size every backend accepts. + + ``supports_block_size`` may be overridden to accept exact sizes only + (CPU_MLA takes 16 and no multiple of it), so candidates are the LCMs of + one supported size per backend, each checked against every backend. + + Raises: + ValueError: If the backends share no supported block size. + + """ + from vllm.config.vllm import set_current_vllm_config + from vllm.v1.attention.backend import MultipleOf + + # Backends may read the current config to decide a preference. + with set_current_vllm_config(vllm_config): + if len(backend_classes) == 1: + return backend_classes[0].get_preferred_block_size(default_block_size) + if all(b.supports_block_size(default_block_size) for b in backend_classes): + return default_block_size + # A backend declaring no sizes accepts any, so it contributes 1. + per_backend_sizes = [ + [ + s.base if isinstance(s, MultipleOf) else s + for s in b.get_supported_kernel_block_sizes() + ] + or [1] + for b in backend_classes + ] + candidates = sorted( + {math.lcm(*sizes) for sizes in itertools.product(*per_backend_sizes)} + ) + for candidate in candidates: + if all(b.supports_block_size(candidate) for b in backend_classes): + return candidate + raise ValueError( + "The attention backends share no supported KV cache block size (" + + "; ".join( + f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}" + for b in backend_classes + ) + + ")." + ) @classmethod def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None: @@ -598,7 +651,6 @@ class Platform: For hybrid models, also aligns block_size with mamba page sizes. """ from vllm.config.cache import CacheConfig - from vllm.config.vllm import set_current_vllm_config cache_config = vllm_config.cache_config model_config = vllm_config.model_config @@ -607,34 +659,36 @@ class Platform: if not model_config: return - backend_cls = cls._find_non_ssm_backend(vllm_config) - if backend_cls is None: + backend_classes = cls._find_non_ssm_backends(vllm_config) + if not backend_classes: return - # Phase 1: Pick block size from backend (skip if user set --block-size) + # Phase 1: Pick a block size every attention backend supports (skip if + # user set --block-size). Models can mix backends with disjoint + # preferences, and a size from the first alone later fails + # select_common_block_size(). if not cache_config.user_specified_block_size: - with set_current_vllm_config(vllm_config): - preferred = backend_cls.get_preferred_block_size( - CacheConfig.DEFAULT_BLOCK_SIZE - ) + preferred = cls._preferred_block_size_for_backends( + backend_classes, CacheConfig.DEFAULT_BLOCK_SIZE, vllm_config + ) if preferred != CacheConfig.DEFAULT_BLOCK_SIZE: logger.info( - "Setting kv cache block size to %d for %s backend.", + "Setting kv cache block size to %d for %s backend(s).", preferred, - backend_cls.get_name(), + "/".join(b.get_name() for b in backend_classes), ) cache_config.block_size = preferred # Phase 2: Align block/mamba sizes for hybrid models # (may override user settings). if model_config.is_hybrid: - cls._align_hybrid_block_size(vllm_config, backend_cls) + cls._align_hybrid_block_size(vllm_config, backend_classes[0]) # Phase 3: Align block/page sizes when multiple KV dtypes share the # block pool (e.g. nvfp4 primary + unquantized skip layers). # May override the user's --block-size. if cache_config.kv_cache_dtype_skip_layers: - cls._align_heterogeneous_kv_block_size(vllm_config, backend_cls) + cls._align_heterogeneous_kv_block_size(vllm_config, backend_classes[0]) @classmethod def _align_heterogeneous_kv_block_size(