[Bugfix] Pick a KV block size supported by every attention backend (#49845)
Signed-off-by: Divy <divy@coralbricks.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import Platform
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
@@ -425,6 +426,62 @@ def test_select_common_block_size_no_valid_option():
|
||||
select_common_block_size(48, [backend_a, backend_b])
|
||||
|
||||
|
||||
def _mock_backend(supported: list, *, exact: bool = False):
|
||||
"""Backend accepting multiples of ``supported``, or only those exact sizes
|
||||
when ``exact`` (as CPU_MLA does)."""
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
|
||||
class _MockBackendCls(AttentionBackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "MOCK_EXACT" if exact else "MOCK"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
return list(supported)
|
||||
|
||||
if exact:
|
||||
|
||||
@classmethod
|
||||
def supports_block_size(cls, block_size: int | None) -> bool:
|
||||
return block_size is None or block_size in supported
|
||||
|
||||
return _MockBackendCls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backends,expected",
|
||||
[
|
||||
# A lone backend keeps the default; MultipleOf(8) makes a
|
||||
# minimum-based regression return 8 instead of 16.
|
||||
([[MultipleOf(8)]], 16),
|
||||
# Sparse-MLA main backend (32 or 64) beside an indexer supporting only
|
||||
# 64: the main backend alone picks 32, which the indexer rejects at
|
||||
# select_common_block_size time.
|
||||
([[32, 64], [64]], 64),
|
||||
([[32, 64]], 32),
|
||||
# Neither divides the other, so only the LCM satisfies both.
|
||||
([[32], [48]], 96),
|
||||
],
|
||||
)
|
||||
def test_preferred_block_size_satisfies_every_backend(backends, expected):
|
||||
classes = [_mock_backend(s) for s in backends]
|
||||
assert Platform._preferred_block_size_for_backends(classes, 16, None) == expected
|
||||
|
||||
|
||||
def test_preferred_block_size_searches_past_an_exact_size_backend():
|
||||
# Extending greedily picks lcm(16, 32) = 32, which the exact backend
|
||||
# rejects; 96 is accepted by both.
|
||||
classes = [_mock_backend([16, 96], exact=True), _mock_backend([MultipleOf(32)])]
|
||||
assert Platform._preferred_block_size_for_backends(classes, 16, None) == 96
|
||||
|
||||
|
||||
def test_preferred_block_size_rejects_backends_with_no_common_size():
|
||||
classes = [_mock_backend([16], exact=True), _mock_backend([MultipleOf(64)])]
|
||||
with pytest.raises(ValueError, match="share no supported KV cache block size"):
|
||||
Platform._preferred_block_size_for_backends(classes, 16, None)
|
||||
|
||||
|
||||
def test_set_active_mm_loras_builds_tower_and_connector_mappings():
|
||||
model = Mock()
|
||||
model.get_mm_lora_token_counts.side_effect = (
|
||||
|
||||
Reference in New Issue
Block a user