[Core] Let SWA layers take the primary block size to avoid inflating the KV block LCM (#53007)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
bnellnm
2026-09-08 16:44:43 -04:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent 808353f477
commit d29c88f162
3 changed files with 45 additions and 11 deletions
+22 -1
View File
@@ -5,12 +5,33 @@
import pytest import pytest
from vllm.config.attention import AttentionConfig from vllm.config.attention import AttentionConfig
from vllm.v1.attention.backend import AttentionType from vllm.model_executor.layers.attention.attention import (
_largest_kernel_block_within,
)
from vllm.v1.attention.backend import AttentionType, MultipleOf
from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.selector import get_attn_spec_kind from vllm.v1.attention.selector import get_attn_spec_kind
from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.kv_cache_interface import KVCacheSpecKind
@pytest.mark.parametrize(
"supported_sizes,expected",
[
([MultipleOf(16)], 1536),
([16, 32], 32),
([2048], 2048),
([MultipleOf(2048)], 2048),
],
)
def test_largest_kernel_block_within(supported_sizes, expected):
class Backend:
@staticmethod
def get_supported_kernel_block_sizes():
return supported_sizes
assert _largest_kernel_block_within(Backend, 1024, 1024 * 1536, 2048) == expected
@pytest.mark.parametrize( @pytest.mark.parametrize(
"signals,expected", "signals,expected",
[ [
@@ -98,7 +98,7 @@ def should_load_quant_weights(quant_method: QuantizeMethodBase | None) -> bool:
def _largest_kernel_block_within( def _largest_kernel_block_within(
attn_backend: "type[AttentionBackend]", attn_backend: "type[AttentionBackend]",
per_token_bytes: int, per_token_bytes: int,
page_budget: int | None, page_budget: int,
fallback: int, fallback: int,
) -> int: ) -> int:
"""Largest supported kernel block size whose page fits in ``page_budget``. """Largest supported kernel block size whose page fits in ``page_budget``.
@@ -106,20 +106,22 @@ def _largest_kernel_block_within(
A padded spec (e.g. skip-quant layer) that pads its page up to a large shared page A padded spec (e.g. skip-quant layer) that pads its page up to a large shared page
wastes ``page_budget - block*per_token`` bytes per block. Picking the largest kernel wastes ``page_budget - block*per_token`` bytes per block. Picking the largest kernel
block whose natural page still fits under ``page_budget`` minimizes that waste. block whose natural page still fits under ``page_budget`` minimizes that waste.
Falls back to the smallest supported block when ``page_budget`` is None (no padding ``MultipleOf`` declarations are expanded to the largest aligned block that fits.
— the block is handled by ``unify``'s integer scaling instead) or nothing fits. Falls back to the smallest supported block when nothing fits.
""" """
from vllm.v1.attention.backend import MultipleOf from vllm.v1.attention.backend import MultipleOf
sizes = attn_backend.get_supported_kernel_block_sizes() sizes = attn_backend.get_supported_kernel_block_sizes()
max_block_size = page_budget // per_token_bytes
candidates = [s for s in sizes if isinstance(s, int)] candidates = [s for s in sizes if isinstance(s, int)]
if not candidates: candidates.extend(
candidates = [s.base for s in sizes if isinstance(s, MultipleOf)] max(s.base, max_block_size // s.base * s.base)
for s in sizes
if isinstance(s, MultipleOf)
)
if not candidates: if not candidates:
return fallback return fallback
smallest = min(candidates) smallest = min(candidates)
if not page_budget or per_token_bytes <= 0:
return smallest
fitting = [b for b in candidates if b * per_token_bytes <= page_budget] fitting = [b for b in candidates if b * per_token_bytes <= page_budget]
return max(fitting) if fitting else smallest return max(fitting) if fitting else smallest
@@ -621,8 +623,15 @@ class Attention(nn.Module, AttentionLayerBase):
# When this SW layer is a padded spec (skip-quant: its page is # When this SW layer is a padded spec (skip-quant: its page is
# padded up to ``skip_page_size_padded``), pick the largest kernel # padded up to ``skip_page_size_padded``), pick the largest kernel
# block that still fits the shared page so we waste fewer padding # block that still fits the shared page so we waste fewer padding
# bytes per block. Otherwise (page_size_padded is None) the smallest # bytes per block. Otherwise (page_size_padded is None) take the
# block is fine — ``unify`` scales it up by an integer ratio. # primary block size when the backend can run it unsplit: if this
# page does not divide the primary page, ``unify`` then pads it
# (padded pages cannot be split) instead of scaling a small block
# to a size coprime with the primary one, which inflates the
# scheduler LCM (e.g. a 1024 B/token SWA draft next to a 1152
# B/token MLA target: 1728 vs 1536 gives LCM 13824). Backends that
# cannot run the primary block start from their smallest block and
# ``unify`` scales it up by an integer ratio.
shared_page = vllm_config.cache_config.skip_page_size_padded shared_page = vllm_config.cache_config.skip_page_size_padded
# The backend owns its packing # The backend owns its packing
sw_per_token = self.attn_backend.customize_spec( sw_per_token = self.attn_backend.customize_spec(
@@ -636,8 +645,9 @@ class Attention(nn.Module, AttentionLayerBase):
sliding_window=self.sliding_window, sliding_window=self.sliding_window,
) )
).real_page_size_bytes ).real_page_size_bytes
page_budget = shared_page or sw_per_token * block_size
sw_block_size = _largest_kernel_block_within( sw_block_size = _largest_kernel_block_within(
self.attn_backend, sw_per_token, shared_page, block_size self.attn_backend, sw_per_token, page_budget, block_size
) )
return SlidingWindowSpec( return SlidingWindowSpec(
block_size=sw_block_size, block_size=sw_block_size,
+3
View File
@@ -729,6 +729,9 @@ def resolve_kv_cache_block_sizes(
] ]
scheduler_block_size = math.lcm(*group_block_sizes) scheduler_block_size = math.lcm(*group_block_sizes)
logger.info("kv cache group sizes %s", group_block_sizes)
logger.info("kv lcm block sizes %s", scheduler_block_size)
# Block hashes are only consumed by prefix caching and KV connectors # Block hashes are only consumed by prefix caching and KV connectors
# (P/D, offloading); when neither is active, keep hash_block_size equal # (P/D, offloading); when neither is active, keep hash_block_size equal
# to the scheduler block size. # to the scheduler block size.