[HMA]Fix corner case when hybrid page_size can not be evenly divided issue (blk_size=64,tp=4) (#37467)
Signed-off-by: Chendi Xue <chendi.xue@intel.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Chendi.Xue <chendi.xue@intel.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Nicolò Lucchesi
parent
b4a2f3ac36
commit
3b1dbaad4e
+175
-34
@@ -21,6 +21,7 @@ if TYPE_CHECKING:
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
else:
|
||||
FlexibleArgumentParser = object
|
||||
@@ -424,29 +425,11 @@ class Platform:
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
|
||||
"""
|
||||
Ensure block_size is compatible with the attention backend.
|
||||
"""
|
||||
from vllm.config.cache import CacheConfig
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config.user_specified_block_size:
|
||||
# User specified --block-size; keep it.
|
||||
return
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
# model_config may be None during testing.
|
||||
# Skip hybrid models — their block_size is managed by
|
||||
# HybridAttentionMambaModelConfig.
|
||||
if model_config is None or model_config.is_hybrid:
|
||||
cache_config.block_size = CacheConfig.DEFAULT_BLOCK_SIZE
|
||||
return
|
||||
|
||||
from vllm.config.vllm import (
|
||||
get_layers_from_vllm_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
def _find_non_ssm_backend(
|
||||
cls, vllm_config: "VllmConfig"
|
||||
) -> "type[AttentionBackend] | None":
|
||||
"""Find the first non-SSM attention backend from model layers."""
|
||||
from vllm.config.vllm import get_layers_from_vllm_config
|
||||
from vllm.model_executor.layers.attention_layer_base import (
|
||||
AttentionLayerBase,
|
||||
)
|
||||
@@ -455,23 +438,181 @@ class Platform:
|
||||
vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
)
|
||||
if not attn_layers:
|
||||
cache_config.block_size = CacheConfig.DEFAULT_BLOCK_SIZE
|
||||
for layer in attn_layers.values():
|
||||
b = layer.get_attn_backend()
|
||||
if not b.is_ssm():
|
||||
return b
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
|
||||
"""
|
||||
Ensure block_size is compatible with the attention backend.
|
||||
For hybrid models, also aligns block_size with mamba page sizes.
|
||||
"""
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
model_config = vllm_config.model_config
|
||||
|
||||
# model_config may be None during testing.
|
||||
if not model_config:
|
||||
return
|
||||
|
||||
first_layer = next(iter(attn_layers.values()))
|
||||
backend_cls = first_layer.get_attn_backend()
|
||||
backend_cls = cls._find_non_ssm_backend(vllm_config)
|
||||
if backend_cls is None:
|
||||
return
|
||||
|
||||
# Phase 1: Pick block size from backend (skip if user set --block-size)
|
||||
if not cache_config.user_specified_block_size:
|
||||
with set_current_vllm_config(vllm_config):
|
||||
preferred = backend_cls.get_preferred_block_size(
|
||||
CacheConfig.DEFAULT_BLOCK_SIZE
|
||||
)
|
||||
if preferred != CacheConfig.DEFAULT_BLOCK_SIZE:
|
||||
logger.info(
|
||||
"Setting kv cache block size to %d for %s backend.",
|
||||
preferred,
|
||||
backend_cls.get_name(),
|
||||
)
|
||||
cache_config.block_size = preferred
|
||||
|
||||
# Phase 2: Align block/mamba sizes for hybrid models
|
||||
# (may override user settings).
|
||||
if model_config.is_hybrid:
|
||||
cls._align_hybrid_block_size(vllm_config, backend_cls)
|
||||
|
||||
@classmethod
|
||||
def _align_hybrid_block_size(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
backend_cls: "type[AttentionBackend]",
|
||||
) -> None:
|
||||
"""
|
||||
For hybrid attention/mamba models, ensure that the attention page
|
||||
size is >= the mamba page size, and pad the mamba page size to match.
|
||||
"""
|
||||
from math import lcm
|
||||
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.model_executor.models import ModelRegistry
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
model_config = vllm_config.model_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
|
||||
if cache_config.cache_dtype == "auto":
|
||||
kv_cache_dtype = model_config.dtype
|
||||
else:
|
||||
kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
|
||||
|
||||
# Compute attention page size for 1 token
|
||||
if model_config.use_mla:
|
||||
attn_page_size_1_token = MLAAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
).page_size_bytes
|
||||
else:
|
||||
attn_page_size_1_token = FullAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
).page_size_bytes
|
||||
|
||||
# Compute mamba page size
|
||||
model_cls, _ = ModelRegistry.resolve_model_cls(
|
||||
model_config.architecture,
|
||||
model_config=model_config,
|
||||
)
|
||||
mamba_page_size = MambaSpec(
|
||||
shapes=model_cls.get_mamba_state_shape_from_config(vllm_config),
|
||||
dtypes=model_cls.get_mamba_state_dtype_from_config(vllm_config),
|
||||
block_size=-1,
|
||||
).page_size_bytes
|
||||
|
||||
if mamba_page_size == 0:
|
||||
return
|
||||
|
||||
# mamba_block_size here should either be user specified value or None
|
||||
mamba_block_size = (
|
||||
cache_config.mamba_block_size
|
||||
if cache_config.user_specified_mamba_block_size
|
||||
else None
|
||||
)
|
||||
|
||||
# Get kernel block alignment from the backend's supported sizes
|
||||
with set_current_vllm_config(vllm_config):
|
||||
preferred = backend_cls.get_preferred_block_size(
|
||||
CacheConfig.DEFAULT_BLOCK_SIZE
|
||||
kernel_block_alignment_size = max(
|
||||
min(
|
||||
s.base if isinstance(s, MultipleOf) else s
|
||||
for s in backend_cls.get_supported_kernel_block_sizes()
|
||||
),
|
||||
cache_config.block_size,
|
||||
)
|
||||
if preferred != CacheConfig.DEFAULT_BLOCK_SIZE:
|
||||
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
# With prefix caching, align to mamba chunk size for kernel perf
|
||||
# TODO(tdoublep): this constraint can be relaxed fairly
|
||||
# easily by changing the way we layout chunks in the
|
||||
# mamba2 kernels.
|
||||
base_chunk_size = mamba_block_size or model_config.get_mamba_chunk_size()
|
||||
assert base_chunk_size is not None
|
||||
attn_tokens_per_mamba_state = cdiv(mamba_page_size, attn_page_size_1_token)
|
||||
chunk_size = lcm(base_chunk_size, kernel_block_alignment_size)
|
||||
attn_block_size = chunk_size * cdiv(attn_tokens_per_mamba_state, chunk_size)
|
||||
cache_config.mamba_block_size = attn_block_size
|
||||
else:
|
||||
# Without prefix caching, use minimum block size that satisfies
|
||||
# both backend alignment and mamba page size compatibility
|
||||
attn_block_size = kernel_block_alignment_size * cdiv(
|
||||
mamba_page_size,
|
||||
kernel_block_alignment_size * attn_page_size_1_token,
|
||||
)
|
||||
|
||||
if cache_config.block_size < attn_block_size:
|
||||
cache_config.block_size = attn_block_size
|
||||
logger.info(
|
||||
"Setting kv cache block size to %d for %s backend.",
|
||||
preferred,
|
||||
backend_cls.get_name(),
|
||||
"Setting attention block size to %d tokens "
|
||||
"to ensure that attention page size is >= mamba page size.",
|
||||
attn_block_size,
|
||||
)
|
||||
|
||||
if cache_config.mamba_cache_mode == "align":
|
||||
cache_config.mamba_block_size = cache_config.block_size
|
||||
|
||||
# Pad mamba page size to exactly match attention page size
|
||||
attn_page_size = cache_config.block_size * attn_page_size_1_token
|
||||
assert attn_page_size >= mamba_page_size
|
||||
|
||||
if attn_page_size == mamba_page_size:
|
||||
return
|
||||
|
||||
if (
|
||||
cache_config.mamba_page_size_padded is None
|
||||
or cache_config.mamba_page_size_padded != attn_page_size
|
||||
):
|
||||
cache_config.mamba_page_size_padded = attn_page_size
|
||||
mamba_padding_pct = (
|
||||
100 * (attn_page_size - mamba_page_size) / mamba_page_size
|
||||
)
|
||||
logger.info(
|
||||
"Padding mamba page size by %.2f%% to ensure "
|
||||
"that mamba page size and attention page size are "
|
||||
"exactly equal.",
|
||||
mamba_padding_pct,
|
||||
)
|
||||
cache_config.block_size = preferred
|
||||
|
||||
@classmethod
|
||||
def verify_model_arch(cls, model_arch: str) -> None:
|
||||
|
||||
Reference in New Issue
Block a user