Support nvfp4 kv with kv-cache-dtype-skip-layers sliding_window (#42890)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ab3b6d97aa
commit
67ff0ae30f
@@ -629,6 +629,123 @@ class Platform:
|
||||
if model_config.is_hybrid:
|
||||
cls._align_hybrid_block_size(vllm_config, backend_cls)
|
||||
|
||||
# Phase 3: Align block/page sizes when multiple KV dtypes share the
|
||||
# block pool (e.g. nvfp4 primary + unquantized skip layers).
|
||||
# May override the user's --block-size.
|
||||
if cache_config.kv_cache_dtype_skip_layers:
|
||||
cls._align_heterogeneous_kv_block_size(vllm_config, backend_cls)
|
||||
|
||||
@classmethod
|
||||
def _align_heterogeneous_kv_block_size(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
backend_cls: "type[AttentionBackend]",
|
||||
) -> None:
|
||||
"""Align block size when several KV dtypes share one block pool.
|
||||
|
||||
A quantized primary (e.g. nvfp4) shares the block pool with one or more
|
||||
higher-precision "padded specs" (skip layers today; the first/last-N
|
||||
sibling in the future). A padded spec's per-token page is larger than
|
||||
the primary's *and not an integer multiple of it*, so the trivial
|
||||
``unify_kv_cache_spec_page_size`` cannot reconcile them. We do it here
|
||||
instead, before the specs are built:
|
||||
|
||||
1. Bump the primary ``block_size`` (kernel-aligned) until the primary
|
||||
page is large enough to cover the largest padded-spec page.
|
||||
2. Record that shared page in each padded spec's ``*_page_size_padded``
|
||||
hint, so it pads up to the shared page.
|
||||
|
||||
``unify`` then sees equal pages and stays trivial.
|
||||
|
||||
To add a padded-spec type: append its per-token page to ``padded_pages``
|
||||
and set its ``*_page_size_padded`` hint below.
|
||||
"""
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, get_kv_quant_mode
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
model_config = vllm_config.model_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if not model_config:
|
||||
return
|
||||
|
||||
def per_token_page_bytes(dtype: "torch.dtype", cache_dtype: str) -> int:
|
||||
"""Bytes one token occupies in one layer, for the given dtype."""
|
||||
return FullAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(cache_dtype),
|
||||
).page_size_bytes
|
||||
|
||||
primary_dtype = (
|
||||
STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
|
||||
if cache_config.cache_dtype != "auto"
|
||||
else model_config.dtype
|
||||
)
|
||||
primary_page = per_token_page_bytes(primary_dtype, cache_config.cache_dtype)
|
||||
|
||||
# Per-token page of every higher-precision padded spec sharing the pool.
|
||||
padded_pages: list[int] = []
|
||||
if cache_config.kv_cache_dtype_skip_layers:
|
||||
padded_pages.append(per_token_page_bytes(model_config.dtype, "auto"))
|
||||
# To add the first/last-N sibling:
|
||||
# padded_pages.append(per_token_page_bytes(<sibling_dtype>, "auto"))
|
||||
if not padded_pages:
|
||||
return
|
||||
|
||||
largest_padded_page = max(padded_pages)
|
||||
assert largest_padded_page >= primary_page, (
|
||||
f"padded-spec per-token page ({largest_padded_page}B) < primary "
|
||||
f"({primary_page}B); a higher-precision padded spec must not be "
|
||||
"smaller than the quantized primary."
|
||||
)
|
||||
if largest_padded_page == primary_page:
|
||||
# Pages already match per token; ``unify`` reconciles the differing
|
||||
# block sizes by integer scaling, so no bump or padding is needed.
|
||||
return
|
||||
|
||||
# Smallest block the kernel supports, and the granularity the primary
|
||||
# block is rounded up to (never below the already-chosen block_size).
|
||||
with set_current_vllm_config(vllm_config):
|
||||
supported = backend_cls.get_supported_kernel_block_sizes()
|
||||
smallest_kernel_block = min(
|
||||
s.base if isinstance(s, MultipleOf) else s for s in supported
|
||||
)
|
||||
block_alignment = max(smallest_kernel_block, cache_config.block_size)
|
||||
|
||||
# Bytes one padded-spec page spans at its own smallest kernel block;
|
||||
# also cover any mamba page a hybrid model already padded.
|
||||
required_page = max(
|
||||
largest_padded_page * smallest_kernel_block,
|
||||
cache_config.mamba_page_size_padded or 0,
|
||||
)
|
||||
|
||||
# Smallest kernel-aligned primary block whose page covers required_page.
|
||||
primary_block_size = block_alignment * cdiv(
|
||||
required_page, block_alignment * primary_page
|
||||
)
|
||||
if cache_config.block_size < primary_block_size:
|
||||
cache_config.block_size = primary_block_size
|
||||
logger.info(
|
||||
"Setting attention block size to %d tokens so the quantized "
|
||||
"primary KV page covers the higher-precision padded-spec page.",
|
||||
primary_block_size,
|
||||
)
|
||||
|
||||
# The shared page that every padded spec (and mamba) pads up to.
|
||||
shared_page = cache_config.block_size * primary_page
|
||||
if cache_config.kv_cache_dtype_skip_layers:
|
||||
cache_config.skip_page_size_padded = shared_page
|
||||
# To add the first/last-N sibling:
|
||||
# cache_config.sibling_page_size_padded = shared_page
|
||||
if cache_config.mamba_page_size_padded is not None:
|
||||
cache_config.mamba_page_size_padded = shared_page
|
||||
|
||||
@classmethod
|
||||
def _align_hybrid_block_size(
|
||||
cls,
|
||||
|
||||
Reference in New Issue
Block a user