Support nvfp4 kv with kv-cache-dtype-skip-layers sliding_window (#42890)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
sychen52
2026-07-04 02:29:13 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ab3b6d97aa
commit 67ff0ae30f
6 changed files with 228 additions and 5 deletions
+117
View File
@@ -629,6 +629,123 @@ class Platform:
if model_config.is_hybrid:
cls._align_hybrid_block_size(vllm_config, backend_cls)
# Phase 3: Align block/page sizes when multiple KV dtypes share the
# block pool (e.g. nvfp4 primary + unquantized skip layers).
# May override the user's --block-size.
if cache_config.kv_cache_dtype_skip_layers:
cls._align_heterogeneous_kv_block_size(vllm_config, backend_cls)
@classmethod
def _align_heterogeneous_kv_block_size(
cls,
vllm_config: "VllmConfig",
backend_cls: "type[AttentionBackend]",
) -> None:
"""Align block size when several KV dtypes share one block pool.
A quantized primary (e.g. nvfp4) shares the block pool with one or more
higher-precision "padded specs" (skip layers today; the first/last-N
sibling in the future). A padded spec's per-token page is larger than
the primary's *and not an integer multiple of it*, so the trivial
``unify_kv_cache_spec_page_size`` cannot reconcile them. We do it here
instead, before the specs are built:
1. Bump the primary ``block_size`` (kernel-aligned) until the primary
page is large enough to cover the largest padded-spec page.
2. Record that shared page in each padded spec's ``*_page_size_padded``
hint, so it pads up to the shared page.
``unify`` then sees equal pages and stays trivial.
To add a padded-spec type: append its per-token page to ``padded_pages``
and set its ``*_page_size_padded`` hint below.
"""
from vllm.config.vllm import set_current_vllm_config
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.attention.backend import MultipleOf
from vllm.v1.kv_cache_interface import FullAttentionSpec, get_kv_quant_mode
cache_config = vllm_config.cache_config
model_config = vllm_config.model_config
parallel_config = vllm_config.parallel_config
if not model_config:
return
def per_token_page_bytes(dtype: "torch.dtype", cache_dtype: str) -> int:
"""Bytes one token occupies in one layer, for the given dtype."""
return FullAttentionSpec(
block_size=1,
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
dtype=dtype,
kv_quant_mode=get_kv_quant_mode(cache_dtype),
).page_size_bytes
primary_dtype = (
STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
if cache_config.cache_dtype != "auto"
else model_config.dtype
)
primary_page = per_token_page_bytes(primary_dtype, cache_config.cache_dtype)
# Per-token page of every higher-precision padded spec sharing the pool.
padded_pages: list[int] = []
if cache_config.kv_cache_dtype_skip_layers:
padded_pages.append(per_token_page_bytes(model_config.dtype, "auto"))
# To add the first/last-N sibling:
# padded_pages.append(per_token_page_bytes(<sibling_dtype>, "auto"))
if not padded_pages:
return
largest_padded_page = max(padded_pages)
assert largest_padded_page >= primary_page, (
f"padded-spec per-token page ({largest_padded_page}B) < primary "
f"({primary_page}B); a higher-precision padded spec must not be "
"smaller than the quantized primary."
)
if largest_padded_page == primary_page:
# Pages already match per token; ``unify`` reconciles the differing
# block sizes by integer scaling, so no bump or padding is needed.
return
# Smallest block the kernel supports, and the granularity the primary
# block is rounded up to (never below the already-chosen block_size).
with set_current_vllm_config(vllm_config):
supported = backend_cls.get_supported_kernel_block_sizes()
smallest_kernel_block = min(
s.base if isinstance(s, MultipleOf) else s for s in supported
)
block_alignment = max(smallest_kernel_block, cache_config.block_size)
# Bytes one padded-spec page spans at its own smallest kernel block;
# also cover any mamba page a hybrid model already padded.
required_page = max(
largest_padded_page * smallest_kernel_block,
cache_config.mamba_page_size_padded or 0,
)
# Smallest kernel-aligned primary block whose page covers required_page.
primary_block_size = block_alignment * cdiv(
required_page, block_alignment * primary_page
)
if cache_config.block_size < primary_block_size:
cache_config.block_size = primary_block_size
logger.info(
"Setting attention block size to %d tokens so the quantized "
"primary KV page covers the higher-precision padded-spec page.",
primary_block_size,
)
# The shared page that every padded spec (and mamba) pads up to.
shared_page = cache_config.block_size * primary_page
if cache_config.kv_cache_dtype_skip_layers:
cache_config.skip_page_size_padded = shared_page
# To add the first/last-N sibling:
# cache_config.sibling_page_size_padded = shared_page
if cache_config.mamba_page_size_padded is not None:
cache_config.mamba_page_size_padded = shared_page
@classmethod
def _align_hybrid_block_size(
cls,