Support nvfp4 kv with kv-cache-dtype-skip-layers sliding_window (#42890)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
sychen52
2026-07-04 02:29:13 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ab3b6d97aa
commit 67ff0ae30f
6 changed files with 228 additions and 5 deletions
@@ -92,6 +92,35 @@ def should_load_quant_weights(quant_method: QuantizeMethodBase | None) -> bool:
)
def _largest_kernel_block_within(
attn_backend: "type[AttentionBackend]",
per_token_bytes: int,
page_budget: int | None,
fallback: int,
) -> int:
"""Largest supported kernel block size whose page fits in ``page_budget``.
A padded spec (e.g. skip-quant layer) that pads its page up to a large shared page
wastes ``page_budget - block*per_token`` bytes per block. Picking the largest kernel
block whose natural page still fits under ``page_budget`` minimizes that waste.
Falls back to the smallest supported block when ``page_budget`` is None (no padding
— the block is handled by ``unify``'s integer scaling instead) or nothing fits.
"""
from vllm.v1.attention.backend import MultipleOf
sizes = attn_backend.get_supported_kernel_block_sizes()
candidates = [s for s in sizes if isinstance(s, int)]
if not candidates:
candidates = [s.base for s in sizes if isinstance(s, MultipleOf)]
if not candidates:
return fallback
smallest = min(candidates)
if not page_budget or per_token_bytes <= 0:
return smallest
fitting = [b for b in candidates if b * per_token_bytes <= page_budget]
return max(fitting) if fitting else smallest
def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) -> None:
"""Sets default quantization scales for the layer."""
if register_buffer:
@@ -601,14 +630,35 @@ class Attention(nn.Module, AttentionLayerBase):
assert not vllm_config.model_config.use_mla, (
"MLA is not supported for slidingwindow"
)
return SlidingWindowSpec(
block_size=block_size,
# SW chooses its own block_size, decoupled from the user's
# ``--block-size`` (which only constrains primary attention).
# When this SW layer is a padded spec (skip-quant: its page is
# padded up to ``skip_page_size_padded``), pick the largest kernel
# block that still fits the shared page so we waste fewer padding
# bytes per block. Otherwise (page_size_padded is None) the smallest
# block is fine — ``unify`` scales it up by an integer ratio.
shared_page = vllm_config.cache_config.skip_page_size_padded
sw_per_token = SlidingWindowSpec(
block_size=1,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size_v,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
).real_page_size_bytes
sw_block_size = _largest_kernel_block_within(
self.attn_backend, sw_per_token, shared_page, block_size
)
return SlidingWindowSpec(
block_size=sw_block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size_v,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
page_size_padded=shared_page,
)
elif self.kv_cache_dtype.startswith("turboquant_"):
from vllm.model_executor.layers.quantization.turboquant.config import (