Support nvfp4 kv with kv-cache-dtype-skip-layers sliding_window (#42890)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ab3b6d97aa
commit
67ff0ae30f
@@ -92,6 +92,35 @@ def should_load_quant_weights(quant_method: QuantizeMethodBase | None) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _largest_kernel_block_within(
|
||||
attn_backend: "type[AttentionBackend]",
|
||||
per_token_bytes: int,
|
||||
page_budget: int | None,
|
||||
fallback: int,
|
||||
) -> int:
|
||||
"""Largest supported kernel block size whose page fits in ``page_budget``.
|
||||
|
||||
A padded spec (e.g. skip-quant layer) that pads its page up to a large shared page
|
||||
wastes ``page_budget - block*per_token`` bytes per block. Picking the largest kernel
|
||||
block whose natural page still fits under ``page_budget`` minimizes that waste.
|
||||
Falls back to the smallest supported block when ``page_budget`` is None (no padding
|
||||
— the block is handled by ``unify``'s integer scaling instead) or nothing fits.
|
||||
"""
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
sizes = attn_backend.get_supported_kernel_block_sizes()
|
||||
candidates = [s for s in sizes if isinstance(s, int)]
|
||||
if not candidates:
|
||||
candidates = [s.base for s in sizes if isinstance(s, MultipleOf)]
|
||||
if not candidates:
|
||||
return fallback
|
||||
smallest = min(candidates)
|
||||
if not page_budget or per_token_bytes <= 0:
|
||||
return smallest
|
||||
fitting = [b for b in candidates if b * per_token_bytes <= page_budget]
|
||||
return max(fitting) if fitting else smallest
|
||||
|
||||
|
||||
def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) -> None:
|
||||
"""Sets default quantization scales for the layer."""
|
||||
if register_buffer:
|
||||
@@ -601,14 +630,35 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
assert not vllm_config.model_config.use_mla, (
|
||||
"MLA is not supported for slidingwindow"
|
||||
)
|
||||
return SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
# SW chooses its own block_size, decoupled from the user's
|
||||
# ``--block-size`` (which only constrains primary attention).
|
||||
# When this SW layer is a padded spec (skip-quant: its page is
|
||||
# padded up to ``skip_page_size_padded``), pick the largest kernel
|
||||
# block that still fits the shared page so we waste fewer padding
|
||||
# bytes per block. Otherwise (page_size_padded is None) the smallest
|
||||
# block is fine — ``unify`` scales it up by an integer ratio.
|
||||
shared_page = vllm_config.cache_config.skip_page_size_padded
|
||||
sw_per_token = SlidingWindowSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size_v,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
).real_page_size_bytes
|
||||
sw_block_size = _largest_kernel_block_within(
|
||||
self.attn_backend, sw_per_token, shared_page, block_size
|
||||
)
|
||||
return SlidingWindowSpec(
|
||||
block_size=sw_block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size_v,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
page_size_padded=shared_page,
|
||||
)
|
||||
elif self.kv_cache_dtype.startswith("turboquant_"):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
|
||||
Reference in New Issue
Block a user