[Kernel] Resubmit PR 48666 - Gemma4 FP8 KV FA4 head dim 512 backend selection (#53175)

Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
This commit is contained in:
Jhao-Ting Chen
2026-09-23 20:29:06 -07:00
committed by GitHub
parent e78e367c6e
commit 4fb767ff0a
47 changed files with 241 additions and 103 deletions
@@ -8,7 +8,10 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, get_current_vllm_config
from vllm.config import (
CacheConfig,
get_current_vllm_config,
)
from vllm.config.vllm import VllmConfig
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
@@ -100,6 +103,7 @@ def _largest_kernel_block_within(
per_token_bytes: int,
page_budget: int,
fallback: int,
kv_cache_spec: KVCacheSpec | None = None,
) -> int:
"""Largest supported kernel block size whose page fits in ``page_budget``.
@@ -111,7 +115,7 @@ def _largest_kernel_block_within(
"""
from vllm.v1.attention.backend import MultipleOf
sizes = attn_backend.get_supported_kernel_block_sizes()
sizes = attn_backend.get_supported_kernel_block_sizes(kv_cache_spec)
max_block_size = page_budget // per_token_bytes
candidates = [s for s in sizes if isinstance(s, int)]
candidates.extend(
@@ -632,7 +636,7 @@ class Attention(nn.Module, AttentionLayerBase):
# ``unify`` scales it up by an integer ratio.
shared_page = vllm_config.cache_config.skip_page_size_padded
# The backend owns its packing
sw_per_token = self.attn_backend.customize_spec(
kv_cache_spec = self.attn_backend.customize_spec(
SlidingWindowSpec(
block_size=1,
num_kv_heads=self.num_kv_heads,
@@ -642,10 +646,15 @@ class Attention(nn.Module, AttentionLayerBase):
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
)
).real_page_size_bytes
)
sw_per_token = kv_cache_spec.real_page_size_bytes
page_budget = shared_page or sw_per_token * block_size
sw_block_size = _largest_kernel_block_within(
self.attn_backend, sw_per_token, page_budget, block_size
self.attn_backend,
sw_per_token,
page_budget,
block_size,
kv_cache_spec,
)
return SlidingWindowSpec(
block_size=sw_block_size,