[Kernel] Resubmit PR 48666 - Gemma4 FP8 KV FA4 head dim 512 backend selection (#53175)
Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
This commit is contained in:
@@ -8,7 +8,10 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, get_current_vllm_config
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.config.vllm import VllmConfig
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -100,6 +103,7 @@ def _largest_kernel_block_within(
|
||||
per_token_bytes: int,
|
||||
page_budget: int,
|
||||
fallback: int,
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
) -> int:
|
||||
"""Largest supported kernel block size whose page fits in ``page_budget``.
|
||||
|
||||
@@ -111,7 +115,7 @@ def _largest_kernel_block_within(
|
||||
"""
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
sizes = attn_backend.get_supported_kernel_block_sizes()
|
||||
sizes = attn_backend.get_supported_kernel_block_sizes(kv_cache_spec)
|
||||
max_block_size = page_budget // per_token_bytes
|
||||
candidates = [s for s in sizes if isinstance(s, int)]
|
||||
candidates.extend(
|
||||
@@ -632,7 +636,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
# ``unify`` scales it up by an integer ratio.
|
||||
shared_page = vllm_config.cache_config.skip_page_size_padded
|
||||
# The backend owns its packing
|
||||
sw_per_token = self.attn_backend.customize_spec(
|
||||
kv_cache_spec = self.attn_backend.customize_spec(
|
||||
SlidingWindowSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
@@ -642,10 +646,15 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
).real_page_size_bytes
|
||||
)
|
||||
sw_per_token = kv_cache_spec.real_page_size_bytes
|
||||
page_budget = shared_page or sw_per_token * block_size
|
||||
sw_block_size = _largest_kernel_block_within(
|
||||
self.attn_backend, sw_per_token, page_budget, block_size
|
||||
self.attn_backend,
|
||||
sw_per_token,
|
||||
page_budget,
|
||||
block_size,
|
||||
kv_cache_spec,
|
||||
)
|
||||
return SlidingWindowSpec(
|
||||
block_size=sw_block_size,
|
||||
|
||||
Reference in New Issue
Block a user