[Kernel] Gemma-4 FA4 FP8 Kernel (#48666)

Signed-off-by: Qixiang Lin <qixiangl@nvidia.com>
Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
Co-authored-by: Qixiang Lin <qixiangl@nvidia.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Jhao-Ting Chen
2026-08-13 17:42:00 -07:00
committed by GitHub
co-authored by Qixiang Lin Claude Opus 4.8 Yongye Zhu
parent 38f097fab8
commit 827a2af806
8 changed files with 152 additions and 29 deletions
@@ -8,7 +8,10 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, get_current_vllm_config
from vllm.config import (
CacheConfig,
get_current_vllm_config,
)
from vllm.config.vllm import VllmConfig
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
@@ -94,6 +97,7 @@ def should_load_quant_weights(quant_method: QuantizeMethodBase | None) -> bool:
def _largest_kernel_block_within(
attn_backend: "type[AttentionBackend]",
vllm_config: VllmConfig,
per_token_bytes: int,
page_budget: int | None,
fallback: int,
@@ -108,7 +112,7 @@ def _largest_kernel_block_within(
"""
from vllm.v1.attention.backend import MultipleOf
sizes = attn_backend.get_supported_kernel_block_sizes()
sizes = attn_backend.get_supported_kernel_block_sizes_for_config(vllm_config)
candidates = [s for s in sizes if isinstance(s, int)]
if not candidates:
candidates = [s.base for s in sizes if isinstance(s, MultipleOf)]
@@ -629,7 +633,11 @@ class Attention(nn.Module, AttentionLayerBase):
sliding_window=self.sliding_window,
).real_page_size_bytes
sw_block_size = _largest_kernel_block_within(
self.attn_backend, sw_per_token, shared_page, block_size
self.attn_backend,
vllm_config,
sw_per_token,
shared_page,
block_size,
)
return SlidingWindowSpec(
block_size=sw_block_size,