[Kernel] Gemma-4 FA4 FP8 Kernel (#48666)
Signed-off-by: Qixiang Lin <qixiangl@nvidia.com> Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com> Co-authored-by: Qixiang Lin <qixiangl@nvidia.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Qixiang Lin
Claude Opus 4.8
Yongye Zhu
parent
38f097fab8
commit
827a2af806
@@ -8,7 +8,10 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, get_current_vllm_config
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.config.vllm import VllmConfig
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -94,6 +97,7 @@ def should_load_quant_weights(quant_method: QuantizeMethodBase | None) -> bool:
|
||||
|
||||
def _largest_kernel_block_within(
|
||||
attn_backend: "type[AttentionBackend]",
|
||||
vllm_config: VllmConfig,
|
||||
per_token_bytes: int,
|
||||
page_budget: int | None,
|
||||
fallback: int,
|
||||
@@ -108,7 +112,7 @@ def _largest_kernel_block_within(
|
||||
"""
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
sizes = attn_backend.get_supported_kernel_block_sizes()
|
||||
sizes = attn_backend.get_supported_kernel_block_sizes_for_config(vllm_config)
|
||||
candidates = [s for s in sizes if isinstance(s, int)]
|
||||
if not candidates:
|
||||
candidates = [s.base for s in sizes if isinstance(s, MultipleOf)]
|
||||
@@ -629,7 +633,11 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
sliding_window=self.sliding_window,
|
||||
).real_page_size_bytes
|
||||
sw_block_size = _largest_kernel_block_within(
|
||||
self.attn_backend, sw_per_token, shared_page, block_size
|
||||
self.attn_backend,
|
||||
vllm_config,
|
||||
sw_per_token,
|
||||
shared_page,
|
||||
block_size,
|
||||
)
|
||||
return SlidingWindowSpec(
|
||||
block_size=sw_block_size,
|
||||
|
||||
Reference in New Issue
Block a user