[GDN] Enable FI Blackwell GDN prefill kernel (#40717)

Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
Artem Perevedentsev
2026-05-20 01:46:55 -07:00
committed by GitHub
parent 9b343dd4f5
commit 1cb224430b
2 changed files with 58 additions and 27 deletions
@@ -83,6 +83,49 @@ if GDN_AITER_TRITON_AVAILABLE:
logger = init_logger(__name__)
def _should_use_flashinfer_gdn_prefill(backend: str, head_k_dim: int | None) -> bool:
"""Whether to use FlashInfer's GDN prefill kernel instead of the
Triton/FLA fallback.
Requirements:
* ``requested in ["flashinfer", "auto"]``;
* ``platform == cuda``;
* one of the following:
- Hopper (SM90) — no further constraints;
- Blackwell (SM10.x) with ``head_k_dim == 128`` and ``cuda_runtime >= 13``.
"""
if backend not in ["flashinfer", "auto"]:
return False
if not current_platform.is_cuda():
return False
if current_platform.is_device_capability(90):
return True # Hopper — no further constraints.
if not current_platform.is_device_capability_family(100):
return False # Neither Hopper nor Blackwell.
if head_k_dim != 128:
return False
return current_platform.get_cuda_runtime_major() >= 13
def _log_gdn_backend_decision(
backend: str, head_k_dim: int | None, use_flashinfer: bool
) -> None:
"""Log the GDN prefill backend choice in the attention-selector style."""
chosen = "FlashInfer" if use_flashinfer else "Triton/FLA"
logger.info_once(
"Using %s GDN prefill kernel (requested=%s, head_k_dim=%s).",
chosen,
backend,
head_k_dim,
)
# JIT-compiled cutlass path is only used on SM90 (Hopper).
if use_flashinfer and current_platform.is_device_capability(90):
logger.warning_once(
"FlashInfer GDN prefill is JIT-compiled; first run may take a "
"while. Set --gdn-prefill-backend triton to skip JIT.",
)
def fi_chunk_gated_delta_rule(
q: torch.Tensor,
k: torch.Tensor,
@@ -134,39 +177,21 @@ def fi_chunk_gated_delta_rule(
@CustomOp.register("chunk_gated_delta_rule")
class ChunkGatedDeltaRule(CustomOp):
def __init__(self) -> None:
def __init__(self, head_k_dim: int | None = None) -> None:
super().__init__()
additional_config = get_current_vllm_config().additional_config
assert isinstance(additional_config, dict)
backend_cfg = additional_config.get("gdn_prefill_backend", "auto")
backend = str(backend_cfg).strip().lower()
supports_flashinfer = (
current_platform.is_cuda() and current_platform.is_device_capability(90)
)
if backend == "flashinfer":
use_flashinfer = supports_flashinfer
if not use_flashinfer:
logger.warning_once(
"GDN prefill backend 'flashinfer' is selected but "
"cannot use this kernel on the current platform. "
"Falling back to Triton/FLA."
)
elif backend == "triton":
use_flashinfer = False
else:
use_flashinfer = supports_flashinfer
if use_flashinfer:
logger.info_once("Using FlashInfer GDN prefill kernel")
logger.info_once(
"FlashInfer GDN prefill kernel is JIT-compiled; first run may "
"take a while to compile. Set `--gdn-prefill-backend triton` to "
"avoid JIT compile time.",
use_flashinfer = _should_use_flashinfer_gdn_prefill(backend, head_k_dim)
if backend == "flashinfer" and not use_flashinfer:
logger.warning_once(
"GDN prefill backend 'flashinfer' is selected but "
"cannot use this kernel on the current platform. "
"Falling back to Triton/FLA."
)
else:
logger.info_once("Using Triton/FLA GDN prefill kernel")
_log_gdn_backend_decision(backend, head_k_dim, use_flashinfer)
self._forward_method = (
self.forward_cuda if use_flashinfer else self.forward_native
@@ -396,7 +421,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
prefix=f"{prefix}.out_proj",
)
self.chunk_gated_delta_rule = ChunkGatedDeltaRule()
self.chunk_gated_delta_rule = ChunkGatedDeltaRule(head_k_dim=self.head_k_dim)
self._prefill_kernels_warmed_up = False
self.enable_packed_recurrent_decode = (
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
+6
View File
@@ -202,6 +202,12 @@ class CudaPlatformBase(Platform):
def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None:
raise NotImplementedError
@classmethod
def get_cuda_runtime_major(cls) -> int:
"""Major ``torch.version.cuda`` version, or ``0`` if undetermined."""
major = (torch.version.cuda or "0").split(".", 1)[0]
return int(major) if major.isdigit() else 0
@classmethod
def get_device_name(cls, device_id: int = 0) -> str:
raise NotImplementedError