[Perf] Integrate FlashInfer KDA kernels (#55364)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
Wei Zhao
2026-09-07 20:29:30 -07:00
committed by GitHub
parent 12579c7f01
commit a69402aaa8
8 changed files with 723 additions and 156 deletions
+284 -36
View File
@@ -50,6 +50,12 @@ from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
from vllm.triton_utils import tl, triton
from vllm.utils.flashinfer import (
flashinfer_fused_kda_decode,
flashinfer_recurrent_kda,
has_flashinfer_fused_kda_decode,
has_flashinfer_recurrent_kda,
)
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import MambaSpec
@@ -155,6 +161,7 @@ def is_fused_kda_decode_supported(
num_spec: int,
input_dtype: torch.dtype,
conv_state_dtype: torch.dtype,
recurrent_state_dtype: torch.dtype,
) -> bool:
# The fused kernel handles both conv-state cache layouts (SD and DS); the
# inner strides are selected from the tensor at launch time.
@@ -165,6 +172,7 @@ def is_fused_kda_decode_supported(
or num_spec != 0
or input_dtype != torch.bfloat16
or conv_state_dtype != torch.bfloat16
or recurrent_state_dtype != torch.float32
or not hasattr(torch.ops._C, "fused_kda_decode")
):
return False
@@ -176,9 +184,93 @@ def is_fused_kda_decode_supported(
)
def is_flashinfer_fused_kda_decode_supported(
num_heads: int,
head_dim: int,
conv_width: int,
num_spec: int,
input_dtype: torch.dtype,
conv_state_dtype: torch.dtype,
recurrent_state_dtype: torch.dtype,
) -> bool:
if not has_flashinfer_fused_kda_decode():
return False
capability = current_platform.get_device_capability()
if capability is None:
return False
compute_capability = (capability.major, capability.minor)
return (
compute_capability in ((10, 0), (10, 3))
and num_heads in (12, 24, 32, 48, 96)
and head_dim == 128
and conv_width == 4
and num_spec == 0
and input_dtype == torch.bfloat16
and conv_state_dtype == torch.bfloat16
and recurrent_state_dtype in (torch.float32, torch.bfloat16)
and not is_conv_state_dim_first()
)
def resolve_kda_decode_backend(
backend: str,
num_heads: int,
head_dim: int,
conv_width: int,
num_spec: int,
input_dtype: torch.dtype,
conv_state_dtype: torch.dtype,
recurrent_state_dtype: torch.dtype,
) -> str:
if backend not in ("auto", "native", "flashinfer", "triton"):
raise ValueError(f"Unsupported KDA decode backend: {backend}")
if backend == "triton":
return "triton"
native_supported = is_fused_kda_decode_supported(
num_heads,
head_dim,
conv_width,
num_spec,
input_dtype,
conv_state_dtype,
recurrent_state_dtype,
)
if native_supported and backend in ("auto", "native"):
logger.info_once("Using native fused KDA decode backend.")
return "native"
if backend == "native":
raise RuntimeError(
"Native fused KDA decode requires a supported CUDA architecture, "
"bfloat16 activations and convolution state, float32 recurrent "
"state, head_dim=128, convolution width 4, and no speculation."
)
flashinfer_supported = is_flashinfer_fused_kda_decode_supported(
num_heads,
head_dim,
conv_width,
num_spec,
input_dtype,
conv_state_dtype,
recurrent_state_dtype,
)
if flashinfer_supported and backend == "flashinfer":
logger.info_once("Using FlashInfer fused KDA decode backend.")
return "flashinfer"
if backend == "flashinfer":
raise RuntimeError(
"FlashInfer fused KDA decode requires CUDA SM100 or SM103, "
"bfloat16 activations and convolution state, bfloat16 or float32 "
"recurrent state, head_dim=128, convolution width 4, no speculation."
)
return "triton"
def is_flashkda_supported(
head_dim: int,
dtype: torch.dtype,
input_dtype: torch.dtype,
recurrent_state_dtype: torch.dtype,
lower_bound: float | None,
) -> bool:
if not current_platform.is_cuda():
@@ -188,7 +280,29 @@ def is_flashkda_supported(
capability is not None
and capability.major in (9, 10, 12)
and head_dim == 128
and dtype == torch.bfloat16
and input_dtype == torch.bfloat16
and recurrent_state_dtype in (torch.bfloat16, torch.float32)
and lower_bound is not None
)
def is_flashinfer_recurrent_kda_prefill_supported(
head_dim: int,
input_dtype: torch.dtype,
recurrent_state_dtype: torch.dtype,
lower_bound: float | None,
) -> bool:
if not current_platform.is_cuda() or not has_flashinfer_recurrent_kda():
return False
capability = current_platform.get_device_capability()
if capability is None:
return False
compute_capability = (capability.major, capability.minor)
return (
compute_capability in ((10, 0), (10, 3))
and head_dim == 128
and input_dtype == torch.bfloat16
and recurrent_state_dtype == torch.bfloat16
and lower_bound is not None
)
@@ -237,6 +351,44 @@ def _flashkda_prefill(
return out, final_state
def _flashinfer_kda_prefill(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
raw_g: torch.Tensor,
raw_beta: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor,
lower_bound: float,
initial_state: torch.Tensor,
cu_seqlens: torch.Tensor,
out: torch.Tensor,
seq_order: torch.Tensor | None = None,
prefill_workspace: object | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
output, _ = flashinfer_recurrent_kda(
q=q.contiguous(),
k=k.contiguous(),
v=v.contiguous(),
g=raw_g.contiguous(),
beta=raw_beta.contiguous(),
A_log=A_log.contiguous(),
dt_bias=dt_bias.contiguous(),
scale=q.shape[-1] ** -0.5,
initial_state=initial_state.contiguous(),
output_final_state=False,
use_qk_l2norm_in_kernel=True,
use_gate_in_kernel=True,
lower_bound=lower_bound,
cu_seqlens=cu_seqlens.contiguous(),
output=out,
beta_is_logit=True,
seq_order=seq_order,
prefill_workspace=prefill_workspace,
)
return output, initial_state
@triton.jit
def _store_cache_checkpoints_kernel(
x_ptr,
@@ -303,18 +455,40 @@ def _store_cache_checkpoints_kernel(
def resolve_kda_prefill_backend(
backend: str,
head_dim: int,
dtype: torch.dtype,
input_dtype: torch.dtype,
recurrent_state_dtype: torch.dtype,
lower_bound: float | None,
) -> str:
if backend not in ("auto", "triton", "flashkda"):
if backend not in ("auto", "triton", "flashkda", "flashinfer"):
raise ValueError(f"Unsupported KDA prefill backend: {backend}")
supported = is_flashkda_supported(head_dim, dtype, lower_bound)
if backend == "flashkda" and not supported:
flashinfer_supported = is_flashinfer_recurrent_kda_prefill_supported(
head_dim,
input_dtype,
recurrent_state_dtype,
lower_bound,
)
flashkda_supported = is_flashkda_supported(
head_dim,
input_dtype,
recurrent_state_dtype,
lower_bound,
)
if backend == "flashinfer" and not flashinfer_supported:
raise RuntimeError(
"FlashKDA requires CUDA SM90/SM10x/SM12x, bfloat16, "
"head_dim=128, and a bounded KDA gate."
"FlashInfer KDA prefill requires CUDA SM100 or SM103, bfloat16 "
"activations and recurrent state, head_dim=128, a bounded KDA "
"gate, and flashinfer-python 0.6.18 or newer."
)
if supported and backend != "triton":
if backend == "flashkda" and not flashkda_supported:
raise RuntimeError(
"FlashKDA requires CUDA SM90/SM10x/SM12x, bfloat16 input, "
"bfloat16 or float32 recurrent state, head_dim=128, and a bounded "
"KDA gate."
)
if flashinfer_supported and backend == "flashinfer":
logger.info_once("Using FlashInfer KDA prefill backend.")
return "flashinfer"
if flashkda_supported and backend in ("auto", "flashkda"):
logger.info_once("Using FlashKDA KDA prefill backend.")
return "flashkda"
return "triton"
@@ -369,7 +543,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
if self.model_config is None or self.cache_config is None:
raise ValueError("model_config and cache_config must be set")
base_dtypes = MambaStateDtypeCalculator.kda_state_dtype(
self.model_config.dtype, self.cache_config.mamba_cache_dtype
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
self.cache_config.mamba_ssm_cache_dtype,
)
if self.cache_config.use_kda_recoverssm:
return MambaStateDtypeCalculator.append_kda_recoverssm_record(
@@ -486,17 +662,25 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
# Keep a width-major copy for fused decode without changing the layout
# consumed by the prefill and fallback decode kernels.
conv_state_dtype = self.get_state_dtype()[0]
decode_conv1d_weight = None
if is_fused_kda_decode_supported(
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()[:2]
additional_config = vllm_config.additional_config
decode_backend = (
additional_config.get("kda_decode_backend", "auto")
if isinstance(additional_config, dict)
else "auto"
)
self.kda_decode_backend = resolve_kda_decode_backend(
decode_backend,
self.local_num_heads,
self.head_dim,
self.conv_size,
self.num_spec,
vllm_config.model_config.dtype,
conv_state_dtype,
):
logger.info_once("Fused KDA decode kernel (conv+KDA+norm) is enabled.")
recurrent_state_dtype,
)
decode_conv1d_weight = None
if self.kda_decode_backend != "triton":
decode_conv1d_weight = torch.empty(
3,
self.conv_size,
@@ -533,7 +717,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
f"Got {self.gate_lower_bound}."
)
additional_config = vllm_config.additional_config
backend = (
additional_config.get("kda_prefill_backend", "auto")
if isinstance(additional_config, dict)
@@ -543,11 +726,15 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
backend,
self.head_dim,
vllm_config.model_config.dtype,
recurrent_state_dtype,
self.gate_lower_bound,
)
self._flashkda_buffer_specs: (
tuple[tuple[tuple[int, ...], torch.dtype], ...] | None
) = None
self._flashinfer_kda_output_spec: tuple[tuple[int, ...], torch.dtype] | None = (
None
)
if self.kda_prefill_backend == "flashkda":
T = vllm_config.scheduler_config.max_num_batched_tokens
N = vllm_config.scheduler_config.max_num_seqs
@@ -561,6 +748,13 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
((N, H, D, D), self.get_state_dtype()[1]),
((workspace_size,), torch.uint8),
)
elif self.kda_prefill_backend == "flashinfer":
T = vllm_config.scheduler_config.max_num_batched_tokens
H, D = self.local_num_heads, self.head_dim
self._flashinfer_kda_output_spec = (
(1, T, H, D),
self.model_config.dtype,
)
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
decode_norm_weight = None
@@ -725,30 +919,50 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
conv_state = conv_state.transpose(-1, -2)
if (
self.decode_conv1d_weight is not None
self.kda_decode_backend != "triton"
and self.decode_conv1d_weight is not None
and self.decode_norm_weight is not None
and not has_spec_decode
and m.num_prefills == 0
and m.num_decodes > 0
):
assert non_spec_state_indices_tensor is not None
ops.fused_kda_decode(
x=mixed_qkv,
weight=self.decode_conv1d_weight,
bias=self.conv1d.bias,
conv_state=conv_state,
raw_g=g1,
raw_beta=beta,
A_log=self.A_log,
dt_bias=self.dt_bias,
state_indices=non_spec_state_indices_tensor[:num_actual_tokens],
state=recurrent_state,
out=core_attn_out[:, :num_actual_tokens],
lower_bound=self.gate_lower_bound,
output_gate=g2[:num_actual_tokens],
norm_weight=self.decode_norm_weight,
norm_eps=self.o_norm.eps,
)
state_indices = non_spec_state_indices_tensor[:num_actual_tokens]
if self.kda_decode_backend == "flashinfer":
flashinfer_fused_kda_decode(
x=mixed_qkv,
weight=self.decode_conv1d_weight,
conv_state=conv_state,
raw_gate=g1,
raw_beta=beta,
A_log=self.A_log,
dt_bias=self.dt_bias,
state_indices=state_indices,
state=recurrent_state,
output_gate=g2[:num_actual_tokens],
norm_weight=self.decode_norm_weight,
lower_bound=self.gate_lower_bound,
norm_eps=self.o_norm.eps,
output=core_attn_out[:, :num_actual_tokens],
)
else:
ops.fused_kda_decode(
x=mixed_qkv,
weight=self.decode_conv1d_weight,
bias=self.conv1d.bias,
conv_state=conv_state,
raw_g=g1,
raw_beta=beta,
A_log=self.A_log,
dt_bias=self.dt_bias,
state_indices=state_indices,
state=recurrent_state,
out=core_attn_out[:, :num_actual_tokens],
lower_bound=self.gate_lower_bound,
output_gate=g2[:num_actual_tokens],
norm_weight=self.decode_norm_weight,
norm_eps=self.o_norm.eps,
)
return
conv_weights = self.conv1d.weight.view(
@@ -991,6 +1205,35 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
final_state=final_state[: initial_state.shape[0]],
workspace=workspace,
)
elif self.kda_prefill_backend == "flashinfer":
assert self.gate_lower_bound is not None
assert m.flashinfer_prefill_query_start_loc is not None
if q_ns.shape[1] > initial_state.shape[0]:
assert m.flashinfer_prefill_seq_order is not None
flashinfer_out = core_attn_out[:, : q_ns.shape[1]]
if has_spec_decode:
assert self._flashinfer_kda_output_spec is not None
(workspace_out,) = current_workspace_manager().get_simultaneous(
self._flashinfer_kda_output_spec
)
flashinfer_out = workspace_out[:, : q_ns.shape[1]]
(
core_attn_out_non_spec,
last_recurrent_state,
) = _flashinfer_kda_prefill(
q=q_ns,
k=k_ns,
v=v_ns,
raw_g=g1_ns,
raw_beta=beta_ns,
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=initial_state,
cu_seqlens=m.flashinfer_prefill_query_start_loc,
out=flashinfer_out,
seq_order=m.flashinfer_prefill_seq_order,
)
else:
(
core_attn_out_non_spec,
@@ -1009,7 +1252,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
use_qk_l2norm_in_kernel=True,
cu_seqlens=non_spec_query_start_loc,
)
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
recurrent_state[non_spec_state_indices_tensor] = (
last_recurrent_state.to(recurrent_state.dtype)
)
else:
# Pure non-speculative decode.
assert non_spec_state_indices_tensor is not None
@@ -1046,7 +1291,10 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
core_attn_out.index_copy_(1, spec_token_indx, core_attn_out_spec)
core_attn_out.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
elif core_attn_out_non_spec is not None:
if self.kda_prefill_backend != "flashkda" or m.num_prefills == 0:
if (
self.kda_prefill_backend not in ("flashkda", "flashinfer")
or m.num_prefills == 0
):
# TODO: decode kernels write directly to core_attn_out
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
0, :num_actual_tokens
@@ -270,6 +270,8 @@ class KDACheckpointMetadata:
@dataclass
class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
flashinfer_prefill_query_start_loc: torch.Tensor | None = None
flashinfer_prefill_seq_order: torch.Tensor | None = None
recoverssm_commit: KDARecoverSSMCommitMetadata | None = None
recoverssm_context: "KDARecoverSSMCommitContext | None" = field(
default=None, repr=False, compare=False
@@ -318,6 +320,11 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
device: torch.device,
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
additional_config = vllm_config.additional_config
self.use_flashinfer_prefill = (
isinstance(additional_config, dict)
and additional_config.get("kda_prefill_backend") == "flashinfer"
)
self.use_recoverssm = vllm_config.cache_config.use_kda_recoverssm
self.spec_state_slots = 1 if self.use_recoverssm else self.num_spec + 1
self.recoverssm_num_accepted_tokens: torch.Tensor | None = None
@@ -721,6 +728,20 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
align=align,
)
flashinfer_prefill_query_start_loc = None
flashinfer_prefill_seq_order = None
if self.use_flashinfer_prefill and num_prefills > 0:
assert non_spec_query_start_loc is not None
flashinfer_prefill_query_start_loc = non_spec_query_start_loc.to(
torch.int64
)
num_non_spec_requests = non_spec_query_start_loc.shape[0] - 1
num_non_spec_tokens = num_prefill_tokens + num_decode_tokens
if num_non_spec_tokens > num_non_spec_requests:
flashinfer_prefill_seq_order = torch.argsort(
flashinfer_prefill_query_start_loc.diff(), descending=True
).to(torch.int32)
return KimiK3KDAMetadata(
num_prefills=num_prefills,
num_prefill_tokens=num_prefill_tokens,
@@ -738,6 +759,8 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
spec_token_indx=spec_token_indx,
non_spec_token_indx=non_spec_token_indx,
num_accepted_tokens=num_accepted_tokens,
flashinfer_prefill_query_start_loc=flashinfer_prefill_query_start_loc,
flashinfer_prefill_seq_order=flashinfer_prefill_seq_order,
recoverssm_commit=recoverssm_commit,
recoverssm_context=(
self._get_recoverssm_context()
+3 -1
View File
@@ -1649,7 +1649,9 @@ class KimiLinearForCausalLM(
vllm_config: "VllmConfig",
) -> tuple[torch.dtype, ...]:
dtypes = MambaStateDtypeCalculator.kda_state_dtype(
vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype
vllm_config.model_config.dtype,
vllm_config.cache_config.mamba_cache_dtype,
vllm_config.cache_config.mamba_ssm_cache_dtype,
)
if vllm_config.cache_config.use_kda_recoverssm:
dtypes = MambaStateDtypeCalculator.append_kda_recoverssm_record(