[Kimi-K3] Add FlashInfer speculative KDA backend (#54255)

Signed-off-by: Duncan Moss <dmoss@nvidia.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Duncan Moss
2026-10-01 02:28:50 +00:00
committed by GitHub
co-authored by Roger Wang mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Thien Tran
parent edaf3e7b61
commit 766b4ae54c
3 changed files with 383 additions and 9 deletions
+241
View File
@@ -30,13 +30,17 @@ from vllm.models.kimi_k3.amd.ops.third_party.kda import (
)
from vllm.models.kimi_k3.nvidia import kda as nvidia_kda
from vllm.models.kimi_k3.nvidia.kda import (
KimiK3DeltaAttention,
_flashinfer_kda_prefill,
_flashkda_prefill,
is_flashinfer_fused_kda_decode_supported,
is_flashinfer_fused_kda_spec_decode_supported,
is_flashinfer_recurrent_kda_prefill_supported,
is_flashkda_supported,
is_fused_kda_decode_supported,
resolve_kda_spec_decode_backend,
)
from vllm.models.kimi_k3.nvidia.kda_metadata import KimiK3KDAMetadata
from vllm.models.kimi_k3.nvidia.model import KimiLinearForCausalLM
from vllm.models.kimi_k3.nvidia.ops import recoverssm as recoverssm_ops
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
@@ -90,6 +94,25 @@ def test_kda_warmup_skips_missing_metadata(monkeypatch):
assert layer._forward(empty, empty, empty, empty, empty) is None
def test_resolve_kda_spec_decode_backend(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(
"vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported",
lambda *args: True,
)
args = (12, 128, 4, 6, torch.bfloat16, torch.bfloat16, torch.float32, -5.0)
assert resolve_kda_spec_decode_backend("auto", *args, False) == "flashinfer"
assert resolve_kda_spec_decode_backend("native", *args, False) == "native"
assert resolve_kda_spec_decode_backend("flashinfer", *args, False) == "flashinfer"
monkeypatch.setattr(
"vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported",
lambda *args: False,
)
assert resolve_kda_spec_decode_backend("auto", *args, False) == "native"
with pytest.raises(RuntimeError, match="packed_fused_kda_decode"):
resolve_kda_spec_decode_backend("flashinfer", *args, False)
def test_kda_recoverssm_config_state_layout():
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
@@ -1182,6 +1205,224 @@ def test_fused_kda_decode_correctness(
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
@torch.inference_mode()
@pytest.mark.parametrize("state_dtype", [torch.float32, torch.bfloat16])
def test_flashinfer_fused_kda_spec_decode_integration(
monkeypatch: pytest.MonkeyPatch,
state_dtype: torch.dtype,
):
if not is_flashinfer_fused_kda_spec_decode_supported(
num_heads=12,
head_dim=128,
conv_width=4,
num_spec=6,
input_dtype=torch.bfloat16,
conv_state_dtype=torch.bfloat16,
recurrent_state_dtype=state_dtype,
lower_bound=-5.0,
use_recoverssm=False,
):
pytest.skip("FlashInfer packed fused KDA decode is not supported")
generator = torch.Generator(device=DEVICE).manual_seed(20260820)
H, D, W, T = 12, 128, 4, 7
hidden_size = H * D
num_rows = 10
num_slots = 16
x = torch.randn(
num_rows,
3 * hidden_size,
dtype=torch.bfloat16,
device=DEVICE,
generator=generator,
)
conv_weight = 0.1 * torch.randn(
3 * hidden_size,
W,
dtype=torch.float32,
device=DEVICE,
generator=generator,
)
fused_weight = conv_weight.view(3, hidden_size, W).transpose(1, 2).contiguous()
raw_gate = torch.randn(
1, num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator
)
raw_beta = torch.randn(
1, num_rows, H, dtype=torch.bfloat16, device=DEVICE, generator=generator
)
output_gate = torch.randn(
num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator
)
A_log = torch.log(torch.arange(1, H + 1, dtype=torch.float32, device=DEVICE))
dt_bias = torch.empty(hidden_size, dtype=torch.float32, device=DEVICE).uniform_(
-7.0, -4.0, generator=generator
)
norm_weight = torch.randn(
D, dtype=torch.float32, device=DEVICE, generator=generator
)
norm_eps = 1e-5
state_indices = torch.tensor(
[
list(range(1, T + 1)),
list(range(T + 1, 2 * T + 1)),
[0] * T,
],
dtype=torch.int32,
device=DEVICE,
)
query_start_loc = torch.tensor([0, 7, 10, 10], dtype=torch.int32, device=DEVICE)
num_accepted_tokens = torch.tensor([1, 4, 1], dtype=torch.int32, device=DEVICE)
conv_seed = (
0.5
* torch.randn(
num_slots,
T + W - 2,
3 * hidden_size,
dtype=torch.bfloat16,
device=DEVICE,
generator=generator,
)
).transpose(1, 2)
state_seed = (
0.5
* torch.randn(
num_slots,
H,
D,
D,
dtype=torch.float32,
device=DEVICE,
generator=generator,
)
).to(state_dtype)
metadata = KimiK3KDAMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=0,
num_decode_tokens=0,
num_spec_decodes=2,
num_spec_decode_tokens=num_rows,
num_actual_tokens=num_rows,
spec_query_start_loc=query_start_loc,
spec_state_indices_tensor=state_indices,
num_accepted_tokens=num_accepted_tokens,
)
monkeypatch.setattr(
"vllm.models.kimi_k3.nvidia.kda.get_forward_context",
lambda: SimpleNamespace(attn_metadata={"test.layer": metadata}),
)
def clone_strided(tensor: torch.Tensor) -> torch.Tensor:
clone = torch.empty_strided(
tensor.shape, tensor.stride(), dtype=tensor.dtype, device=tensor.device
)
clone.copy_(tensor)
return clone
def composed_reference(
conv_state: torch.Tensor,
recurrent_state: torch.Tensor,
) -> torch.Tensor:
mixed_qkv = causal_conv1d_update(
x,
conv_state,
conv_weight,
bias=None,
activation="silu",
conv_state_indices=state_indices[:, 0],
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc,
max_query_len=T,
validate_data=False,
out=torch.empty_like(x),
)
q, k, v = mixed_qkv.view(num_rows, 3, H, D).unbind(1)
recurrent_out, _ = fused_recurrent_kda_nvidia(
q=q.unsqueeze(0),
k=k.unsqueeze(0),
v=v.unsqueeze(0),
raw_g=raw_gate,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=-5.0,
initial_state=recurrent_state,
cu_seqlens=query_start_loc,
ssm_state_indices=state_indices,
num_accepted_tokens=num_accepted_tokens,
)
recurrent_float = recurrent_out.float()
return (
recurrent_float
* torch.rsqrt(
recurrent_float.square().mean(dim=-1, keepdim=True) + norm_eps
)
* norm_weight
* output_gate.float().sigmoid().unsqueeze(0)
).to(torch.bfloat16)
def run_flashinfer(
conv_state: torch.Tensor,
recurrent_state: torch.Tensor,
output: torch.Tensor,
) -> torch.Tensor:
layer = SimpleNamespace(
prefix="test.layer",
kv_cache=(conv_state.transpose(-1, -2), recurrent_state),
kda_spec_decode_backend="flashinfer",
decode_conv1d_weight=fused_weight,
decode_norm_weight=norm_weight,
gate_lower_bound=-5.0,
A_log=A_log,
dt_bias=dt_bias,
o_norm=SimpleNamespace(eps=norm_eps),
)
KimiK3DeltaAttention._forward(
layer,
mixed_qkv=x,
g1=raw_gate,
g2=output_gate,
beta=raw_beta,
core_attn_out=output,
)
return output
expected_conv = clone_strided(conv_seed)
expected_state = state_seed.clone()
expected = composed_reference(expected_conv, expected_state)
actual_conv = clone_strided(conv_seed)
actual_state = state_seed.clone()
output = torch.empty_like(expected)
actual = run_flashinfer(actual_conv, actual_state, output)
assert actual.data_ptr() == output.data_ptr()
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=3e-2)
assert torch.count_nonzero(actual[:, :7]) > 0
torch.testing.assert_close(actual_state[0], state_seed[0], atol=0, rtol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = run_flashinfer(actual_conv, actual_state, output)
actual_conv.copy_(conv_seed)
actual_state.copy_(state_seed)
query_start_loc.copy_(torch.tensor([0, 6, 10, 10], device=DEVICE))
num_accepted_tokens.copy_(torch.tensor([3, 2, 1], device=DEVICE))
expected_conv.copy_(conv_seed)
expected_state.copy_(state_seed)
expected = composed_reference(expected_conv, expected_state)
graph.replay()
torch.accelerator.synchronize()
assert captured.data_ptr() == output.data_ptr()
torch.testing.assert_close(output, expected, atol=2e-2, rtol=3e-2)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
def test_fused_kda_decode_rejects_speculative_conv_state():
assert not is_fused_kda_decode_supported(
num_heads=12,
+130 -9
View File
@@ -55,8 +55,10 @@ from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
from vllm.utils.flashinfer import (
flashinfer_fused_kda_decode,
flashinfer_packed_fused_kda_decode,
flashinfer_recurrent_kda,
has_flashinfer_fused_kda_decode,
has_flashinfer_packed_fused_kda_decode,
has_flashinfer_recurrent_kda,
)
from vllm.v1.attention.backend import AttentionBackend
@@ -214,6 +216,73 @@ def is_flashinfer_fused_kda_decode_supported(
)
def is_flashinfer_fused_kda_spec_decode_supported(
num_heads: int,
head_dim: int,
conv_width: int,
num_spec: int,
input_dtype: torch.dtype,
conv_state_dtype: torch.dtype,
recurrent_state_dtype: torch.dtype,
lower_bound: float | None,
use_recoverssm: bool,
) -> bool:
return (
has_flashinfer_packed_fused_kda_decode()
and current_platform.is_device_capability_family(100)
and num_heads in (12, 24, 32, 48, 96)
and head_dim == 128
and conv_width == 4
and 1 <= num_spec <= 7
and input_dtype == torch.bfloat16
and conv_state_dtype == torch.bfloat16
and recurrent_state_dtype in (torch.float32, torch.bfloat16)
and lower_bound is not None
and lower_bound < 0
and not use_recoverssm
and not is_conv_state_dim_first()
)
def resolve_kda_spec_decode_backend(
backend: str,
num_heads: int,
head_dim: int,
conv_width: int,
num_spec: int,
input_dtype: torch.dtype,
conv_state_dtype: torch.dtype,
recurrent_state_dtype: torch.dtype,
lower_bound: float | None,
use_recoverssm: bool,
) -> str:
if backend not in ("auto", "native", "flashinfer"):
raise ValueError(f"Unsupported KDA speculative decode backend: {backend}")
supported = is_flashinfer_fused_kda_spec_decode_supported(
num_heads,
head_dim,
conv_width,
num_spec,
input_dtype,
conv_state_dtype,
recurrent_state_dtype,
lower_bound,
use_recoverssm,
)
if backend == "flashinfer" and not supported:
raise RuntimeError(
"FlashInfer packed fused KDA decode requires its "
"packed_fused_kda_decode API, CUDA SM10x, bfloat16 inputs and "
"convolution state, float32 or bfloat16 recurrent state, D=128, "
"W=4, a bounded gate, 1-7 speculative tokens, a supported head "
"count, the SD convolution-state layout, and RecoverSSM disabled."
)
if supported and backend != "native":
logger.info_once("Using FlashInfer packed fused KDA speculative decode.")
return "flashinfer"
return "native"
def resolve_kda_decode_backend(
backend: str,
num_heads: int,
@@ -537,6 +606,13 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
self.projection_size = self.head_dim * self.num_heads
self.local_projection_size = divide(self.projection_size, self.tp_size)
self.conv_size = kda_config["short_conv_kernel_size"]
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
if self.gate_lower_bound is not None:
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
"KDA gate lower bound must be in "
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
f"Got {self.gate_lower_bound}."
)
assert kda_config.get("use_full_rank_gate", False), (
"KimiK3DeltaAttention requires a full-rank gate"
)
@@ -618,8 +694,28 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
conv_state_dtype,
recurrent_state_dtype,
)
spec_decode_backend = (
additional_config.get("kda_spec_decode_backend", "auto")
if isinstance(additional_config, dict)
else "auto"
)
self.kda_spec_decode_backend = resolve_kda_spec_decode_backend(
spec_decode_backend,
self.local_num_heads,
self.head_dim,
self.conv_size,
self.num_spec,
vllm_config.model_config.dtype,
conv_state_dtype,
recurrent_state_dtype,
self.gate_lower_bound,
self.use_recoverssm,
)
decode_conv1d_weight = None
if self.kda_decode_backend != "triton":
if (
self.kda_decode_backend != "triton"
or self.kda_spec_decode_backend == "flashinfer"
):
decode_conv1d_weight = torch.empty(
3,
self.conv_size,
@@ -648,14 +744,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
)
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
if self.gate_lower_bound is not None:
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
"KDA gate lower bound must be in "
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
f"Got {self.gate_lower_bound}."
)
backend = (
additional_config.get("kda_prefill_backend", "auto")
if isinstance(additional_config, dict)
@@ -860,6 +948,39 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
if not is_conv_state_dim_first():
conv_state = conv_state.transpose(-1, -2)
if (
self.kda_spec_decode_backend == "flashinfer"
and has_spec_decode
and m.num_prefills == 0
and m.num_decodes == 0
):
assert self.decode_conv1d_weight is not None
assert self.decode_norm_weight is not None
assert spec_state_indices_tensor is not None
assert spec_query_start_loc is not None
assert num_accepted_tokens is not None
assert self.gate_lower_bound is not None
# "Packed" is FlashInfer's name for the varlen T>1 speculative path.
flashinfer_packed_fused_kda_decode(
x=mixed_qkv,
weight=self.decode_conv1d_weight,
conv_state=conv_state,
raw_gate=g1,
raw_beta=beta,
A_log=self.A_log,
dt_bias=self.dt_bias,
state_indices=spec_state_indices_tensor,
query_start_loc=spec_query_start_loc,
num_accepted_tokens=num_accepted_tokens,
state=recurrent_state,
output_gate=g2[:num_actual_tokens],
norm_weight=self.decode_norm_weight,
lower_bound=self.gate_lower_bound,
norm_eps=self.o_norm.eps,
output=core_attn_out[:, :num_actual_tokens],
)
return
if (
self.kda_decode_backend != "triton"
and self.decode_conv1d_weight is not None
+12
View File
@@ -313,6 +313,9 @@ flashinfer_xqa_batch_decode_with_kv_cache = _lazy_import_wrapper(
"flashinfer.decode",
"xqa_batch_decode_with_kv_cache",
)
flashinfer_packed_fused_kda_decode = _lazy_import_wrapper(
"flashinfer", "packed_fused_kda_decode"
)
flashinfer_recurrent_kda = _lazy_import_wrapper(
"flashinfer.kda",
"recurrent_kda",
@@ -337,6 +340,15 @@ def has_flashinfer_comm() -> bool:
return has_flashinfer() and importlib.util.find_spec("flashinfer.comm") is not None
@functools.cache
def has_flashinfer_packed_fused_kda_decode() -> bool:
"""Return whether FlashInfer's packed fused KDA decode API is available."""
if not has_flashinfer():
return False
module = _get_submodule("flashinfer")
return bool(module and callable(getattr(module, "packed_fused_kda_decode", None)))
@functools.cache
def has_flashinfer_nvlink_two_sided() -> bool:
"""Return `True` if FlashInfer mnnvl all2all is available."""