diff --git a/tests/models/kimi_k3/test_kda.py b/tests/models/kimi_k3/test_kda.py index eaf4b69b40..88acdb6197 100644 --- a/tests/models/kimi_k3/test_kda.py +++ b/tests/models/kimi_k3/test_kda.py @@ -30,13 +30,17 @@ from vllm.models.kimi_k3.amd.ops.third_party.kda import ( ) from vllm.models.kimi_k3.nvidia import kda as nvidia_kda from vllm.models.kimi_k3.nvidia.kda import ( + KimiK3DeltaAttention, _flashinfer_kda_prefill, _flashkda_prefill, is_flashinfer_fused_kda_decode_supported, + is_flashinfer_fused_kda_spec_decode_supported, is_flashinfer_recurrent_kda_prefill_supported, is_flashkda_supported, is_fused_kda_decode_supported, + resolve_kda_spec_decode_backend, ) +from vllm.models.kimi_k3.nvidia.kda_metadata import KimiK3KDAMetadata from vllm.models.kimi_k3.nvidia.model import KimiLinearForCausalLM from vllm.models.kimi_k3.nvidia.ops import recoverssm as recoverssm_ops from vllm.models.kimi_k3.nvidia.ops.recoverssm import ( @@ -90,6 +94,25 @@ def test_kda_warmup_skips_missing_metadata(monkeypatch): assert layer._forward(empty, empty, empty, empty, empty) is None +def test_resolve_kda_spec_decode_backend(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported", + lambda *args: True, + ) + args = (12, 128, 4, 6, torch.bfloat16, torch.bfloat16, torch.float32, -5.0) + assert resolve_kda_spec_decode_backend("auto", *args, False) == "flashinfer" + assert resolve_kda_spec_decode_backend("native", *args, False) == "native" + assert resolve_kda_spec_decode_backend("flashinfer", *args, False) == "flashinfer" + + monkeypatch.setattr( + "vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported", + lambda *args: False, + ) + assert resolve_kda_spec_decode_backend("auto", *args, False) == "native" + with pytest.raises(RuntimeError, match="packed_fused_kda_decode"): + resolve_kda_spec_decode_backend("flashinfer", *args, False) + + def test_kda_recoverssm_config_state_layout(): vllm_config = SimpleNamespace( model_config=SimpleNamespace( @@ -1182,6 +1205,224 @@ def test_fused_kda_decode_correctness( torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2) +@torch.inference_mode() +@pytest.mark.parametrize("state_dtype", [torch.float32, torch.bfloat16]) +def test_flashinfer_fused_kda_spec_decode_integration( + monkeypatch: pytest.MonkeyPatch, + state_dtype: torch.dtype, +): + if not is_flashinfer_fused_kda_spec_decode_supported( + num_heads=12, + head_dim=128, + conv_width=4, + num_spec=6, + input_dtype=torch.bfloat16, + conv_state_dtype=torch.bfloat16, + recurrent_state_dtype=state_dtype, + lower_bound=-5.0, + use_recoverssm=False, + ): + pytest.skip("FlashInfer packed fused KDA decode is not supported") + + generator = torch.Generator(device=DEVICE).manual_seed(20260820) + H, D, W, T = 12, 128, 4, 7 + hidden_size = H * D + num_rows = 10 + num_slots = 16 + + x = torch.randn( + num_rows, + 3 * hidden_size, + dtype=torch.bfloat16, + device=DEVICE, + generator=generator, + ) + conv_weight = 0.1 * torch.randn( + 3 * hidden_size, + W, + dtype=torch.float32, + device=DEVICE, + generator=generator, + ) + fused_weight = conv_weight.view(3, hidden_size, W).transpose(1, 2).contiguous() + raw_gate = torch.randn( + 1, num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator + ) + raw_beta = torch.randn( + 1, num_rows, H, dtype=torch.bfloat16, device=DEVICE, generator=generator + ) + output_gate = torch.randn( + num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator + ) + A_log = torch.log(torch.arange(1, H + 1, dtype=torch.float32, device=DEVICE)) + dt_bias = torch.empty(hidden_size, dtype=torch.float32, device=DEVICE).uniform_( + -7.0, -4.0, generator=generator + ) + norm_weight = torch.randn( + D, dtype=torch.float32, device=DEVICE, generator=generator + ) + norm_eps = 1e-5 + state_indices = torch.tensor( + [ + list(range(1, T + 1)), + list(range(T + 1, 2 * T + 1)), + [0] * T, + ], + dtype=torch.int32, + device=DEVICE, + ) + query_start_loc = torch.tensor([0, 7, 10, 10], dtype=torch.int32, device=DEVICE) + num_accepted_tokens = torch.tensor([1, 4, 1], dtype=torch.int32, device=DEVICE) + conv_seed = ( + 0.5 + * torch.randn( + num_slots, + T + W - 2, + 3 * hidden_size, + dtype=torch.bfloat16, + device=DEVICE, + generator=generator, + ) + ).transpose(1, 2) + state_seed = ( + 0.5 + * torch.randn( + num_slots, + H, + D, + D, + dtype=torch.float32, + device=DEVICE, + generator=generator, + ) + ).to(state_dtype) + metadata = KimiK3KDAMetadata( + num_prefills=0, + num_prefill_tokens=0, + num_decodes=0, + num_decode_tokens=0, + num_spec_decodes=2, + num_spec_decode_tokens=num_rows, + num_actual_tokens=num_rows, + spec_query_start_loc=query_start_loc, + spec_state_indices_tensor=state_indices, + num_accepted_tokens=num_accepted_tokens, + ) + monkeypatch.setattr( + "vllm.models.kimi_k3.nvidia.kda.get_forward_context", + lambda: SimpleNamespace(attn_metadata={"test.layer": metadata}), + ) + + def clone_strided(tensor: torch.Tensor) -> torch.Tensor: + clone = torch.empty_strided( + tensor.shape, tensor.stride(), dtype=tensor.dtype, device=tensor.device + ) + clone.copy_(tensor) + return clone + + def composed_reference( + conv_state: torch.Tensor, + recurrent_state: torch.Tensor, + ) -> torch.Tensor: + mixed_qkv = causal_conv1d_update( + x, + conv_state, + conv_weight, + bias=None, + activation="silu", + conv_state_indices=state_indices[:, 0], + num_accepted_tokens=num_accepted_tokens, + query_start_loc=query_start_loc, + max_query_len=T, + validate_data=False, + out=torch.empty_like(x), + ) + q, k, v = mixed_qkv.view(num_rows, 3, H, D).unbind(1) + recurrent_out, _ = fused_recurrent_kda_nvidia( + q=q.unsqueeze(0), + k=k.unsqueeze(0), + v=v.unsqueeze(0), + raw_g=raw_gate, + raw_beta=raw_beta, + A_log=A_log, + dt_bias=dt_bias, + lower_bound=-5.0, + initial_state=recurrent_state, + cu_seqlens=query_start_loc, + ssm_state_indices=state_indices, + num_accepted_tokens=num_accepted_tokens, + ) + recurrent_float = recurrent_out.float() + return ( + recurrent_float + * torch.rsqrt( + recurrent_float.square().mean(dim=-1, keepdim=True) + norm_eps + ) + * norm_weight + * output_gate.float().sigmoid().unsqueeze(0) + ).to(torch.bfloat16) + + def run_flashinfer( + conv_state: torch.Tensor, + recurrent_state: torch.Tensor, + output: torch.Tensor, + ) -> torch.Tensor: + layer = SimpleNamespace( + prefix="test.layer", + kv_cache=(conv_state.transpose(-1, -2), recurrent_state), + kda_spec_decode_backend="flashinfer", + decode_conv1d_weight=fused_weight, + decode_norm_weight=norm_weight, + gate_lower_bound=-5.0, + A_log=A_log, + dt_bias=dt_bias, + o_norm=SimpleNamespace(eps=norm_eps), + ) + KimiK3DeltaAttention._forward( + layer, + mixed_qkv=x, + g1=raw_gate, + g2=output_gate, + beta=raw_beta, + core_attn_out=output, + ) + return output + + expected_conv = clone_strided(conv_seed) + expected_state = state_seed.clone() + expected = composed_reference(expected_conv, expected_state) + actual_conv = clone_strided(conv_seed) + actual_state = state_seed.clone() + output = torch.empty_like(expected) + actual = run_flashinfer(actual_conv, actual_state, output) + + assert actual.data_ptr() == output.data_ptr() + torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0) + torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2) + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=3e-2) + assert torch.count_nonzero(actual[:, :7]) > 0 + torch.testing.assert_close(actual_state[0], state_seed[0], atol=0, rtol=0) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = run_flashinfer(actual_conv, actual_state, output) + + actual_conv.copy_(conv_seed) + actual_state.copy_(state_seed) + query_start_loc.copy_(torch.tensor([0, 6, 10, 10], device=DEVICE)) + num_accepted_tokens.copy_(torch.tensor([3, 2, 1], device=DEVICE)) + expected_conv.copy_(conv_seed) + expected_state.copy_(state_seed) + expected = composed_reference(expected_conv, expected_state) + graph.replay() + torch.accelerator.synchronize() + + assert captured.data_ptr() == output.data_ptr() + torch.testing.assert_close(output, expected, atol=2e-2, rtol=3e-2) + torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0) + torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2) + + def test_fused_kda_decode_rejects_speculative_conv_state(): assert not is_fused_kda_decode_supported( num_heads=12, diff --git a/vllm/models/kimi_k3/nvidia/kda.py b/vllm/models/kimi_k3/nvidia/kda.py index 97b4029084..26fe7cb5b4 100644 --- a/vllm/models/kimi_k3/nvidia/kda.py +++ b/vllm/models/kimi_k3/nvidia/kda.py @@ -55,8 +55,10 @@ from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig from vllm.utils.flashinfer import ( flashinfer_fused_kda_decode, + flashinfer_packed_fused_kda_decode, flashinfer_recurrent_kda, has_flashinfer_fused_kda_decode, + has_flashinfer_packed_fused_kda_decode, has_flashinfer_recurrent_kda, ) from vllm.v1.attention.backend import AttentionBackend @@ -214,6 +216,73 @@ def is_flashinfer_fused_kda_decode_supported( ) +def is_flashinfer_fused_kda_spec_decode_supported( + num_heads: int, + head_dim: int, + conv_width: int, + num_spec: int, + input_dtype: torch.dtype, + conv_state_dtype: torch.dtype, + recurrent_state_dtype: torch.dtype, + lower_bound: float | None, + use_recoverssm: bool, +) -> bool: + return ( + has_flashinfer_packed_fused_kda_decode() + and current_platform.is_device_capability_family(100) + and num_heads in (12, 24, 32, 48, 96) + and head_dim == 128 + and conv_width == 4 + and 1 <= num_spec <= 7 + and input_dtype == torch.bfloat16 + and conv_state_dtype == torch.bfloat16 + and recurrent_state_dtype in (torch.float32, torch.bfloat16) + and lower_bound is not None + and lower_bound < 0 + and not use_recoverssm + and not is_conv_state_dim_first() + ) + + +def resolve_kda_spec_decode_backend( + backend: str, + num_heads: int, + head_dim: int, + conv_width: int, + num_spec: int, + input_dtype: torch.dtype, + conv_state_dtype: torch.dtype, + recurrent_state_dtype: torch.dtype, + lower_bound: float | None, + use_recoverssm: bool, +) -> str: + if backend not in ("auto", "native", "flashinfer"): + raise ValueError(f"Unsupported KDA speculative decode backend: {backend}") + supported = is_flashinfer_fused_kda_spec_decode_supported( + num_heads, + head_dim, + conv_width, + num_spec, + input_dtype, + conv_state_dtype, + recurrent_state_dtype, + lower_bound, + use_recoverssm, + ) + if backend == "flashinfer" and not supported: + raise RuntimeError( + "FlashInfer packed fused KDA decode requires its " + "packed_fused_kda_decode API, CUDA SM10x, bfloat16 inputs and " + "convolution state, float32 or bfloat16 recurrent state, D=128, " + "W=4, a bounded gate, 1-7 speculative tokens, a supported head " + "count, the SD convolution-state layout, and RecoverSSM disabled." + ) + if supported and backend != "native": + logger.info_once("Using FlashInfer packed fused KDA speculative decode.") + return "flashinfer" + return "native" + + def resolve_kda_decode_backend( backend: str, num_heads: int, @@ -537,6 +606,13 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): self.projection_size = self.head_dim * self.num_heads self.local_projection_size = divide(self.projection_size, self.tp_size) self.conv_size = kda_config["short_conv_kernel_size"] + self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None) + if self.gate_lower_bound is not None: + assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, ( + "KDA gate lower bound must be in " + f"[{_KDA_GATE_LOGBOUND_MIN}, 0). " + f"Got {self.gate_lower_bound}." + ) assert kda_config.get("use_full_rank_gate", False), ( "KimiK3DeltaAttention requires a full-rank gate" ) @@ -618,8 +694,28 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): conv_state_dtype, recurrent_state_dtype, ) + spec_decode_backend = ( + additional_config.get("kda_spec_decode_backend", "auto") + if isinstance(additional_config, dict) + else "auto" + ) + self.kda_spec_decode_backend = resolve_kda_spec_decode_backend( + spec_decode_backend, + self.local_num_heads, + self.head_dim, + self.conv_size, + self.num_spec, + vllm_config.model_config.dtype, + conv_state_dtype, + recurrent_state_dtype, + self.gate_lower_bound, + self.use_recoverssm, + ) decode_conv1d_weight = None - if self.kda_decode_backend != "triton": + if ( + self.kda_decode_backend != "triton" + or self.kda_spec_decode_backend == "flashinfer" + ): decode_conv1d_weight = torch.empty( 3, self.conv_size, @@ -648,14 +744,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): ) set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)}) - self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None) - if self.gate_lower_bound is not None: - assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, ( - "KDA gate lower bound must be in " - f"[{_KDA_GATE_LOGBOUND_MIN}, 0). " - f"Got {self.gate_lower_bound}." - ) - backend = ( additional_config.get("kda_prefill_backend", "auto") if isinstance(additional_config, dict) @@ -860,6 +948,39 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): if not is_conv_state_dim_first(): conv_state = conv_state.transpose(-1, -2) + if ( + self.kda_spec_decode_backend == "flashinfer" + and has_spec_decode + and m.num_prefills == 0 + and m.num_decodes == 0 + ): + assert self.decode_conv1d_weight is not None + assert self.decode_norm_weight is not None + assert spec_state_indices_tensor is not None + assert spec_query_start_loc is not None + assert num_accepted_tokens is not None + assert self.gate_lower_bound is not None + # "Packed" is FlashInfer's name for the varlen T>1 speculative path. + flashinfer_packed_fused_kda_decode( + x=mixed_qkv, + weight=self.decode_conv1d_weight, + conv_state=conv_state, + raw_gate=g1, + raw_beta=beta, + A_log=self.A_log, + dt_bias=self.dt_bias, + state_indices=spec_state_indices_tensor, + query_start_loc=spec_query_start_loc, + num_accepted_tokens=num_accepted_tokens, + state=recurrent_state, + output_gate=g2[:num_actual_tokens], + norm_weight=self.decode_norm_weight, + lower_bound=self.gate_lower_bound, + norm_eps=self.o_norm.eps, + output=core_attn_out[:, :num_actual_tokens], + ) + return + if ( self.kda_decode_backend != "triton" and self.decode_conv1d_weight is not None diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index 44da20a3ed..d7fa807c33 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -313,6 +313,9 @@ flashinfer_xqa_batch_decode_with_kv_cache = _lazy_import_wrapper( "flashinfer.decode", "xqa_batch_decode_with_kv_cache", ) +flashinfer_packed_fused_kda_decode = _lazy_import_wrapper( + "flashinfer", "packed_fused_kda_decode" +) flashinfer_recurrent_kda = _lazy_import_wrapper( "flashinfer.kda", "recurrent_kda", @@ -337,6 +340,15 @@ def has_flashinfer_comm() -> bool: return has_flashinfer() and importlib.util.find_spec("flashinfer.comm") is not None +@functools.cache +def has_flashinfer_packed_fused_kda_decode() -> bool: + """Return whether FlashInfer's packed fused KDA decode API is available.""" + if not has_flashinfer(): + return False + module = _get_submodule("flashinfer") + return bool(module and callable(getattr(module, "packed_fused_kda_decode", None))) + + @functools.cache def has_flashinfer_nvlink_two_sided() -> bool: """Return `True` if FlashInfer mnnvl all2all is available."""