From 67516b6c42ba6740d6e263893bfa5e42263f3f1a Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 13 Oct 2026 14:55:57 -0500 Subject: [PATCH] [Bugfix][ROCm][Kimi-K3] Stage fused KDA decode weights after loading (#61207) Signed-off-by: Andreas Karatzas Co-authored-by: Claude --- tests/models/kimi_k3/test_amd_kda_decode.py | 56 +++++++++++++++++++++ vllm/models/kimi_k3/amd/kda.py | 41 +++++++-------- vllm/models/kimi_k3/amd/ops/kda_decode.py | 54 +------------------- 3 files changed, 77 insertions(+), 74 deletions(-) diff --git a/tests/models/kimi_k3/test_amd_kda_decode.py b/tests/models/kimi_k3/test_amd_kda_decode.py index 7978e4fa7b..647c982c7d 100644 --- a/tests/models/kimi_k3/test_amd_kda_decode.py +++ b/tests/models/kimi_k3/test_amd_kda_decode.py @@ -289,3 +289,59 @@ def test_fused_kda_decode_skips_null_block_padding() -> None: torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0) torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0) torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3) + + +def test_decode_weights_staged_after_loading(dist_init) -> None: + """The fused-decode weight copies must not depend on the weight loaders. + + Loaders that bypass ``weight_loader`` (e.g. ``--load-format dummy``) left + them uninitialized, so decode could read NaN. Loading runs with grad mode + on, so the copies must also stay out of autograd. + """ + _requires_kernel() + from tests.v1.attention.utils import create_vllm_config + from vllm.config import set_current_vllm_config + from vllm.model_executor.model_loader.utils import process_weights_after_loading + from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention + from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig + + config = KimiLinearConfig( + hidden_size=256, + linear_attn_config={ + "kda_layers": [1], + "full_attn_layers": [], + "head_dim": HEAD_DIM, + "num_heads": 12, + "short_conv_kernel_size": CONV_WIDTH, + "use_full_rank_gate": True, + "gate_lower_bound": GATE_LOWER_BOUND, + }, + ) + vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B", dtype=DTYPE) + with set_current_vllm_config(vllm_config), torch.device("cuda"): + layer = KimiK3DeltaAttention( + config, vllm_config, prefix="model.layers.0.self_attn" + ) + assert layer.decode_conv1d_weight is not None + with torch.no_grad(): + for param in layer.parameters(): + param.normal_() + layer.decode_conv1d_weight.fill_(float("nan")) + layer.decode_norm_weight.fill_(float("nan")) + + process_weights_after_loading( + torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda") + ) + + dim = layer.local_projection_size + conv_weights = layer.conv1d.weight.squeeze(1) + expected_conv = torch.stack( + [conv_weights[i * dim : (i + 1) * dim].transpose(0, 1) for i in range(3)] + ) + torch.testing.assert_close( + layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0 + ) + torch.testing.assert_close( + layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0 + ) + assert not layer.decode_norm_weight.requires_grad diff --git a/vllm/models/kimi_k3/amd/kda.py b/vllm/models/kimi_k3/amd/kda.py index 409cf6b1bf..96529734bb 100644 --- a/vllm/models/kimi_k3/amd/kda.py +++ b/vllm/models/kimi_k3/amd/kda.py @@ -43,7 +43,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( ) from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader -from vllm.model_executor.utils import set_weight_attrs +from vllm.model_executor.utils import is_weights_pre_processed, set_weight_attrs from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend from vllm.models.kimi_k3.amd.ops.kda_checkpoint import ( KimiK3ROCmKDAPrefillCheckpointExporter, @@ -54,8 +54,6 @@ from vllm.models.kimi_k3.amd.ops.kda_chunk import ( ) from vllm.models.kimi_k3.amd.ops.kda_decode import ( is_fused_kda_decode_supported, - make_decode_conv1d_weight_loader, - make_decode_norm_weight_loader, ) from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill from vllm.models.kimi_k3.amd.ops.third_party.kda import ( @@ -204,19 +202,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): self.register_buffer( "decode_conv1d_weight", decode_conv1d_weight, persistent=False ) - if decode_conv1d_weight is None: - conv1d_weight_loader = _make_fused_conv1d_weight_loader( - [self.projection_size] * 3, - self.tp_size, - self.tp_rank, - ) - else: - conv1d_weight_loader = make_decode_conv1d_weight_loader( - [self.projection_size] * 3, - self.tp_size, - self.tp_rank, - decode_conv1d_weight, - ) + conv1d_weight_loader = _make_fused_conv1d_weight_loader( + [self.projection_size] * 3, + self.tp_size, + self.tp_rank, + ) set_weight_attrs(self.conv1d.weight, {"weight_loader": conv1d_weight_loader}) self.A_log = nn.Parameter( @@ -275,12 +265,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): dtype=torch.float32, device=self.o_norm.weight.device, ) - if hasattr(self.o_norm.weight, "weight_loader"): - delattr(self.o_norm.weight, "weight_loader") - set_weight_attrs( - self.o_norm.weight, - {"weight_loader": make_decode_norm_weight_loader(decode_norm_weight)}, - ) self.register_buffer("decode_norm_weight", decode_norm_weight, persistent=False) self.o_proj = RowParallelLinear( self.projection_size, @@ -295,6 +279,19 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self + @torch.no_grad() + def process_weights_after_loading(self, act_dtype: torch.dtype) -> None: + if is_weights_pre_processed(): + return + if self.decode_conv1d_weight is not None: + self.decode_conv1d_weight.copy_( + self.conv1d.weight.view( + 3, self.local_projection_size, self.conv_size + ).transpose(1, 2) + ) + if self.decode_norm_weight is not None: + self.decode_norm_weight.copy_(self.o_norm.weight) + def rearrange_mixed_qkv( self, mixed_qkv: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: diff --git a/vllm/models/kimi_k3/amd/ops/kda_decode.py b/vllm/models/kimi_k3/amd/ops/kda_decode.py index 682776a807..d944e1eb04 100644 --- a/vllm/models/kimi_k3/amd/ops/kda_decode.py +++ b/vllm/models/kimi_k3/amd/ops/kda_decode.py @@ -7,17 +7,14 @@ replaces, for a pure non-speculative decode batch, the three Triton launches and two copies the AMD KDA layer otherwise runs per layer: the packed causal conv1d update, the recurrent delta-rule step, and the gated output RMSNorm. -The kernel wants a width-major conv weight and an fp32 norm weight, so both are -staged once at load time by the weight loaders below. +The kernel wants a width-major conv weight and an fp32 norm weight, so the +layer stages both once in ``process_weights_after_loading``. """ -from collections.abc import Callable - import torch from vllm.logger import init_logger from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first -from vllm.model_executor.model_loader.weight_utils import default_weight_loader logger = init_logger(__name__) @@ -51,50 +48,3 @@ def is_fused_kda_decode_supported( # gfx950 (MI355X) and gfx942 (MI325X): both CDNA, sharing the wave64 / DPP / # bf16 primitives the kernel relies on. return on_gfx950() or on_gfx942() - - -def make_decode_conv1d_weight_loader( - dims: list[int], - tp_size: int, - tp_rank: int, - decode_conv1d_weight: torch.Tensor | None, -) -> Callable[..., None]: - """Load the packed conv1d weight, mirroring a width-major fp32 copy. - - The fused kernel indexes the weight as ``[qkv, width, channel]`` so the - channel dimension is the contiguous one; the Triton prefill and fallback - decode kernels keep the ``[channel, width]`` layout. - """ - sharded_dims = [dim // tp_size for dim in dims] - - def weight_loader( - param: torch.Tensor, - loaded_weight: torch.Tensor, - loaded_shard_id: int, - ) -> None: - if loaded_weight.dim() == 2: - loaded_weight = loaded_weight.unsqueeze(1) - shard_size = sharded_dims[loaded_shard_id] - source_start = tp_rank * shard_size - target_start = sum(sharded_dims[:loaded_shard_id]) - loaded_shard = loaded_weight[source_start : source_start + shard_size] - param.data[target_start : target_start + shard_size].copy_(loaded_shard) - if decode_conv1d_weight is not None and not param.is_meta: - decode_conv1d_weight[loaded_shard_id].copy_( - loaded_shard.squeeze(1).transpose(0, 1) - ) - - return weight_loader - - -def make_decode_norm_weight_loader( - decode_norm_weight: torch.Tensor, -) -> Callable[..., None]: - """Load the gated-norm weight, mirroring an fp32 copy for the kernel.""" - - def weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: - default_weight_loader(param, loaded_weight) - if not param.is_meta: - decode_norm_weight.copy_(param.data) - - return weight_loader