[Bugfix][ROCm][Kimi-K3] Stage fused KDA decode weights after loading (#61207)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Andreas Karatzas
2026-10-13 14:55:57 -05:00
committed by GitHub
co-authored by Claude
parent 93d0bd9d50
commit 67516b6c42
3 changed files with 77 additions and 74 deletions
+19 -22
View File
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.model_executor.utils import is_weights_pre_processed, set_weight_attrs
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
KimiK3ROCmKDAPrefillCheckpointExporter,
@@ -54,8 +54,6 @@ from vllm.models.kimi_k3.amd.ops.kda_chunk import (
)
from vllm.models.kimi_k3.amd.ops.kda_decode import (
is_fused_kda_decode_supported,
make_decode_conv1d_weight_loader,
make_decode_norm_weight_loader,
)
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
@@ -204,19 +202,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
self.register_buffer(
"decode_conv1d_weight", decode_conv1d_weight, persistent=False
)
if decode_conv1d_weight is None:
conv1d_weight_loader = _make_fused_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
)
else:
conv1d_weight_loader = make_decode_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
decode_conv1d_weight,
)
conv1d_weight_loader = _make_fused_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
)
set_weight_attrs(self.conv1d.weight, {"weight_loader": conv1d_weight_loader})
self.A_log = nn.Parameter(
@@ -275,12 +265,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
dtype=torch.float32,
device=self.o_norm.weight.device,
)
if hasattr(self.o_norm.weight, "weight_loader"):
delattr(self.o_norm.weight, "weight_loader")
set_weight_attrs(
self.o_norm.weight,
{"weight_loader": make_decode_norm_weight_loader(decode_norm_weight)},
)
self.register_buffer("decode_norm_weight", decode_norm_weight, persistent=False)
self.o_proj = RowParallelLinear(
self.projection_size,
@@ -295,6 +279,19 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
@torch.no_grad()
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
if is_weights_pre_processed():
return
if self.decode_conv1d_weight is not None:
self.decode_conv1d_weight.copy_(
self.conv1d.weight.view(
3, self.local_projection_size, self.conv_size
).transpose(1, 2)
)
if self.decode_norm_weight is not None:
self.decode_norm_weight.copy_(self.o_norm.weight)
def rearrange_mixed_qkv(
self, mixed_qkv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+2 -52
View File
@@ -7,17 +7,14 @@ replaces, for a pure non-speculative decode batch, the three Triton launches
and two copies the AMD KDA layer otherwise runs per layer: the packed causal
conv1d update, the recurrent delta-rule step, and the gated output RMSNorm.
The kernel wants a width-major conv weight and an fp32 norm weight, so both are
staged once at load time by the weight loaders below.
The kernel wants a width-major conv weight and an fp32 norm weight, so the
layer stages both once in ``process_weights_after_loading``.
"""
from collections.abc import Callable
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
logger = init_logger(__name__)
@@ -51,50 +48,3 @@ def is_fused_kda_decode_supported(
# gfx950 (MI355X) and gfx942 (MI325X): both CDNA, sharing the wave64 / DPP /
# bf16 primitives the kernel relies on.
return on_gfx950() or on_gfx942()
def make_decode_conv1d_weight_loader(
dims: list[int],
tp_size: int,
tp_rank: int,
decode_conv1d_weight: torch.Tensor | None,
) -> Callable[..., None]:
"""Load the packed conv1d weight, mirroring a width-major fp32 copy.
The fused kernel indexes the weight as ``[qkv, width, channel]`` so the
channel dimension is the contiguous one; the Triton prefill and fallback
decode kernels keep the ``[channel, width]`` layout.
"""
sharded_dims = [dim // tp_size for dim in dims]
def weight_loader(
param: torch.Tensor,
loaded_weight: torch.Tensor,
loaded_shard_id: int,
) -> None:
if loaded_weight.dim() == 2:
loaded_weight = loaded_weight.unsqueeze(1)
shard_size = sharded_dims[loaded_shard_id]
source_start = tp_rank * shard_size
target_start = sum(sharded_dims[:loaded_shard_id])
loaded_shard = loaded_weight[source_start : source_start + shard_size]
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
if decode_conv1d_weight is not None and not param.is_meta:
decode_conv1d_weight[loaded_shard_id].copy_(
loaded_shard.squeeze(1).transpose(0, 1)
)
return weight_loader
def make_decode_norm_weight_loader(
decode_norm_weight: torch.Tensor,
) -> Callable[..., None]:
"""Load the gated-norm weight, mirroring an fp32 copy for the kernel."""
def weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
default_weight_loader(param, loaded_weight)
if not param.is_meta:
decode_norm_weight.copy_(param.data)
return weight_loader