[Bugfix][ROCm][Kimi-K3] Stage fused KDA decode weights after loading (#61207)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Andreas Karatzas
2026-10-13 14:55:57 -05:00
committed by GitHub
co-authored by Claude
parent 93d0bd9d50
commit 67516b6c42
3 changed files with 77 additions and 74 deletions
@@ -289,3 +289,59 @@ def test_fused_kda_decode_skips_null_block_padding() -> None:
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
def test_decode_weights_staged_after_loading(dist_init) -> None:
"""The fused-decode weight copies must not depend on the weight loaders.
Loaders that bypass ``weight_loader`` (e.g. ``--load-format dummy``) left
them uninitialized, so decode could read NaN. Loading runs with grad mode
on, so the copies must also stay out of autograd.
"""
_requires_kernel()
from tests.v1.attention.utils import create_vllm_config
from vllm.config import set_current_vllm_config
from vllm.model_executor.model_loader.utils import process_weights_after_loading
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
config = KimiLinearConfig(
hidden_size=256,
linear_attn_config={
"kda_layers": [1],
"full_attn_layers": [],
"head_dim": HEAD_DIM,
"num_heads": 12,
"short_conv_kernel_size": CONV_WIDTH,
"use_full_rank_gate": True,
"gate_lower_bound": GATE_LOWER_BOUND,
},
)
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B", dtype=DTYPE)
with set_current_vllm_config(vllm_config), torch.device("cuda"):
layer = KimiK3DeltaAttention(
config, vllm_config, prefix="model.layers.0.self_attn"
)
assert layer.decode_conv1d_weight is not None
with torch.no_grad():
for param in layer.parameters():
param.normal_()
layer.decode_conv1d_weight.fill_(float("nan"))
layer.decode_norm_weight.fill_(float("nan"))
process_weights_after_loading(
torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda")
)
dim = layer.local_projection_size
conv_weights = layer.conv1d.weight.squeeze(1)
expected_conv = torch.stack(
[conv_weights[i * dim : (i + 1) * dim].transpose(0, 1) for i in range(3)]
)
torch.testing.assert_close(
layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0
)
torch.testing.assert_close(
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
)
assert not layer.decode_norm_weight.requires_grad
+19 -22
View File
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.model_executor.utils import is_weights_pre_processed, set_weight_attrs
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
KimiK3ROCmKDAPrefillCheckpointExporter,
@@ -54,8 +54,6 @@ from vllm.models.kimi_k3.amd.ops.kda_chunk import (
)
from vllm.models.kimi_k3.amd.ops.kda_decode import (
is_fused_kda_decode_supported,
make_decode_conv1d_weight_loader,
make_decode_norm_weight_loader,
)
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
@@ -204,19 +202,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
self.register_buffer(
"decode_conv1d_weight", decode_conv1d_weight, persistent=False
)
if decode_conv1d_weight is None:
conv1d_weight_loader = _make_fused_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
)
else:
conv1d_weight_loader = make_decode_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
decode_conv1d_weight,
)
conv1d_weight_loader = _make_fused_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
)
set_weight_attrs(self.conv1d.weight, {"weight_loader": conv1d_weight_loader})
self.A_log = nn.Parameter(
@@ -275,12 +265,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
dtype=torch.float32,
device=self.o_norm.weight.device,
)
if hasattr(self.o_norm.weight, "weight_loader"):
delattr(self.o_norm.weight, "weight_loader")
set_weight_attrs(
self.o_norm.weight,
{"weight_loader": make_decode_norm_weight_loader(decode_norm_weight)},
)
self.register_buffer("decode_norm_weight", decode_norm_weight, persistent=False)
self.o_proj = RowParallelLinear(
self.projection_size,
@@ -295,6 +279,19 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
@torch.no_grad()
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
if is_weights_pre_processed():
return
if self.decode_conv1d_weight is not None:
self.decode_conv1d_weight.copy_(
self.conv1d.weight.view(
3, self.local_projection_size, self.conv_size
).transpose(1, 2)
)
if self.decode_norm_weight is not None:
self.decode_norm_weight.copy_(self.o_norm.weight)
def rearrange_mixed_qkv(
self, mixed_qkv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+2 -52
View File
@@ -7,17 +7,14 @@ replaces, for a pure non-speculative decode batch, the three Triton launches
and two copies the AMD KDA layer otherwise runs per layer: the packed causal
conv1d update, the recurrent delta-rule step, and the gated output RMSNorm.
The kernel wants a width-major conv weight and an fp32 norm weight, so both are
staged once at load time by the weight loaders below.
The kernel wants a width-major conv weight and an fp32 norm weight, so the
layer stages both once in ``process_weights_after_loading``.
"""
from collections.abc import Callable
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
logger = init_logger(__name__)
@@ -51,50 +48,3 @@ def is_fused_kda_decode_supported(
# gfx950 (MI355X) and gfx942 (MI325X): both CDNA, sharing the wave64 / DPP /
# bf16 primitives the kernel relies on.
return on_gfx950() or on_gfx942()
def make_decode_conv1d_weight_loader(
dims: list[int],
tp_size: int,
tp_rank: int,
decode_conv1d_weight: torch.Tensor | None,
) -> Callable[..., None]:
"""Load the packed conv1d weight, mirroring a width-major fp32 copy.
The fused kernel indexes the weight as ``[qkv, width, channel]`` so the
channel dimension is the contiguous one; the Triton prefill and fallback
decode kernels keep the ``[channel, width]`` layout.
"""
sharded_dims = [dim // tp_size for dim in dims]
def weight_loader(
param: torch.Tensor,
loaded_weight: torch.Tensor,
loaded_shard_id: int,
) -> None:
if loaded_weight.dim() == 2:
loaded_weight = loaded_weight.unsqueeze(1)
shard_size = sharded_dims[loaded_shard_id]
source_start = tp_rank * shard_size
target_start = sum(sharded_dims[:loaded_shard_id])
loaded_shard = loaded_weight[source_start : source_start + shard_size]
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
if decode_conv1d_weight is not None and not param.is_meta:
decode_conv1d_weight[loaded_shard_id].copy_(
loaded_shard.squeeze(1).transpose(0, 1)
)
return weight_loader
def make_decode_norm_weight_loader(
decode_norm_weight: torch.Tensor,
) -> Callable[..., None]:
"""Load the gated-norm weight, mirroring an fp32 copy for the kernel."""
def weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
default_weight_loader(param, loaded_weight)
if not param.is_meta:
decode_norm_weight.copy_(param.data)
return weight_loader