[Bugfix][ROCm][Kimi-K3] Stage fused KDA decode weights after loading (#61207)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
93d0bd9d50
commit
67516b6c42
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.model_executor.utils import is_weights_pre_processed, set_weight_attrs
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
|
||||
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
|
||||
KimiK3ROCmKDAPrefillCheckpointExporter,
|
||||
@@ -54,8 +54,6 @@ from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_decode import (
|
||||
is_fused_kda_decode_supported,
|
||||
make_decode_conv1d_weight_loader,
|
||||
make_decode_norm_weight_loader,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
@@ -204,19 +202,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
self.register_buffer(
|
||||
"decode_conv1d_weight", decode_conv1d_weight, persistent=False
|
||||
)
|
||||
if decode_conv1d_weight is None:
|
||||
conv1d_weight_loader = _make_fused_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
else:
|
||||
conv1d_weight_loader = make_decode_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
decode_conv1d_weight,
|
||||
)
|
||||
conv1d_weight_loader = _make_fused_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
set_weight_attrs(self.conv1d.weight, {"weight_loader": conv1d_weight_loader})
|
||||
|
||||
self.A_log = nn.Parameter(
|
||||
@@ -275,12 +265,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
dtype=torch.float32,
|
||||
device=self.o_norm.weight.device,
|
||||
)
|
||||
if hasattr(self.o_norm.weight, "weight_loader"):
|
||||
delattr(self.o_norm.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.o_norm.weight,
|
||||
{"weight_loader": make_decode_norm_weight_loader(decode_norm_weight)},
|
||||
)
|
||||
self.register_buffer("decode_norm_weight", decode_norm_weight, persistent=False)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.projection_size,
|
||||
@@ -295,6 +279,19 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
@torch.no_grad()
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||
if is_weights_pre_processed():
|
||||
return
|
||||
if self.decode_conv1d_weight is not None:
|
||||
self.decode_conv1d_weight.copy_(
|
||||
self.conv1d.weight.view(
|
||||
3, self.local_projection_size, self.conv_size
|
||||
).transpose(1, 2)
|
||||
)
|
||||
if self.decode_norm_weight is not None:
|
||||
self.decode_norm_weight.copy_(self.o_norm.weight)
|
||||
|
||||
def rearrange_mixed_qkv(
|
||||
self, mixed_qkv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -7,17 +7,14 @@ replaces, for a pure non-speculative decode batch, the three Triton launches
|
||||
and two copies the AMD KDA layer otherwise runs per layer: the packed causal
|
||||
conv1d update, the recurrent delta-rule step, and the gated output RMSNorm.
|
||||
|
||||
The kernel wants a width-major conv weight and an fp32 norm weight, so both are
|
||||
staged once at load time by the weight loaders below.
|
||||
The kernel wants a width-major conv weight and an fp32 norm weight, so the
|
||||
layer stages both once in ``process_weights_after_loading``.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -51,50 +48,3 @@ def is_fused_kda_decode_supported(
|
||||
# gfx950 (MI355X) and gfx942 (MI325X): both CDNA, sharing the wave64 / DPP /
|
||||
# bf16 primitives the kernel relies on.
|
||||
return on_gfx950() or on_gfx942()
|
||||
|
||||
|
||||
def make_decode_conv1d_weight_loader(
|
||||
dims: list[int],
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
decode_conv1d_weight: torch.Tensor | None,
|
||||
) -> Callable[..., None]:
|
||||
"""Load the packed conv1d weight, mirroring a width-major fp32 copy.
|
||||
|
||||
The fused kernel indexes the weight as ``[qkv, width, channel]`` so the
|
||||
channel dimension is the contiguous one; the Triton prefill and fallback
|
||||
decode kernels keep the ``[channel, width]`` layout.
|
||||
"""
|
||||
sharded_dims = [dim // tp_size for dim in dims]
|
||||
|
||||
def weight_loader(
|
||||
param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: int,
|
||||
) -> None:
|
||||
if loaded_weight.dim() == 2:
|
||||
loaded_weight = loaded_weight.unsqueeze(1)
|
||||
shard_size = sharded_dims[loaded_shard_id]
|
||||
source_start = tp_rank * shard_size
|
||||
target_start = sum(sharded_dims[:loaded_shard_id])
|
||||
loaded_shard = loaded_weight[source_start : source_start + shard_size]
|
||||
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
|
||||
if decode_conv1d_weight is not None and not param.is_meta:
|
||||
decode_conv1d_weight[loaded_shard_id].copy_(
|
||||
loaded_shard.squeeze(1).transpose(0, 1)
|
||||
)
|
||||
|
||||
return weight_loader
|
||||
|
||||
|
||||
def make_decode_norm_weight_loader(
|
||||
decode_norm_weight: torch.Tensor,
|
||||
) -> Callable[..., None]:
|
||||
"""Load the gated-norm weight, mirroring an fp32 copy for the kernel."""
|
||||
|
||||
def weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
default_weight_loader(param, loaded_weight)
|
||||
if not param.is_meta:
|
||||
decode_norm_weight.copy_(param.data)
|
||||
|
||||
return weight_loader
|
||||
|
||||
Reference in New Issue
Block a user