[Bugfix] [ROCm] [DSV4] Fix AITER MXFP4 MoE weight loading and shuffle… (#42595)

Co-authored-by: MHYangAMD <MHYangAMD@users.noreply.github.com>
This commit is contained in:
MHYangAMD
2026-05-29 04:08:33 -07:00
committed by GitHub
co-authored by MHYangAMD
parent 0cff0741ff
commit ab12aab127
2 changed files with 37 additions and 52 deletions
+14 -14
View File
@@ -727,16 +727,19 @@ class FusedMoE(PluggableLayer):
# Only narrow if the loaded_weight is not a scalar (0-dim tensor) # Only narrow if the loaded_weight is not a scalar (0-dim tensor)
# and we're not loading the full weight # and we're not loading the full weight
if not load_full and loaded_weight.ndim > 0: if not load_full and loaded_weight.ndim > 0:
# Handle padding: loaded_weight might be smaller than shard_size on last # When the parameter has been padded (e.g. MXFP4 rounding up
# TP rank # intermediate_size_per_partition), shard_size is the padded
start_offset = shard_size * tp_rank # size. Compute the offset into the checkpoint weight using
# the *unpadded* per-rank size so that every TP rank lands at
# the correct slice.
tp_size = self.moe_config.moe_parallel_config.tp_size
loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size
start_offset = loaded_per_rank * tp_rank
available = loaded_weight.shape[shard_dim] - start_offset available = loaded_weight.shape[shard_dim] - start_offset
if available <= 0: if available <= 0:
# If there is no available weight to load for this TP rank # If there is no available weight to load for this TP rank
# (can happen on last TP rank with padding), we can skip
# loading and return early
return return
narrow_size = min(shard_size, available) narrow_size = min(loaded_per_rank, available)
loaded_weight = loaded_weight.narrow(shard_dim, start_offset, narrow_size) loaded_weight = loaded_weight.narrow(shard_dim, start_offset, narrow_size)
# Narrow parameter and load. # Narrow parameter and load.
# w1, gate_proj: Load into first logical weight of w13. # w1, gate_proj: Load into first logical weight of w13.
@@ -765,21 +768,18 @@ class FusedMoE(PluggableLayer):
): ):
# Index the loaded weight for tp sharding. # Index the loaded weight for tp sharding.
# down_proj: "RowParallel" so tp sharding on input_dim # down_proj: "RowParallel" so tp sharding on input_dim
# Narrow parameter and load.
shard_size = expert_data.shape[shard_dim]
# Only narrow if the loaded_weight is not a scalar (0-dim tensor) # Only narrow if the loaded_weight is not a scalar (0-dim tensor)
# and we're not loading the full weight # and we're not loading the full weight
if not load_full and loaded_weight.ndim > 0: if not load_full and loaded_weight.ndim > 0:
# Handle padding: loaded_weight might be smaller than shard_size on last # Same padding fix as _load_w13: use unpadded per-rank size.
# TP rank tp_size = self.moe_config.moe_parallel_config.tp_size
start_offset = shard_size * tp_rank loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size
start_offset = loaded_per_rank * tp_rank
available = loaded_weight.shape[shard_dim] - start_offset available = loaded_weight.shape[shard_dim] - start_offset
if available <= 0: if available <= 0:
# If there is no available weight to load for this TP rank # If there is no available weight to load for this TP rank
# (can happen on last TP rank with padding), we can skip
# loading and return early
return return
narrow_size = min(shard_size, available) narrow_size = min(loaded_per_rank, available)
loaded_weight = loaded_weight.narrow(shard_dim, start_offset, narrow_size) loaded_weight = loaded_weight.narrow(shard_dim, start_offset, narrow_size)
# w2, down_proj: Load into only logical weight of w2. # w2, down_proj: Load into only logical weight of w2.
hidden_dim = self._get_hidden_dim(shard_dim, expert_data.ndim) hidden_dim = self._get_hidden_dim(shard_dim, expert_data.ndim)
@@ -622,16 +622,15 @@ def select_deepseek_v4_mxfp4_moe_backend(
assert last_error is not None assert last_error is not None
raise last_error raise last_error
# DeepSeek-V4 on ROCm is more accurate with the unfused Triton MXFP4 path # DeepSeek-V4 on ROCm: prefer AITER FlyDSL MoE (better perf + accuracy
# than the default AITER path. Prefer Triton-unfused for this routing mode, # after shuffle/TP-offset fixes), with Triton-unfused as fallback.
# while keeping AITER as a fallback if Triton-unfused rejects the config.
if ( if (
current_platform.is_rocm() current_platform.is_rocm()
and config.routing_method == RoutingMethodType.DeepseekV4 and config.routing_method == RoutingMethodType.DeepseekV4
): ):
priority_backends = [ priority_backends = [
Mxfp4MoeBackend.TRITON_UNFUSED,
Mxfp4MoeBackend.AITER_MXFP4_BF16, Mxfp4MoeBackend.AITER_MXFP4_BF16,
Mxfp4MoeBackend.TRITON_UNFUSED,
] ]
else: else:
priority_backends = _get_priority_backends() priority_backends = _get_priority_backends()
@@ -1421,7 +1420,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
) )
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16: elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
from vllm._aiter_ops import rocm_aiter_ops from vllm._aiter_ops import rocm_aiter_ops # noqa: F401
if w13_bias is not None: if w13_bias is not None:
w13_bias = w13_bias.data.to(torch.float32) w13_bias = w13_bias.data.to(torch.float32)
@@ -1430,45 +1429,31 @@ def convert_weight_to_mxfp4_moe_kernel_format(
e, n, k = w13_weight.shape e, n, k = w13_weight.shape
w13_weight.view(torch.uint8).copy_( # No de-interleave: standard _load_w13 already produces
w13_weight.data.view(torch.uint8) # [gate_all, up_all] layout. Use aiter-native shuffle functions
.view(e, n // 2, 2, k) # (matching aiter/ops/flydsl/test_flydsl_moe_a4w4.py pattern).
.permute(0, 2, 1, 3) from aiter.ops.shuffle import shuffle_weight as _shuf_w
.contiguous() from aiter.utility.fp4_utils import e8m0_shuffle as _e8m0_shuf
.view(e, n, k)
# w13 (gate+up, stage1): shuffle_weight with layout (16,16)
w13_weight = torch.nn.Parameter(
_shuf_w(w13_weight.data.view(torch.float4_e2m1fn_x2), (16, 16)),
requires_grad=False,
) )
w13_weight_scale.data = ( shuffled_w13_scale = _e8m0_shuf(
w13_weight_scale.data.view(e, n // 2, 2, -1) w13_weight_scale.view(-1, w13_weight_scale.shape[-1])
.permute(0, 2, 1, 3)
.contiguous()
.view(e, n, -1)
) )
w13_weight.data = w13_weight.data.view(torch.float4_e2m1fn_x2) # w2 (down-proj, stage2): same shuffle as w13 for a4w4 fp4x2
w2_weight.data = w2_weight.data.view(torch.float4_e2m1fn_x2) # (tuning script uses shuffle_weight((16,16)) + e8m0_shuffle for both)
w2_weight = torch.nn.Parameter(
w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(w13_weight, 16, True) _shuf_w(w2_weight.data.view(torch.float4_e2m1fn_x2), (16, 16)),
shuffled_w13_scale = rocm_aiter_ops.shuffle_scale_a16w4( requires_grad=False,
w13_weight_scale.view(-1, w13_weight_scale.shape[-1]),
num_experts,
True,
) )
shuffled_w2_scale = _e8m0_shuf(
w2_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(w2_weight, 16, False) w2_weight_scale.view(-1, w2_weight_scale.shape[-1])
shuffled_w2_scale = rocm_aiter_ops.shuffle_scale_a16w4(
w2_weight_scale.view(-1, w2_weight_scale.shape[-1]),
num_experts,
False,
) )
if w13_bias is not None:
w13_bias = (
w13_bias.data.view(-1, n // 2, 2)
.permute(0, 2, 1)
.contiguous()
.view(-1, n)
)
return ( return (
w13_weight, w13_weight,
w2_weight, w2_weight,