diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index d276db43a9..4ff43ce21b 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -727,16 +727,19 @@ class FusedMoE(PluggableLayer): # Only narrow if the loaded_weight is not a scalar (0-dim tensor) # and we're not loading the full weight if not load_full and loaded_weight.ndim > 0: - # Handle padding: loaded_weight might be smaller than shard_size on last - # TP rank - start_offset = shard_size * tp_rank + # When the parameter has been padded (e.g. MXFP4 rounding up + # intermediate_size_per_partition), shard_size is the padded + # size. Compute the offset into the checkpoint weight using + # the *unpadded* per-rank size so that every TP rank lands at + # the correct slice. + tp_size = self.moe_config.moe_parallel_config.tp_size + loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size + start_offset = loaded_per_rank * tp_rank available = loaded_weight.shape[shard_dim] - start_offset if available <= 0: # If there is no available weight to load for this TP rank - # (can happen on last TP rank with padding), we can skip - # loading and return early return - narrow_size = min(shard_size, available) + narrow_size = min(loaded_per_rank, available) loaded_weight = loaded_weight.narrow(shard_dim, start_offset, narrow_size) # Narrow parameter and load. # w1, gate_proj: Load into first logical weight of w13. @@ -765,21 +768,18 @@ class FusedMoE(PluggableLayer): ): # Index the loaded weight for tp sharding. # down_proj: "RowParallel" so tp sharding on input_dim - # Narrow parameter and load. - shard_size = expert_data.shape[shard_dim] # Only narrow if the loaded_weight is not a scalar (0-dim tensor) # and we're not loading the full weight if not load_full and loaded_weight.ndim > 0: - # Handle padding: loaded_weight might be smaller than shard_size on last - # TP rank - start_offset = shard_size * tp_rank + # Same padding fix as _load_w13: use unpadded per-rank size. + tp_size = self.moe_config.moe_parallel_config.tp_size + loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size + start_offset = loaded_per_rank * tp_rank available = loaded_weight.shape[shard_dim] - start_offset if available <= 0: # If there is no available weight to load for this TP rank - # (can happen on last TP rank with padding), we can skip - # loading and return early return - narrow_size = min(shard_size, available) + narrow_size = min(loaded_per_rank, available) loaded_weight = loaded_weight.narrow(shard_dim, start_offset, narrow_size) # w2, down_proj: Load into only logical weight of w2. hidden_dim = self._get_hidden_dim(shard_dim, expert_data.ndim) diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index a01718c587..a506eaffd0 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -622,16 +622,15 @@ def select_deepseek_v4_mxfp4_moe_backend( assert last_error is not None raise last_error - # DeepSeek-V4 on ROCm is more accurate with the unfused Triton MXFP4 path - # than the default AITER path. Prefer Triton-unfused for this routing mode, - # while keeping AITER as a fallback if Triton-unfused rejects the config. + # DeepSeek-V4 on ROCm: prefer AITER FlyDSL MoE (better perf + accuracy + # after shuffle/TP-offset fixes), with Triton-unfused as fallback. if ( current_platform.is_rocm() and config.routing_method == RoutingMethodType.DeepseekV4 ): priority_backends = [ - Mxfp4MoeBackend.TRITON_UNFUSED, Mxfp4MoeBackend.AITER_MXFP4_BF16, + Mxfp4MoeBackend.TRITON_UNFUSED, ] else: priority_backends = _get_priority_backends() @@ -1421,7 +1420,7 @@ def convert_weight_to_mxfp4_moe_kernel_format( ) elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16: - from vllm._aiter_ops import rocm_aiter_ops + from vllm._aiter_ops import rocm_aiter_ops # noqa: F401 if w13_bias is not None: w13_bias = w13_bias.data.to(torch.float32) @@ -1430,45 +1429,31 @@ def convert_weight_to_mxfp4_moe_kernel_format( e, n, k = w13_weight.shape - w13_weight.view(torch.uint8).copy_( - w13_weight.data.view(torch.uint8) - .view(e, n // 2, 2, k) - .permute(0, 2, 1, 3) - .contiguous() - .view(e, n, k) + # No de-interleave: standard _load_w13 already produces + # [gate_all, up_all] layout. Use aiter-native shuffle functions + # (matching aiter/ops/flydsl/test_flydsl_moe_a4w4.py pattern). + from aiter.ops.shuffle import shuffle_weight as _shuf_w + from aiter.utility.fp4_utils import e8m0_shuffle as _e8m0_shuf + + # w13 (gate+up, stage1): shuffle_weight with layout (16,16) + w13_weight = torch.nn.Parameter( + _shuf_w(w13_weight.data.view(torch.float4_e2m1fn_x2), (16, 16)), + requires_grad=False, ) - w13_weight_scale.data = ( - w13_weight_scale.data.view(e, n // 2, 2, -1) - .permute(0, 2, 1, 3) - .contiguous() - .view(e, n, -1) + shuffled_w13_scale = _e8m0_shuf( + w13_weight_scale.view(-1, w13_weight_scale.shape[-1]) ) - w13_weight.data = w13_weight.data.view(torch.float4_e2m1fn_x2) - w2_weight.data = w2_weight.data.view(torch.float4_e2m1fn_x2) - - w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(w13_weight, 16, True) - shuffled_w13_scale = rocm_aiter_ops.shuffle_scale_a16w4( - w13_weight_scale.view(-1, w13_weight_scale.shape[-1]), - num_experts, - True, + # w2 (down-proj, stage2): same shuffle as w13 for a4w4 fp4x2 + # (tuning script uses shuffle_weight((16,16)) + e8m0_shuffle for both) + w2_weight = torch.nn.Parameter( + _shuf_w(w2_weight.data.view(torch.float4_e2m1fn_x2), (16, 16)), + requires_grad=False, ) - - w2_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(w2_weight, 16, False) - shuffled_w2_scale = rocm_aiter_ops.shuffle_scale_a16w4( - w2_weight_scale.view(-1, w2_weight_scale.shape[-1]), - num_experts, - False, + shuffled_w2_scale = _e8m0_shuf( + w2_weight_scale.view(-1, w2_weight_scale.shape[-1]) ) - if w13_bias is not None: - w13_bias = ( - w13_bias.data.view(-1, n // 2, 2) - .permute(0, 2, 1) - .contiguous() - .view(-1, n) - ) - return ( w13_weight, w2_weight,