[ROCm][Perf] Add opt-in a4w4 (FP4 activation) MoE for DeepSeek V4.1 on AITER (#58819)
Signed-off-by: fai <fangzhouai@gmail.com> Signed-off-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Co-authored-by: Cursor Agent (Claude Sonnet 5) <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor Agent
parent
e5f3d08d72
commit
77e8b02c8d
@@ -6,6 +6,7 @@ from typing import Union
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ParallelConfig, SchedulerConfig
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.distributed import get_dp_group, get_pcp_group, get_tensor_model_parallel_rank
|
||||
@@ -1247,6 +1248,12 @@ class FusedMoEParallelConfig:
|
||||
)
|
||||
|
||||
|
||||
# Model types validated for VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1. This flag
|
||||
# also changes the MXFP4 weight shuffle layout, so using it on other models
|
||||
# can produce garbled output.
|
||||
_AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES = ("deepseek_v41", "deepseek_v41_text")
|
||||
|
||||
|
||||
# Adapted from pplx-kernels tests/all_to_all_utils.py
|
||||
@dataclass
|
||||
class FusedMoEConfig:
|
||||
@@ -1314,6 +1321,11 @@ class FusedMoEConfig:
|
||||
tp_shard_with_padding: bool = False
|
||||
rocm_aiter_fmoe_enabled: bool = False
|
||||
aiter_fmoe_shared_expert_enabled: bool = False
|
||||
# Whether to force MXFP4 (a4w4) MoE activations for DeepSeek V4.1 on
|
||||
# ROCm/AITER. Opt-in via VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1; rejected
|
||||
# for any other model type. Resolved here, not in the forward path,
|
||||
# because get_current_vllm_config() isn't set there.
|
||||
use_mxfp4_w4a4_dsv4: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
@@ -1345,6 +1357,29 @@ class FusedMoEConfig:
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
|
||||
if self.rocm_aiter_fmoe_enabled and envs.VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4:
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
model_type = (
|
||||
getattr(vllm_config.model_config.hf_config, "model_type", None)
|
||||
if vllm_config is not None
|
||||
else None
|
||||
)
|
||||
if model_type not in _AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES:
|
||||
raise ValueError(
|
||||
f"VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 only supports "
|
||||
f"model_type in {_AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES}, "
|
||||
f"got {model_type!r}. Unset this env var for this model."
|
||||
)
|
||||
if not rocm_aiter_ops.fused_moe_supports_quant_dtype_a():
|
||||
raise ValueError(
|
||||
"VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 needs an AITER build "
|
||||
"with fused_moe(quant_dtype_a=...) support "
|
||||
"(ROCm/aiter#5439+). Upgrade AITER or unset this env var."
|
||||
)
|
||||
self.use_mxfp4_w4a4_dsv4 = True
|
||||
|
||||
if self.use_mori_kernels:
|
||||
assert self.rocm_aiter_fmoe_enabled, (
|
||||
"Mori needs to be used with aiter fused_moe for now."
|
||||
|
||||
@@ -375,15 +375,13 @@ def rocm_aiter_fused_experts(
|
||||
intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1)
|
||||
)
|
||||
|
||||
# https://github.com/ROCm/aiter/pull/3123 specialized the AITER stage1 GEMMs
|
||||
# for interleaved vs separated gate and up weights.
|
||||
# For gpt-oss i.e. use_mxfp4_w4a16=True, the weights are shuffled by
|
||||
# `rocm_aiter_ops.shuffle_weight_a16w4` in `oracle/mxfp4.py`,
|
||||
# which always sets `is_guinterleave=True`.
|
||||
# Hence, we pass in GateMode.INTERLEAVE to match the weight shuffling.
|
||||
# AITER's stage1 GEMM needs gate_mode to match how weights were
|
||||
# shuffled at load time (oracle/mxfp4.py). gpt-oss uses INTERLEAVE;
|
||||
# DeepSeek V4.1 a4w4 uses SEPARATED (see below).
|
||||
from aiter.ops.flydsl.moe_common import GateMode
|
||||
|
||||
gate_mode = ""
|
||||
q_dtype_a = None
|
||||
if activation == MoEActivation.SITU:
|
||||
gate_mode = (
|
||||
GateMode.INTERLEAVE.value
|
||||
@@ -391,7 +389,15 @@ def rocm_aiter_fused_experts(
|
||||
else GateMode.SEPARATED.value
|
||||
)
|
||||
elif quant_config.use_mxfp4_w4a16:
|
||||
gate_mode = GateMode.INTERLEAVE.value
|
||||
if moe_config.use_mxfp4_w4a4_dsv4:
|
||||
# Opt-in a4w4 for DeepSeek V4.1 (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1).
|
||||
# Must match the SEPARATED weight shuffle from load time.
|
||||
from aiter import dtypes
|
||||
|
||||
gate_mode = GateMode.SEPARATED.value
|
||||
q_dtype_a = dtypes.fp4x2
|
||||
else:
|
||||
gate_mode = GateMode.INTERLEAVE.value
|
||||
elif activation_interleave is not None:
|
||||
gate_mode = (
|
||||
GateMode.INTERLEAVE.value
|
||||
@@ -433,6 +439,7 @@ def rocm_aiter_fused_experts(
|
||||
shared_w1_scale=shared_w1_scale,
|
||||
shared_w2_scale=shared_w2_scale,
|
||||
shared_expert_id=shared_expert_id,
|
||||
q_dtype_a=q_dtype_a,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1382,6 +1382,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
_cache_permute_indices: dict[torch.Size, torch.Tensor] | None = None,
|
||||
activation: MoEActivation | None = None,
|
||||
use_separated_a4w4: bool = False,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
@@ -1626,10 +1627,16 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
from aiter.ops.shuffle import shuffle_scale as _shuf_s
|
||||
from aiter.ops.shuffle import shuffle_weight as _shuf_w
|
||||
|
||||
# DeepSeek V4.1 a4w4 uses ATOM's SEPARATED gate/up layout instead of
|
||||
# the default INTERLEAVE shuffle (INTERLEAVE + fp4x2 has no tuned
|
||||
# kernel and produces garbage output). Must match GateMode.SEPARATED
|
||||
# in rocm_aiter_moe.py.
|
||||
is_guinterleave = not use_separated_a4w4
|
||||
|
||||
w13_weight = torch.nn.Parameter(
|
||||
_shuf_w(
|
||||
w13_weight.data.view(torch.float4_e2m1fn_x2),
|
||||
is_guinterleave=True,
|
||||
is_guinterleave=is_guinterleave,
|
||||
gate_up=True,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -1637,14 +1644,14 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
shuffled_w13_scale = _shuf_s(
|
||||
w13_weight_scale.reshape(-1, w13_weight_scale.shape[-1]),
|
||||
num_experts,
|
||||
True,
|
||||
is_guinterleave,
|
||||
True,
|
||||
)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
_shuf_w(
|
||||
w2_weight.data.view(torch.float4_e2m1fn_x2),
|
||||
is_guinterleave=True,
|
||||
is_guinterleave=is_guinterleave,
|
||||
gate_up=False,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -1653,7 +1660,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
shuffled_w2_scale = _shuf_s(
|
||||
w2_weight_scale.reshape(-1, w2_weight_scale.shape[-1]),
|
||||
num_experts,
|
||||
True,
|
||||
is_guinterleave,
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
@@ -736,6 +736,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w2_bias=w2_bias,
|
||||
_cache_permute_indices=self._cache_permute_indices,
|
||||
activation=self.moe.activation,
|
||||
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -220,6 +220,7 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
w2_weight_scale=w2_scale,
|
||||
w13_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user