[ROCm][Perf][DeepSeek V4] Fuse native FP8 shared expert with MXFP4 routed experts (#53161)

Signed-off-by: fai <fangzhouai@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Cursor Agent <agent@cursor.com>
This commit is contained in:
Fangzhou Ai
2026-09-06 19:42:21 -07:00
committed by GitHub
co-authored by OpenAI Codex Cursor Agent
parent 294fbb4f59
commit de69e821b7
4 changed files with 839 additions and 6 deletions
@@ -2,8 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
import inspect
import sys
from copy import deepcopy
from types import SimpleNamespace
from types import ModuleType, SimpleNamespace
from typing import Any, cast
from unittest.mock import patch
@@ -414,6 +416,343 @@ def test_deepseek_v4_shared_expert_fse_uses_mtp_quantization_config_prefix(
assert reason is None
def test_deepseek_v4_heterogeneous_fhmoe_keeps_native_intermediate_width() -> None:
from vllm.models.deepseek_v4.amd.model import _prepare_native_fp8_shared_expert
hidden_size = 7168
intermediate_size = 384
w13 = torch.empty((2 * intermediate_size, hidden_size), dtype=torch.float8_e4m3fn)
w2 = torch.empty((hidden_size, intermediate_size), dtype=torch.float8_e4m3fn)
w13_scale_bytes = (torch.arange(6 * 56).reshape(6, 56).remainder(20) + 0x60).to(
torch.uint8
)
w2_scale_bytes = (torch.arange(56 * 3).reshape(56, 3).remainder(30) + 0x50).to(
torch.uint8
)
w13_scale = w13_scale_bytes.view(torch.float8_e8m0fnu)
w2_scale = w2_scale_bytes.view(torch.float8_e8m0fnu)
prepared = _prepare_native_fp8_shared_expert(
w13, w2, w13_scale, w2_scale, intermediate_size
)
assert prepared[0].shape == (1, 768, 7168)
assert prepared[1].shape == (1, 7168, 384)
assert prepared[2].shape == (768, 224)
assert prepared[3].shape == (7168, 16)
expected_w13_scale = w13_scale_bytes.repeat_interleave(
128, dim=0
).repeat_interleave(4, dim=1)
expected_w2_scale = w2_scale_bytes.repeat_interleave(128, dim=0).repeat_interleave(
4, dim=1
)
assert torch.equal(prepared[2].view(torch.uint8), expected_w13_scale)
assert torch.equal(prepared[3].view(torch.uint8)[:, :12], expected_w2_scale)
assert torch.all(prepared[3].view(torch.uint8)[:, 12:] == 0x7F)
@pytest.mark.parametrize(
("num_tokens", "supported_through", "expected"),
[
(0, 4096, False),
(1, 4096, True),
(1536, 4096, True),
(2048, 4096, True),
(2049, 4096, True),
(4096, 4096, True),
(4097, 4096, False),
(1536, 0, False),
],
)
def test_deepseek_v4_heterogeneous_fhmoe_token_policy(
monkeypatch: pytest.MonkeyPatch,
num_tokens: int,
supported_through: int,
expected: bool,
) -> None:
from vllm.models.deepseek_v4.amd import model as deepseek_v4_model
checked_tokens: list[int] = []
def supports(num_tokens: int) -> bool:
checked_tokens.append(num_tokens)
return num_tokens <= supported_through
monkeypatch.setattr(
deepseek_v4_model.rocm_aiter_ops,
"fused_moe_supports_heterogeneous_shared_expert",
supports,
)
assert deepseek_v4_model._use_heterogeneous_fhmoe(num_tokens) is expected
assert checked_tokens == ([] if num_tokens == 0 else [num_tokens])
def _supported_fhmoe_signature(
shared_w1=None,
shared_w2=None,
shared_w1_scale=None,
shared_w2_scale=None,
shared_expert_id=-1,
) -> None:
pass
def _incomplete_fhmoe_signature(
shared_w1=None,
shared_w2=None,
shared_w1_scale=None,
shared_w2_scale=None,
) -> None:
pass
def _install_fake_aiter_fhmoe(
monkeypatch: pytest.MonkeyPatch,
supports_dsv4_i384_fhmoe: object,
fused_moe: object,
) -> None:
fake_aiter = ModuleType("aiter")
fake_aiter.__path__ = []
fake_fhmoe = ModuleType("aiter.fhmoe")
if supports_dsv4_i384_fhmoe is not None:
fake_fhmoe.__dict__["supports_dsv4_i384_fhmoe"] = supports_dsv4_i384_fhmoe
fake_fused_moe = ModuleType("aiter.fused_moe")
fake_fused_moe.__dict__["fused_moe"] = fused_moe
fake_aiter.__dict__["fhmoe"] = fake_fhmoe
fake_aiter.__dict__["fused_moe"] = fake_fused_moe
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setitem(sys.modules, "aiter.fhmoe", fake_fhmoe)
monkeypatch.setitem(sys.modules, "aiter.fused_moe", fake_fused_moe)
def _supports_fhmoe_through_2047(max_tokens: int) -> bool:
return max_tokens <= 2047
def _supports_fhmoe_through_2048(max_tokens: int) -> bool:
return max_tokens <= 2048
def _supports_fhmoe_through_4096(max_tokens: int) -> bool:
return max_tokens <= 4096
def _raises_fhmoe_config_error(max_tokens: int) -> bool:
raise OSError
def _returns_truthy_non_bool(max_tokens: int) -> int:
return 1
@pytest.mark.parametrize(
("num_tokens", "capability", "fused_moe", "expected"),
[
(0, _supports_fhmoe_through_2048, _supported_fhmoe_signature, False),
(True, _supports_fhmoe_through_2048, _supported_fhmoe_signature, False),
(2048, None, _supported_fhmoe_signature, False),
(2048, True, _supported_fhmoe_signature, False),
(2048, _raises_fhmoe_config_error, _supported_fhmoe_signature, False),
(2048, _returns_truthy_non_bool, _supported_fhmoe_signature, False),
(2048, _supports_fhmoe_through_2047, _supported_fhmoe_signature, False),
(2048, _supports_fhmoe_through_2048, _incomplete_fhmoe_signature, False),
(2048, _supports_fhmoe_through_2048, _supported_fhmoe_signature, True),
(2049, _supports_fhmoe_through_2048, _supported_fhmoe_signature, False),
(4096, _supports_fhmoe_through_4096, _supported_fhmoe_signature, True),
(4097, _supports_fhmoe_through_4096, _supported_fhmoe_signature, False),
],
)
def test_deepseek_v4_heterogeneous_fhmoe_aiter_capability(
monkeypatch: pytest.MonkeyPatch,
num_tokens: object,
capability: object,
fused_moe: object,
expected: bool,
) -> None:
from vllm._aiter_ops import rocm_aiter_ops
_install_fake_aiter_fhmoe(monkeypatch, capability, fused_moe)
assert (
rocm_aiter_ops._probe_dsv4_i384_fhmoe_capability(cast(int, num_tokens))
is expected
)
def test_deepseek_v4_heterogeneous_fhmoe_aiter_capability_catches_import_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from vllm._aiter_ops import rocm_aiter_ops
fake_aiter = ModuleType("aiter")
fake_aiter.__path__ = []
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setitem(sys.modules, "aiter.fhmoe", None)
assert not rocm_aiter_ops._probe_dsv4_i384_fhmoe_capability(2048)
@pytest.mark.parametrize("error", [TypeError, ValueError])
def test_deepseek_v4_heterogeneous_fhmoe_aiter_capability_catches_signature_error(
monkeypatch: pytest.MonkeyPatch,
error: type[Exception],
) -> None:
from vllm._aiter_ops import rocm_aiter_ops
_install_fake_aiter_fhmoe(
monkeypatch, _supports_fhmoe_through_2048, _supported_fhmoe_signature
)
def raise_signature_error(_: object) -> inspect.Signature:
raise error
monkeypatch.setattr(inspect, "signature", raise_signature_error)
assert not rocm_aiter_ops._probe_dsv4_i384_fhmoe_capability(2048)
@pytest.mark.parametrize(
("setting", "value", "expected"),
[
("data_parallel_size", 1, True),
("data_parallel_size", 2, False),
("prefill_context_parallel_size", 2, False),
("topk_method", "greedy", False),
("fhmoe_supported", False, False),
],
)
def test_deepseek_v4_heterogeneous_fhmoe_compatibility_gates(
monkeypatch: pytest.MonkeyPatch,
setting: str,
value: object,
expected: bool,
) -> None:
from vllm.models.deepseek_v4.amd import model as deepseek_v4_model
hf_config = SimpleNamespace(
n_routed_experts=384,
num_experts_per_tok=6,
n_shared_experts=1,
hidden_size=7168,
moe_intermediate_size=3072,
hidden_act="silu",
expert_dtype="fp4",
topk_method="noaux_tc",
)
parallel_config = SimpleNamespace(
enable_expert_parallel=False,
enable_eplb=False,
tensor_parallel_size=8,
data_parallel_size=1,
prefill_context_parallel_size=1,
)
if setting == "topk_method":
hf_config.topk_method = value
elif setting != "fhmoe_supported":
setattr(parallel_config, setting, value)
quant_config = SimpleNamespace(
get_name=lambda: "deepseek_v4_fp8",
moe_quant_algo="",
weight_block_size=[128, 128],
is_checkpoint_fp8_serialized=True,
is_scale_e8m0=True,
ignored_layers=None,
)
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(hf_config=hf_config, dtype=torch.bfloat16),
quant_config=quant_config,
parallel_config=parallel_config,
kernel_config=SimpleNamespace(moe_backend="aiter"),
offload_config=None,
)
monkeypatch.setattr(deepseek_v4_model.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr(
deepseek_v4_model.envs,
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS",
True,
)
monkeypatch.setattr(deepseek_v4_model, "on_gfx950", lambda: True)
monkeypatch.setattr(
deepseek_v4_model.rocm_aiter_ops,
"is_fusion_moe_shared_experts_enabled",
lambda: True,
)
checked_tokens: list[int] = []
fhmoe_supported = setting != "fhmoe_supported" or value is True
def supports(num_tokens: int) -> bool:
checked_tokens.append(num_tokens)
return fhmoe_supported
monkeypatch.setattr(
deepseek_v4_model.rocm_aiter_ops,
"fused_moe_supports_heterogeneous_shared_expert",
supports,
)
assert (
deepseek_v4_model._heterogeneous_shared_expert_enabled(
cast(VllmConfig, vllm_config)
)
is expected
)
assert checked_tokens == [1]
@pytest.mark.parametrize(
("weights_shape", "ids_shape", "message"),
[
((2, 7, 1), (2, 7, 1), "equal two-dimensional shapes"),
((2, 7), (3, 7), "equal two-dimensional shapes"),
((2, 6), (2, 6), "exactly 7 columns"),
((2, 8), (2, 8), "exactly 7 columns"),
],
)
def test_deepseek_v4_heterogeneous_fhmoe_rejects_invalid_routes(
weights_shape: tuple[int, ...],
ids_shape: tuple[int, ...],
message: str,
) -> None:
from vllm.models.deepseek_v4.amd.model import _validate_heterogeneous_routes
with pytest.raises(ValueError, match=message):
_validate_heterogeneous_routes(
torch.empty(weights_shape),
torch.empty(ids_shape, dtype=torch.int64),
experts_per_token=6,
)
def test_deepseek_v4_heterogeneous_fhmoe_accepts_appended_shared_route() -> None:
from vllm.models.deepseek_v4.amd.model import _validate_heterogeneous_routes
_validate_heterogeneous_routes(
torch.empty((2, 7)),
torch.empty((2, 7), dtype=torch.int64),
experts_per_token=6,
)
def test_aiter_fused_moe_validates_shared_expert_arguments() -> None:
from vllm._aiter_ops import (
_validate_rocm_aiter_fused_moe_shared_expert_args,
)
validate = _validate_rocm_aiter_fused_moe_shared_expert_args
shared_tensor = torch.empty(0)
assert not validate(None, None, None, None, -1)
with pytest.raises(ValueError, match="requires shared weights and scales"):
validate(None, None, None, None, 0)
with pytest.raises(ValueError, match="both shared weights and scales"):
validate(shared_tensor, None, None, None, 0)
with pytest.raises(ValueError, match="non-negative shared expert ID"):
validate(shared_tensor, shared_tensor, shared_tensor, shared_tensor, -1)
assert validate(shared_tensor, shared_tensor, shared_tensor, shared_tensor, 0)
def test_is_model_fused_shared_expert_compatible() -> None:
class MoE(nn.Module):
def __init__(self, enabled: bool) -> None:
+107
View File
@@ -211,6 +211,31 @@ def if_aiter_supported(func: Callable) -> Callable:
return wrapper
def _validate_rocm_aiter_fused_moe_shared_expert_args(
shared_w1: torch.Tensor | None,
shared_w2: torch.Tensor | None,
shared_w1_scale: torch.Tensor | None,
shared_w2_scale: torch.Tensor | None,
shared_expert_id: int,
) -> bool:
shared_tensors = (shared_w1, shared_w2, shared_w1_scale, shared_w2_scale)
has_shared_expert = any(tensor is not None for tensor in shared_tensors)
if has_shared_expert:
if not all(tensor is not None for tensor in shared_tensors):
raise ValueError(
"Heterogeneous fused MoE requires both shared weights and scales."
)
if shared_expert_id < 0:
raise ValueError(
"Heterogeneous fused MoE requires a non-negative shared expert ID."
)
elif shared_expert_id >= 0:
raise ValueError(
"A non-negative shared expert ID requires shared weights and scales."
)
return has_shared_expert
def _rocm_aiter_fused_moe_impl(
hidden_states: torch.Tensor,
w1: torch.Tensor,
@@ -236,7 +261,20 @@ def _rocm_aiter_fused_moe_impl(
swiglu_limit: float = 0.0,
beta: float | None = None,
linear_beta: float | None = None,
shared_w1: torch.Tensor | None = None,
shared_w2: torch.Tensor | None = None,
shared_w1_scale: torch.Tensor | None = None,
shared_w2_scale: torch.Tensor | None = None,
shared_expert_id: int = -1,
) -> torch.Tensor:
has_shared_expert = _validate_rocm_aiter_fused_moe_shared_expert_args(
shared_w1,
shared_w2,
shared_w1_scale,
shared_w2_scale,
shared_expert_id,
)
from aiter import ActivationType, QuantType
from aiter.fused_moe import fused_moe
@@ -253,6 +291,15 @@ def _rocm_aiter_fused_moe_impl(
extra_kwargs["beta"] = beta
extra_kwargs["linear_beta"] = linear_beta
if has_shared_expert:
extra_kwargs.update(
shared_w1=shared_w1,
shared_w2=shared_w2,
shared_w1_scale=shared_w1_scale,
shared_w2_scale=shared_w2_scale,
shared_expert_id=shared_expert_id,
)
return fused_moe(
hidden_states,
w1,
@@ -304,6 +351,11 @@ def _rocm_aiter_fused_moe_fake(
swiglu_limit: float = 0.0,
beta: float | None = None,
linear_beta: float | None = None,
shared_w1: torch.Tensor | None = None,
shared_w2: torch.Tensor | None = None,
shared_w1_scale: torch.Tensor | None = None,
shared_w2_scale: torch.Tensor | None = None,
shared_expert_id: int = -1,
) -> torch.Tensor:
if output_dtype is not None:
return torch.empty_like(hidden_states, dtype=output_dtype)
@@ -1997,6 +2049,44 @@ class rocm_aiter_ops:
return "gate_mode" in inspect.signature(fused_moe).parameters
@staticmethod
def _probe_dsv4_i384_fhmoe_capability(num_tokens: int) -> bool:
"""Probe AITER's CSV-backed DSV4 native-I384 FHMoE contract."""
if type(num_tokens) is not int or num_tokens <= 0:
return False
try:
import inspect
import aiter.fhmoe as fhmoe
from aiter.fused_moe import fused_moe
params = inspect.signature(fused_moe).parameters
supports_dsv4_i384_fhmoe = getattr(fhmoe, "supports_dsv4_i384_fhmoe", None)
if not callable(supports_dsv4_i384_fhmoe):
return False
supports_num_tokens = supports_dsv4_i384_fhmoe(num_tokens)
except Exception:
return False
return supports_num_tokens is True and all(
name in params
for name in (
"shared_w1",
"shared_w2",
"shared_w1_scale",
"shared_w2_scale",
"shared_expert_id",
)
)
@classmethod
@if_aiter_supported
@functools.cache
def fused_moe_supports_heterogeneous_shared_expert(cls, num_tokens: int) -> bool:
"""Whether AITER has DSV4 native-I384 configs through the given M."""
return cls._probe_dsv4_i384_fhmoe_capability(num_tokens)
@staticmethod
def register_ops_once() -> None:
global _OPS_REGISTERED
@@ -2465,6 +2555,11 @@ class rocm_aiter_ops:
swiglu_limit: float = 0.0,
beta: float | None = None,
linear_beta: float | None = None,
shared_w1: torch.Tensor | None = None,
shared_w2: torch.Tensor | None = None,
shared_w1_scale: torch.Tensor | None = None,
shared_w2_scale: torch.Tensor | None = None,
shared_expert_id: int = -1,
) -> torch.Tensor:
return torch.ops.vllm.rocm_aiter_fused_moe(
hidden_states,
@@ -2491,6 +2586,11 @@ class rocm_aiter_ops:
swiglu_limit,
beta,
linear_beta,
shared_w1,
shared_w2,
shared_w1_scale,
shared_w2_scale,
shared_expert_id,
)
@staticmethod
@@ -3074,6 +3174,13 @@ class rocm_aiter_ops:
return shuffle_scale_a16w4(tensor, num_experts, gate_up)
@staticmethod
def shuffle_scale(tensor: torch.Tensor) -> torch.Tensor:
"""Shuffle a non-gate/up E8M0 scale tensor for AITER."""
from aiter.ops.shuffle import shuffle_scale
return shuffle_scale(tensor)
@staticmethod
def shuffle_weights(
*tensors: torch.Tensor, layout: tuple[int, int] = (16, 16)
@@ -250,6 +250,11 @@ def rocm_aiter_fused_experts(
num_local_tokens: torch.Tensor | None = None,
output_dtype: torch.dtype | None = None,
moe_sorting_dispatch_policy: int = 0,
shared_w1: torch.Tensor | None = None,
shared_w2: torch.Tensor | None = None,
shared_w1_scale: torch.Tensor | None = None,
shared_w2_scale: torch.Tensor | None = None,
shared_expert_id: int = -1,
) -> torch.Tensor:
"""ROCm AITER fused MoE expert computation."""
if quant_config is None:
@@ -419,8 +424,18 @@ def rocm_aiter_fused_experts(
bias1=quant_config.w1_bias if quant_config.use_mxfp4_w4a16 else None,
bias2=quant_config.w2_bias if quant_config.use_mxfp4_w4a16 else None,
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
swiglu_limit=(
0.0
if moe_config.swiglu_limit is None
else float(moe_config.swiglu_limit)
),
beta=moe_config.activation_situ_beta,
linear_beta=moe_config.activation_situ_linear_beta,
shared_w1=shared_w1,
shared_w2=shared_w2,
shared_w1_scale=shared_w1_scale,
shared_w2_scale=shared_w2_scale,
shared_expert_id=shared_expert_id,
)
+377 -5
View File
@@ -1,7 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import typing
import weakref
from collections.abc import Callable, Iterable
from dataclasses import replace
from itertools import islice
import regex as re
@@ -20,9 +22,14 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
FusedMoEQuantConfig,
GateLinear,
RoutedExperts,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
rocm_aiter_fused_experts,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
@@ -41,6 +48,7 @@ from vllm.model_executor.layers.mhc import (
MHCPreOp,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
from vllm.model_executor.layers.quantization.utils.config_utils import (
is_shared_expert_quant_fse_compatible,
)
@@ -65,6 +73,7 @@ from vllm.model_executor.models.utils import (
)
from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950
from vllm.sequence import IntermediateTensors
logger = init_logger(__name__)
@@ -163,6 +172,336 @@ class DeepseekV4MLP(nn.Module):
return x
def _use_heterogeneous_fhmoe(num_tokens: int) -> bool:
return num_tokens > 0 and (
rocm_aiter_ops.fused_moe_supports_heterogeneous_shared_expert(num_tokens)
is True
)
def _validate_heterogeneous_routes(
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
experts_per_token: int,
) -> None:
if (
topk_weights.ndim != 2
or topk_ids.ndim != 2
or topk_weights.shape != topk_ids.shape
):
raise ValueError(
"Heterogeneous fused MoE routes must have equal two-dimensional shapes."
)
expected_columns = experts_per_token + 1
if topk_weights.shape[1] != expected_columns:
raise ValueError(
"Heterogeneous fused MoE routes must have exactly "
f"{expected_columns} columns, got {topk_weights.shape[1]}."
)
def _heterogeneous_shared_expert_enabled(vllm_config: VllmConfig) -> bool:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
parallel_config = vllm_config.parallel_config
reasons: list[str] = []
if not rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
return False
if not on_gfx950():
reasons.append("the device is not gfx950")
if parallel_config.enable_expert_parallel:
reasons.append("expert parallelism is enabled")
if parallel_config.enable_eplb:
reasons.append("EPLB is enabled")
if parallel_config.tensor_parallel_size != 8:
reasons.append("tensor parallelism is not 8")
if parallel_config.data_parallel_size != 1:
reasons.append("data parallelism is enabled")
if parallel_config.prefill_context_parallel_size != 1:
reasons.append("prefill context parallelism is enabled")
if vllm_config.kernel_config.moe_backend != "aiter":
reasons.append("the MoE backend is not AITER")
if vllm_config.model_config.dtype != torch.bfloat16:
reasons.append("the model dtype is not BF16")
if not rocm_aiter_ops.fused_moe_supports_heterogeneous_shared_expert(1):
reasons.append(
"AITER lacks CSV-backed DSV4 I384 heterogeneous shared-expert support"
)
expected_model = {
"n_routed_experts": 384,
"num_experts_per_tok": 6,
"n_shared_experts": 1,
"hidden_size": 7168,
"moe_intermediate_size": 3072,
"hidden_act": "silu",
"expert_dtype": "fp4",
"topk_method": "noaux_tc",
}
for name, expected in expected_model.items():
if getattr(config, name, None) != expected:
reasons.append(f"{name} is not {expected!r}")
if quant_config is None or quant_config.get_name() != "deepseek_v4_fp8":
reasons.append("the DeepSeek V4 FP8 quantization config is not active")
else:
if getattr(quant_config, "moe_quant_algo", "").upper() == "NVFP4":
reasons.append("routed experts use NVFP4")
if getattr(quant_config, "weight_block_size", None) != [128, 128]:
reasons.append("shared experts are not block-128 FP8")
if not getattr(quant_config, "is_checkpoint_fp8_serialized", False):
reasons.append("shared experts are not serialized as FP8")
if not getattr(quant_config, "is_scale_e8m0", False):
reasons.append("shared-expert scales are not E8M0")
if getattr(quant_config, "ignored_layers", None):
reasons.append("the quantization config has ignored layers")
offload_config = getattr(vllm_config, "offload_config", None)
if offload_config is not None and (
getattr(getattr(offload_config, "uva", None), "cpu_offload_gb", 0) > 0
or getattr(
getattr(offload_config, "prefetch", None),
"offload_group_size",
0,
)
> 0
):
reasons.append("weight offloading is enabled")
if reasons:
logger.debug_once(
"DeepSeek V4 heterogeneous shared-expert fusion is unavailable: %s. "
"Continuing with the existing shared-expert path.",
"; ".join(reasons),
)
return False
logger.info_once(
"Using AITER DeepSeek V4 heterogeneous MoE for CSV-covered MoE input "
"rows (M) per invocation (native intermediate width 384)."
)
return True
@torch.no_grad()
def _prepare_native_fp8_shared_expert(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
intermediate_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Expand native block-128 E8M0 scales to FHMoE's 1x32 layout."""
if w13.dtype != torch.float8_e4m3fn or w2.dtype != torch.float8_e4m3fn:
raise ValueError("Heterogeneous shared-expert weights must be FP8 E4M3.")
hidden_size = w13.shape[1]
if w13.shape != (2 * intermediate_size, hidden_size):
raise ValueError(f"Unexpected shared W13 shape {tuple(w13.shape)}.")
if w2.shape != (hidden_size, intermediate_size):
raise ValueError(f"Unexpected shared W2 shape {tuple(w2.shape)}.")
def scale_bytes(scale: torch.Tensor) -> torch.Tensor:
if scale.dtype == torch.float8_e8m0fnu:
return scale.contiguous().view(torch.uint8)
if scale.dtype == torch.uint8:
return scale.contiguous()
if scale.dtype == torch.float32:
bits = scale.contiguous().view(torch.int32)
exponent = (bits >> 23).to(torch.uint8)
if not torch.equal(bits, exponent.to(torch.int32) << 23):
raise ValueError("FP32 shared-expert scales are not exact E8M0.")
return exponent
raise ValueError(f"Unsupported shared-expert scale dtype {scale.dtype}.")
w13_scale = scale_bytes(w13_scale)
w2_scale = scale_bytes(w2_scale)
expected_w13_scale = (2 * intermediate_size // 128, hidden_size // 128)
expected_w2_scale = (hidden_size // 128, intermediate_size // 128)
if tuple(w13_scale.shape) != expected_w13_scale:
raise ValueError(
f"Expected shared W13 scale shape {expected_w13_scale}, got "
f"{tuple(w13_scale.shape)}."
)
if tuple(w2_scale.shape) != expected_w2_scale:
raise ValueError(
f"Expected shared W2 scale shape {expected_w2_scale}, got "
f"{tuple(w2_scale.shape)}."
)
w13_scale = w13_scale.repeat_interleave(128, dim=0).repeat_interleave(4, dim=1)
w2_scale = w2_scale.repeat_interleave(128, dim=0).repeat_interleave(4, dim=1)
scale_cols = ((w2_scale.shape[1] + 7) // 8) * 8
if scale_cols != w2_scale.shape[1]:
padded_w2_scale = torch.full(
(w2_scale.shape[0], scale_cols),
0x7F,
dtype=torch.uint8,
device=w2_scale.device,
)
padded_w2_scale[:, : w2_scale.shape[1]].copy_(w2_scale)
w2_scale = padded_w2_scale
return (
w13.unsqueeze(0),
w2.unsqueeze(0),
w13_scale.view(torch.float8_e8m0fnu),
w2_scale.view(torch.float8_e8m0fnu),
)
class DeepseekV4HeterogeneousMxfp4MoEMethod(Mxfp4MoEMethod):
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
super().process_weights_after_loading(layer)
assert isinstance(layer, DeepseekV4HeterogeneousSharedRoutedExperts)
layer.prepare_heterogeneous_shared_expert()
class DeepseekV4HeterogeneousSharedRoutedExperts(RoutedExperts):
def __init__(
self,
*args,
shared_expert: DeepseekV4MLP,
shared_expert_id: int,
**kwargs,
):
self._shared_expert_ref = weakref.ref(shared_expert)
self.shared_expert_id = shared_expert_id
self._routed_quant_config: FusedMoEQuantConfig | None = None
super().__init__(*args, **kwargs)
if self.expert_map_manager.num_fused_shared_experts != 1:
raise ValueError("Heterogeneous fusion requires one appended expert.")
if self.shared_expert_id != self.global_num_experts:
raise ValueError("The shared expert ID must name the appended slot.")
self.register_buffer("shared_w1", None, persistent=False)
self.register_buffer("shared_w2", None, persistent=False)
self.register_buffer("shared_w1_scale", None, persistent=False)
self.register_buffer("shared_w2_scale", None, persistent=False)
def _get_quant_method(self, prefix, quant_config, moe_config):
quant_method = super()._get_quant_method(prefix, quant_config, moe_config)
if not isinstance(quant_method, Mxfp4MoEMethod):
raise ValueError("Heterogeneous fusion requires MXFP4 routed experts.")
return DeepseekV4HeterogeneousMxfp4MoEMethod(moe_config)
@torch.no_grad()
def prepare_heterogeneous_shared_expert(self) -> None:
shared_expert = self._shared_expert_ref()
if shared_expert is None:
raise RuntimeError("The native shared-expert module was released.")
shared = _prepare_native_fp8_shared_expert(
shared_expert.gate_up_proj.weight,
shared_expert.down_proj.weight,
shared_expert.gate_up_proj.weight_scale_inv,
shared_expert.down_proj.weight_scale_inv,
self.moe_config.intermediate_size_per_partition,
)
self.shared_w1 = rocm_aiter_ops.shuffle_weight_a16w4(shared[0], 16, True)
self.shared_w2 = rocm_aiter_ops.shuffle_weight_a16w4(shared[1], 16, False)
self.shared_w1_scale = rocm_aiter_ops.shuffle_scale_a16w4(shared[2], 1, True)
self.shared_w2_scale = rocm_aiter_ops.shuffle_scale(shared[3])
self._ensure_moe_quant_config_init()
quant_config = self.quant_method.moe_quant_config
if quant_config is None:
raise RuntimeError("The routed MXFP4 quantization config is unavailable.")
if quant_config.w1_scale is None or quant_config.w2_scale is None:
raise RuntimeError("The routed MXFP4 scales are unavailable.")
def routed_scales(scale: torch.Tensor) -> torch.Tensor:
scale = scale.reshape(self.local_num_experts, -1, scale.shape[-1])[
: self.shared_expert_id
]
return scale.reshape(-1, scale.shape[-1]).contiguous()
self._routed_quant_config = replace(
quant_config,
_w1=replace(
quant_config._w1,
scale=routed_scales(quant_config.w1_scale),
),
_w2=replace(
quant_config._w2,
scale=routed_scales(quant_config.w2_scale),
),
)
def forward_modular(
self,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts=None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
if shared_experts is not None or shared_experts_input is not None:
raise ValueError("The heterogeneous expert owns the shared path.")
_validate_heterogeneous_routes(
topk_weights,
topk_ids,
self.moe_config.experts_per_token,
)
if any(
tensor is None
for tensor in (
self.shared_w1,
self.shared_w2,
self.shared_w1_scale,
self.shared_w2_scale,
)
):
raise RuntimeError("Heterogeneous shared weights were not prepared.")
if _use_heterogeneous_fhmoe(x.shape[0]):
self._ensure_moe_quant_config_init()
quant_config = self.quant_method.moe_quant_config
if quant_config is None:
raise RuntimeError("The routed MXFP4 config is unavailable.")
return rocm_aiter_fused_experts(
hidden_states=x,
w1=self.w13_weight,
w2=self.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
moe_config=self.moe_config,
activation=self.activation,
expert_mask=self.expert_map,
quant_config=quant_config,
output_dtype=x.dtype,
moe_sorting_dispatch_policy=(rocm_aiter_ops.get_moe_dispatch_policy()),
shared_w1=self.shared_w1,
shared_w2=self.shared_w2,
shared_w1_scale=self.shared_w1_scale,
shared_w2_scale=self.shared_w2_scale,
shared_expert_id=self.shared_expert_id,
)
shared_expert = self._shared_expert_ref()
if shared_expert is None or self._routed_quant_config is None:
raise RuntimeError("The separate shared-expert fallback is unavailable.")
shared_out = shared_expert(x)
routed_w1 = self.w13_weight[: self.shared_expert_id]
routed_w2 = self.w2_weight[: self.shared_expert_id]
routed_w1.is_shuffled = True
routed_w2.is_shuffled = True
routed = rocm_aiter_fused_experts(
hidden_states=x,
w1=routed_w1,
w2=routed_w2,
topk_weights=topk_weights[:, :-1].contiguous(),
topk_ids=topk_ids[:, :-1].contiguous(),
moe_config=self.moe_config,
activation=self.activation,
expert_mask=self.expert_map,
quant_config=self._routed_quant_config,
output_dtype=x.dtype,
moe_sorting_dispatch_policy=rocm_aiter_ops.get_moe_dispatch_policy(),
)
return routed + shared_out
def _fuse_shared_experts_enabled(config) -> bool:
return bool(
getattr(config, "n_shared_experts", None)
@@ -176,6 +515,7 @@ class DeepseekV4MoE(nn.Module):
self,
vllm_config: VllmConfig,
prefix: str = "",
fuse_heterogeneous_shared_expert: bool = False,
):
super().__init__()
@@ -230,7 +570,12 @@ class DeepseekV4MoE(nn.Module):
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
# is checked to enable FSE for DeepSeek-v4, despite AITER not being used.
# This should be cleaned up and use `resolve_layer_fused_shared_expert`.
fse_requested = _fuse_shared_experts_enabled(config)
self.fuse_heterogeneous_shared_expert = fuse_heterogeneous_shared_expert
fse_requested = (
_fuse_shared_experts_enabled(config)
and not self.fuse_heterogeneous_shared_expert
)
fse_compatible = False
if fse_requested:
fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible(
quant_config,
@@ -267,12 +612,15 @@ class DeepseekV4MoE(nn.Module):
self.experts_start_idx = self.tp_rank * self.n_local_experts
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
fuse_shared_into_routed = (
self.is_fused_shared_expert_enabled or self.fuse_heterogeneous_shared_expert
)
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
shared_experts=(None if fuse_shared_into_routed else self.shared_experts),
n_shared_experts=(
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
config.n_shared_experts if fuse_shared_into_routed else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
fuse_shared_experts=fuse_shared_into_routed,
gate=self.gate,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
@@ -287,6 +635,19 @@ class DeepseekV4MoE(nn.Module):
hash_indices_table=self.gate.tid2eid,
swiglu_limit=self.swiglu_limit,
router_logits_dtype=torch.float32,
routed_experts_cls=(
DeepseekV4HeterogeneousSharedRoutedExperts
if self.fuse_heterogeneous_shared_expert
else None
),
routed_experts_args=(
{
"shared_expert": self.shared_experts,
"shared_expert_id": config.n_routed_experts,
}
if self.fuse_heterogeneous_shared_expert
else None
),
)
def forward(
@@ -316,6 +677,7 @@ class DeepseekV4DecoderLayer(nn.Module):
prefix,
topk_indices_buffer: torch.Tensor | None = None,
aux_stream_list: list[torch.cuda.Stream] | None = None,
fuse_heterogeneous_shared_expert: bool = False,
):
super().__init__()
@@ -333,7 +695,11 @@ class DeepseekV4DecoderLayer(nn.Module):
topk_indices_buffer=topk_indices_buffer,
aux_stream_list=aux_stream_list,
)
self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn")
self.ffn = DeepseekV4MoE(
vllm_config,
prefix=f"{prefix}.ffn",
fuse_heterogeneous_shared_expert=fuse_heterogeneous_shared_expert,
)
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
@@ -598,6 +964,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
else:
self.embed_tokens = PPMissingLayer()
self.fuse_heterogeneous_shared_expert = _heterogeneous_shared_expert_enabled(
vllm_config
)
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: DeepseekV4DecoderLayer(
@@ -605,6 +974,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
prefix=prefix,
topk_indices_buffer=self.topk_indices_buffer,
aux_stream_list=aux_stream_list,
fuse_heterogeneous_shared_expert=(
self.fuse_heterogeneous_shared_expert
),
),
prefix=f"{prefix}.layers",
)