diff --git a/tests/model_executor/layers/test_fused_shared_expert.py b/tests/model_executor/layers/test_fused_shared_expert.py index cb935811e8..37693f18c4 100644 --- a/tests/model_executor/layers/test_fused_shared_expert.py +++ b/tests/model_executor/layers/test_fused_shared_expert.py @@ -2,8 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import importlib +import inspect +import sys from copy import deepcopy -from types import SimpleNamespace +from types import ModuleType, SimpleNamespace from typing import Any, cast from unittest.mock import patch @@ -414,6 +416,343 @@ def test_deepseek_v4_shared_expert_fse_uses_mtp_quantization_config_prefix( assert reason is None +def test_deepseek_v4_heterogeneous_fhmoe_keeps_native_intermediate_width() -> None: + from vllm.models.deepseek_v4.amd.model import _prepare_native_fp8_shared_expert + + hidden_size = 7168 + intermediate_size = 384 + w13 = torch.empty((2 * intermediate_size, hidden_size), dtype=torch.float8_e4m3fn) + w2 = torch.empty((hidden_size, intermediate_size), dtype=torch.float8_e4m3fn) + w13_scale_bytes = (torch.arange(6 * 56).reshape(6, 56).remainder(20) + 0x60).to( + torch.uint8 + ) + w2_scale_bytes = (torch.arange(56 * 3).reshape(56, 3).remainder(30) + 0x50).to( + torch.uint8 + ) + w13_scale = w13_scale_bytes.view(torch.float8_e8m0fnu) + w2_scale = w2_scale_bytes.view(torch.float8_e8m0fnu) + + prepared = _prepare_native_fp8_shared_expert( + w13, w2, w13_scale, w2_scale, intermediate_size + ) + + assert prepared[0].shape == (1, 768, 7168) + assert prepared[1].shape == (1, 7168, 384) + assert prepared[2].shape == (768, 224) + assert prepared[3].shape == (7168, 16) + expected_w13_scale = w13_scale_bytes.repeat_interleave( + 128, dim=0 + ).repeat_interleave(4, dim=1) + expected_w2_scale = w2_scale_bytes.repeat_interleave(128, dim=0).repeat_interleave( + 4, dim=1 + ) + assert torch.equal(prepared[2].view(torch.uint8), expected_w13_scale) + assert torch.equal(prepared[3].view(torch.uint8)[:, :12], expected_w2_scale) + assert torch.all(prepared[3].view(torch.uint8)[:, 12:] == 0x7F) + + +@pytest.mark.parametrize( + ("num_tokens", "supported_through", "expected"), + [ + (0, 4096, False), + (1, 4096, True), + (1536, 4096, True), + (2048, 4096, True), + (2049, 4096, True), + (4096, 4096, True), + (4097, 4096, False), + (1536, 0, False), + ], +) +def test_deepseek_v4_heterogeneous_fhmoe_token_policy( + monkeypatch: pytest.MonkeyPatch, + num_tokens: int, + supported_through: int, + expected: bool, +) -> None: + from vllm.models.deepseek_v4.amd import model as deepseek_v4_model + + checked_tokens: list[int] = [] + + def supports(num_tokens: int) -> bool: + checked_tokens.append(num_tokens) + return num_tokens <= supported_through + + monkeypatch.setattr( + deepseek_v4_model.rocm_aiter_ops, + "fused_moe_supports_heterogeneous_shared_expert", + supports, + ) + + assert deepseek_v4_model._use_heterogeneous_fhmoe(num_tokens) is expected + assert checked_tokens == ([] if num_tokens == 0 else [num_tokens]) + + +def _supported_fhmoe_signature( + shared_w1=None, + shared_w2=None, + shared_w1_scale=None, + shared_w2_scale=None, + shared_expert_id=-1, +) -> None: + pass + + +def _incomplete_fhmoe_signature( + shared_w1=None, + shared_w2=None, + shared_w1_scale=None, + shared_w2_scale=None, +) -> None: + pass + + +def _install_fake_aiter_fhmoe( + monkeypatch: pytest.MonkeyPatch, + supports_dsv4_i384_fhmoe: object, + fused_moe: object, +) -> None: + fake_aiter = ModuleType("aiter") + fake_aiter.__path__ = [] + fake_fhmoe = ModuleType("aiter.fhmoe") + if supports_dsv4_i384_fhmoe is not None: + fake_fhmoe.__dict__["supports_dsv4_i384_fhmoe"] = supports_dsv4_i384_fhmoe + fake_fused_moe = ModuleType("aiter.fused_moe") + fake_fused_moe.__dict__["fused_moe"] = fused_moe + fake_aiter.__dict__["fhmoe"] = fake_fhmoe + fake_aiter.__dict__["fused_moe"] = fake_fused_moe + monkeypatch.setitem(sys.modules, "aiter", fake_aiter) + monkeypatch.setitem(sys.modules, "aiter.fhmoe", fake_fhmoe) + monkeypatch.setitem(sys.modules, "aiter.fused_moe", fake_fused_moe) + + +def _supports_fhmoe_through_2047(max_tokens: int) -> bool: + return max_tokens <= 2047 + + +def _supports_fhmoe_through_2048(max_tokens: int) -> bool: + return max_tokens <= 2048 + + +def _supports_fhmoe_through_4096(max_tokens: int) -> bool: + return max_tokens <= 4096 + + +def _raises_fhmoe_config_error(max_tokens: int) -> bool: + raise OSError + + +def _returns_truthy_non_bool(max_tokens: int) -> int: + return 1 + + +@pytest.mark.parametrize( + ("num_tokens", "capability", "fused_moe", "expected"), + [ + (0, _supports_fhmoe_through_2048, _supported_fhmoe_signature, False), + (True, _supports_fhmoe_through_2048, _supported_fhmoe_signature, False), + (2048, None, _supported_fhmoe_signature, False), + (2048, True, _supported_fhmoe_signature, False), + (2048, _raises_fhmoe_config_error, _supported_fhmoe_signature, False), + (2048, _returns_truthy_non_bool, _supported_fhmoe_signature, False), + (2048, _supports_fhmoe_through_2047, _supported_fhmoe_signature, False), + (2048, _supports_fhmoe_through_2048, _incomplete_fhmoe_signature, False), + (2048, _supports_fhmoe_through_2048, _supported_fhmoe_signature, True), + (2049, _supports_fhmoe_through_2048, _supported_fhmoe_signature, False), + (4096, _supports_fhmoe_through_4096, _supported_fhmoe_signature, True), + (4097, _supports_fhmoe_through_4096, _supported_fhmoe_signature, False), + ], +) +def test_deepseek_v4_heterogeneous_fhmoe_aiter_capability( + monkeypatch: pytest.MonkeyPatch, + num_tokens: object, + capability: object, + fused_moe: object, + expected: bool, +) -> None: + from vllm._aiter_ops import rocm_aiter_ops + + _install_fake_aiter_fhmoe(monkeypatch, capability, fused_moe) + + assert ( + rocm_aiter_ops._probe_dsv4_i384_fhmoe_capability(cast(int, num_tokens)) + is expected + ) + + +def test_deepseek_v4_heterogeneous_fhmoe_aiter_capability_catches_import_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from vllm._aiter_ops import rocm_aiter_ops + + fake_aiter = ModuleType("aiter") + fake_aiter.__path__ = [] + monkeypatch.setitem(sys.modules, "aiter", fake_aiter) + monkeypatch.setitem(sys.modules, "aiter.fhmoe", None) + + assert not rocm_aiter_ops._probe_dsv4_i384_fhmoe_capability(2048) + + +@pytest.mark.parametrize("error", [TypeError, ValueError]) +def test_deepseek_v4_heterogeneous_fhmoe_aiter_capability_catches_signature_error( + monkeypatch: pytest.MonkeyPatch, + error: type[Exception], +) -> None: + from vllm._aiter_ops import rocm_aiter_ops + + _install_fake_aiter_fhmoe( + monkeypatch, _supports_fhmoe_through_2048, _supported_fhmoe_signature + ) + + def raise_signature_error(_: object) -> inspect.Signature: + raise error + + monkeypatch.setattr(inspect, "signature", raise_signature_error) + + assert not rocm_aiter_ops._probe_dsv4_i384_fhmoe_capability(2048) + + +@pytest.mark.parametrize( + ("setting", "value", "expected"), + [ + ("data_parallel_size", 1, True), + ("data_parallel_size", 2, False), + ("prefill_context_parallel_size", 2, False), + ("topk_method", "greedy", False), + ("fhmoe_supported", False, False), + ], +) +def test_deepseek_v4_heterogeneous_fhmoe_compatibility_gates( + monkeypatch: pytest.MonkeyPatch, + setting: str, + value: object, + expected: bool, +) -> None: + from vllm.models.deepseek_v4.amd import model as deepseek_v4_model + + hf_config = SimpleNamespace( + n_routed_experts=384, + num_experts_per_tok=6, + n_shared_experts=1, + hidden_size=7168, + moe_intermediate_size=3072, + hidden_act="silu", + expert_dtype="fp4", + topk_method="noaux_tc", + ) + parallel_config = SimpleNamespace( + enable_expert_parallel=False, + enable_eplb=False, + tensor_parallel_size=8, + data_parallel_size=1, + prefill_context_parallel_size=1, + ) + if setting == "topk_method": + hf_config.topk_method = value + elif setting != "fhmoe_supported": + setattr(parallel_config, setting, value) + + quant_config = SimpleNamespace( + get_name=lambda: "deepseek_v4_fp8", + moe_quant_algo="", + weight_block_size=[128, 128], + is_checkpoint_fp8_serialized=True, + is_scale_e8m0=True, + ignored_layers=None, + ) + vllm_config = SimpleNamespace( + model_config=SimpleNamespace(hf_config=hf_config, dtype=torch.bfloat16), + quant_config=quant_config, + parallel_config=parallel_config, + kernel_config=SimpleNamespace(moe_backend="aiter"), + offload_config=None, + ) + monkeypatch.setattr(deepseek_v4_model.current_platform, "is_rocm", lambda: True) + monkeypatch.setattr( + deepseek_v4_model.envs, + "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", + True, + ) + monkeypatch.setattr(deepseek_v4_model, "on_gfx950", lambda: True) + monkeypatch.setattr( + deepseek_v4_model.rocm_aiter_ops, + "is_fusion_moe_shared_experts_enabled", + lambda: True, + ) + checked_tokens: list[int] = [] + fhmoe_supported = setting != "fhmoe_supported" or value is True + + def supports(num_tokens: int) -> bool: + checked_tokens.append(num_tokens) + return fhmoe_supported + + monkeypatch.setattr( + deepseek_v4_model.rocm_aiter_ops, + "fused_moe_supports_heterogeneous_shared_expert", + supports, + ) + + assert ( + deepseek_v4_model._heterogeneous_shared_expert_enabled( + cast(VllmConfig, vllm_config) + ) + is expected + ) + assert checked_tokens == [1] + + +@pytest.mark.parametrize( + ("weights_shape", "ids_shape", "message"), + [ + ((2, 7, 1), (2, 7, 1), "equal two-dimensional shapes"), + ((2, 7), (3, 7), "equal two-dimensional shapes"), + ((2, 6), (2, 6), "exactly 7 columns"), + ((2, 8), (2, 8), "exactly 7 columns"), + ], +) +def test_deepseek_v4_heterogeneous_fhmoe_rejects_invalid_routes( + weights_shape: tuple[int, ...], + ids_shape: tuple[int, ...], + message: str, +) -> None: + from vllm.models.deepseek_v4.amd.model import _validate_heterogeneous_routes + + with pytest.raises(ValueError, match=message): + _validate_heterogeneous_routes( + torch.empty(weights_shape), + torch.empty(ids_shape, dtype=torch.int64), + experts_per_token=6, + ) + + +def test_deepseek_v4_heterogeneous_fhmoe_accepts_appended_shared_route() -> None: + from vllm.models.deepseek_v4.amd.model import _validate_heterogeneous_routes + + _validate_heterogeneous_routes( + torch.empty((2, 7)), + torch.empty((2, 7), dtype=torch.int64), + experts_per_token=6, + ) + + +def test_aiter_fused_moe_validates_shared_expert_arguments() -> None: + from vllm._aiter_ops import ( + _validate_rocm_aiter_fused_moe_shared_expert_args, + ) + + validate = _validate_rocm_aiter_fused_moe_shared_expert_args + shared_tensor = torch.empty(0) + + assert not validate(None, None, None, None, -1) + with pytest.raises(ValueError, match="requires shared weights and scales"): + validate(None, None, None, None, 0) + with pytest.raises(ValueError, match="both shared weights and scales"): + validate(shared_tensor, None, None, None, 0) + with pytest.raises(ValueError, match="non-negative shared expert ID"): + validate(shared_tensor, shared_tensor, shared_tensor, shared_tensor, -1) + assert validate(shared_tensor, shared_tensor, shared_tensor, shared_tensor, 0) + + def test_is_model_fused_shared_expert_compatible() -> None: class MoE(nn.Module): def __init__(self, enabled: bool) -> None: diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index cde07eb8eb..1b9bf15735 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -211,6 +211,31 @@ def if_aiter_supported(func: Callable) -> Callable: return wrapper +def _validate_rocm_aiter_fused_moe_shared_expert_args( + shared_w1: torch.Tensor | None, + shared_w2: torch.Tensor | None, + shared_w1_scale: torch.Tensor | None, + shared_w2_scale: torch.Tensor | None, + shared_expert_id: int, +) -> bool: + shared_tensors = (shared_w1, shared_w2, shared_w1_scale, shared_w2_scale) + has_shared_expert = any(tensor is not None for tensor in shared_tensors) + if has_shared_expert: + if not all(tensor is not None for tensor in shared_tensors): + raise ValueError( + "Heterogeneous fused MoE requires both shared weights and scales." + ) + if shared_expert_id < 0: + raise ValueError( + "Heterogeneous fused MoE requires a non-negative shared expert ID." + ) + elif shared_expert_id >= 0: + raise ValueError( + "A non-negative shared expert ID requires shared weights and scales." + ) + return has_shared_expert + + def _rocm_aiter_fused_moe_impl( hidden_states: torch.Tensor, w1: torch.Tensor, @@ -236,7 +261,20 @@ def _rocm_aiter_fused_moe_impl( swiglu_limit: float = 0.0, beta: float | None = None, linear_beta: float | None = None, + shared_w1: torch.Tensor | None = None, + shared_w2: torch.Tensor | None = None, + shared_w1_scale: torch.Tensor | None = None, + shared_w2_scale: torch.Tensor | None = None, + shared_expert_id: int = -1, ) -> torch.Tensor: + has_shared_expert = _validate_rocm_aiter_fused_moe_shared_expert_args( + shared_w1, + shared_w2, + shared_w1_scale, + shared_w2_scale, + shared_expert_id, + ) + from aiter import ActivationType, QuantType from aiter.fused_moe import fused_moe @@ -253,6 +291,15 @@ def _rocm_aiter_fused_moe_impl( extra_kwargs["beta"] = beta extra_kwargs["linear_beta"] = linear_beta + if has_shared_expert: + extra_kwargs.update( + shared_w1=shared_w1, + shared_w2=shared_w2, + shared_w1_scale=shared_w1_scale, + shared_w2_scale=shared_w2_scale, + shared_expert_id=shared_expert_id, + ) + return fused_moe( hidden_states, w1, @@ -304,6 +351,11 @@ def _rocm_aiter_fused_moe_fake( swiglu_limit: float = 0.0, beta: float | None = None, linear_beta: float | None = None, + shared_w1: torch.Tensor | None = None, + shared_w2: torch.Tensor | None = None, + shared_w1_scale: torch.Tensor | None = None, + shared_w2_scale: torch.Tensor | None = None, + shared_expert_id: int = -1, ) -> torch.Tensor: if output_dtype is not None: return torch.empty_like(hidden_states, dtype=output_dtype) @@ -1997,6 +2049,44 @@ class rocm_aiter_ops: return "gate_mode" in inspect.signature(fused_moe).parameters + @staticmethod + def _probe_dsv4_i384_fhmoe_capability(num_tokens: int) -> bool: + """Probe AITER's CSV-backed DSV4 native-I384 FHMoE contract.""" + if type(num_tokens) is not int or num_tokens <= 0: + return False + + try: + import inspect + + import aiter.fhmoe as fhmoe + from aiter.fused_moe import fused_moe + + params = inspect.signature(fused_moe).parameters + supports_dsv4_i384_fhmoe = getattr(fhmoe, "supports_dsv4_i384_fhmoe", None) + if not callable(supports_dsv4_i384_fhmoe): + return False + supports_num_tokens = supports_dsv4_i384_fhmoe(num_tokens) + except Exception: + return False + + return supports_num_tokens is True and all( + name in params + for name in ( + "shared_w1", + "shared_w2", + "shared_w1_scale", + "shared_w2_scale", + "shared_expert_id", + ) + ) + + @classmethod + @if_aiter_supported + @functools.cache + def fused_moe_supports_heterogeneous_shared_expert(cls, num_tokens: int) -> bool: + """Whether AITER has DSV4 native-I384 configs through the given M.""" + return cls._probe_dsv4_i384_fhmoe_capability(num_tokens) + @staticmethod def register_ops_once() -> None: global _OPS_REGISTERED @@ -2465,6 +2555,11 @@ class rocm_aiter_ops: swiglu_limit: float = 0.0, beta: float | None = None, linear_beta: float | None = None, + shared_w1: torch.Tensor | None = None, + shared_w2: torch.Tensor | None = None, + shared_w1_scale: torch.Tensor | None = None, + shared_w2_scale: torch.Tensor | None = None, + shared_expert_id: int = -1, ) -> torch.Tensor: return torch.ops.vllm.rocm_aiter_fused_moe( hidden_states, @@ -2491,6 +2586,11 @@ class rocm_aiter_ops: swiglu_limit, beta, linear_beta, + shared_w1, + shared_w2, + shared_w1_scale, + shared_w2_scale, + shared_expert_id, ) @staticmethod @@ -3074,6 +3174,13 @@ class rocm_aiter_ops: return shuffle_scale_a16w4(tensor, num_experts, gate_up) + @staticmethod + def shuffle_scale(tensor: torch.Tensor) -> torch.Tensor: + """Shuffle a non-gate/up E8M0 scale tensor for AITER.""" + from aiter.ops.shuffle import shuffle_scale + + return shuffle_scale(tensor) + @staticmethod def shuffle_weights( *tensors: torch.Tensor, layout: tuple[int, int] = (16, 16) diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 066aba97cd..cb4225c92a 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -250,6 +250,11 @@ def rocm_aiter_fused_experts( num_local_tokens: torch.Tensor | None = None, output_dtype: torch.dtype | None = None, moe_sorting_dispatch_policy: int = 0, + shared_w1: torch.Tensor | None = None, + shared_w2: torch.Tensor | None = None, + shared_w1_scale: torch.Tensor | None = None, + shared_w2_scale: torch.Tensor | None = None, + shared_expert_id: int = -1, ) -> torch.Tensor: """ROCm AITER fused MoE expert computation.""" if quant_config is None: @@ -419,8 +424,18 @@ def rocm_aiter_fused_experts( bias1=quant_config.w1_bias if quant_config.use_mxfp4_w4a16 else None, bias2=quant_config.w2_bias if quant_config.use_mxfp4_w4a16 else None, moe_sorting_dispatch_policy=moe_sorting_dispatch_policy, + swiglu_limit=( + 0.0 + if moe_config.swiglu_limit is None + else float(moe_config.swiglu_limit) + ), beta=moe_config.activation_situ_beta, linear_beta=moe_config.activation_situ_linear_beta, + shared_w1=shared_w1, + shared_w2=shared_w2, + shared_w1_scale=shared_w1_scale, + shared_w2_scale=shared_w2_scale, + shared_expert_id=shared_expert_id, ) diff --git a/vllm/models/deepseek_v4/amd/model.py b/vllm/models/deepseek_v4/amd/model.py index 177775e413..903635ffe7 100644 --- a/vllm/models/deepseek_v4/amd/model.py +++ b/vllm/models/deepseek_v4/amd/model.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import typing +import weakref from collections.abc import Callable, Iterable +from dataclasses import replace from itertools import islice import regex as re @@ -20,9 +22,14 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, + FusedMoEQuantConfig, GateLinear, + RoutedExperts, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + rocm_aiter_fused_experts, +) from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, ) @@ -41,6 +48,7 @@ from vllm.model_executor.layers.mhc import ( MHCPreOp, ) from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod from vllm.model_executor.layers.quantization.utils.config_utils import ( is_shared_expert_quant_fse_compatible, ) @@ -65,6 +73,7 @@ from vllm.model_executor.models.utils import ( ) from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention from vllm.platforms import current_platform +from vllm.platforms.rocm import on_gfx950 from vllm.sequence import IntermediateTensors logger = init_logger(__name__) @@ -163,6 +172,336 @@ class DeepseekV4MLP(nn.Module): return x +def _use_heterogeneous_fhmoe(num_tokens: int) -> bool: + return num_tokens > 0 and ( + rocm_aiter_ops.fused_moe_supports_heterogeneous_shared_expert(num_tokens) + is True + ) + + +def _validate_heterogeneous_routes( + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + experts_per_token: int, +) -> None: + if ( + topk_weights.ndim != 2 + or topk_ids.ndim != 2 + or topk_weights.shape != topk_ids.shape + ): + raise ValueError( + "Heterogeneous fused MoE routes must have equal two-dimensional shapes." + ) + expected_columns = experts_per_token + 1 + if topk_weights.shape[1] != expected_columns: + raise ValueError( + "Heterogeneous fused MoE routes must have exactly " + f"{expected_columns} columns, got {topk_weights.shape[1]}." + ) + + +def _heterogeneous_shared_expert_enabled(vllm_config: VllmConfig) -> bool: + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + parallel_config = vllm_config.parallel_config + reasons: list[str] = [] + + if not rocm_aiter_ops.is_fusion_moe_shared_experts_enabled(): + return False + if not on_gfx950(): + reasons.append("the device is not gfx950") + if parallel_config.enable_expert_parallel: + reasons.append("expert parallelism is enabled") + if parallel_config.enable_eplb: + reasons.append("EPLB is enabled") + if parallel_config.tensor_parallel_size != 8: + reasons.append("tensor parallelism is not 8") + if parallel_config.data_parallel_size != 1: + reasons.append("data parallelism is enabled") + if parallel_config.prefill_context_parallel_size != 1: + reasons.append("prefill context parallelism is enabled") + if vllm_config.kernel_config.moe_backend != "aiter": + reasons.append("the MoE backend is not AITER") + if vllm_config.model_config.dtype != torch.bfloat16: + reasons.append("the model dtype is not BF16") + if not rocm_aiter_ops.fused_moe_supports_heterogeneous_shared_expert(1): + reasons.append( + "AITER lacks CSV-backed DSV4 I384 heterogeneous shared-expert support" + ) + + expected_model = { + "n_routed_experts": 384, + "num_experts_per_tok": 6, + "n_shared_experts": 1, + "hidden_size": 7168, + "moe_intermediate_size": 3072, + "hidden_act": "silu", + "expert_dtype": "fp4", + "topk_method": "noaux_tc", + } + for name, expected in expected_model.items(): + if getattr(config, name, None) != expected: + reasons.append(f"{name} is not {expected!r}") + + if quant_config is None or quant_config.get_name() != "deepseek_v4_fp8": + reasons.append("the DeepSeek V4 FP8 quantization config is not active") + else: + if getattr(quant_config, "moe_quant_algo", "").upper() == "NVFP4": + reasons.append("routed experts use NVFP4") + if getattr(quant_config, "weight_block_size", None) != [128, 128]: + reasons.append("shared experts are not block-128 FP8") + if not getattr(quant_config, "is_checkpoint_fp8_serialized", False): + reasons.append("shared experts are not serialized as FP8") + if not getattr(quant_config, "is_scale_e8m0", False): + reasons.append("shared-expert scales are not E8M0") + if getattr(quant_config, "ignored_layers", None): + reasons.append("the quantization config has ignored layers") + + offload_config = getattr(vllm_config, "offload_config", None) + if offload_config is not None and ( + getattr(getattr(offload_config, "uva", None), "cpu_offload_gb", 0) > 0 + or getattr( + getattr(offload_config, "prefetch", None), + "offload_group_size", + 0, + ) + > 0 + ): + reasons.append("weight offloading is enabled") + + if reasons: + logger.debug_once( + "DeepSeek V4 heterogeneous shared-expert fusion is unavailable: %s. " + "Continuing with the existing shared-expert path.", + "; ".join(reasons), + ) + return False + + logger.info_once( + "Using AITER DeepSeek V4 heterogeneous MoE for CSV-covered MoE input " + "rows (M) per invocation (native intermediate width 384)." + ) + return True + + +@torch.no_grad() +def _prepare_native_fp8_shared_expert( + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, + intermediate_size: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Expand native block-128 E8M0 scales to FHMoE's 1x32 layout.""" + if w13.dtype != torch.float8_e4m3fn or w2.dtype != torch.float8_e4m3fn: + raise ValueError("Heterogeneous shared-expert weights must be FP8 E4M3.") + hidden_size = w13.shape[1] + if w13.shape != (2 * intermediate_size, hidden_size): + raise ValueError(f"Unexpected shared W13 shape {tuple(w13.shape)}.") + if w2.shape != (hidden_size, intermediate_size): + raise ValueError(f"Unexpected shared W2 shape {tuple(w2.shape)}.") + + def scale_bytes(scale: torch.Tensor) -> torch.Tensor: + if scale.dtype == torch.float8_e8m0fnu: + return scale.contiguous().view(torch.uint8) + if scale.dtype == torch.uint8: + return scale.contiguous() + if scale.dtype == torch.float32: + bits = scale.contiguous().view(torch.int32) + exponent = (bits >> 23).to(torch.uint8) + if not torch.equal(bits, exponent.to(torch.int32) << 23): + raise ValueError("FP32 shared-expert scales are not exact E8M0.") + return exponent + raise ValueError(f"Unsupported shared-expert scale dtype {scale.dtype}.") + + w13_scale = scale_bytes(w13_scale) + w2_scale = scale_bytes(w2_scale) + expected_w13_scale = (2 * intermediate_size // 128, hidden_size // 128) + expected_w2_scale = (hidden_size // 128, intermediate_size // 128) + if tuple(w13_scale.shape) != expected_w13_scale: + raise ValueError( + f"Expected shared W13 scale shape {expected_w13_scale}, got " + f"{tuple(w13_scale.shape)}." + ) + if tuple(w2_scale.shape) != expected_w2_scale: + raise ValueError( + f"Expected shared W2 scale shape {expected_w2_scale}, got " + f"{tuple(w2_scale.shape)}." + ) + + w13_scale = w13_scale.repeat_interleave(128, dim=0).repeat_interleave(4, dim=1) + w2_scale = w2_scale.repeat_interleave(128, dim=0).repeat_interleave(4, dim=1) + + scale_cols = ((w2_scale.shape[1] + 7) // 8) * 8 + if scale_cols != w2_scale.shape[1]: + padded_w2_scale = torch.full( + (w2_scale.shape[0], scale_cols), + 0x7F, + dtype=torch.uint8, + device=w2_scale.device, + ) + padded_w2_scale[:, : w2_scale.shape[1]].copy_(w2_scale) + w2_scale = padded_w2_scale + + return ( + w13.unsqueeze(0), + w2.unsqueeze(0), + w13_scale.view(torch.float8_e8m0fnu), + w2_scale.view(torch.float8_e8m0fnu), + ) + + +class DeepseekV4HeterogeneousMxfp4MoEMethod(Mxfp4MoEMethod): + def process_weights_after_loading(self, layer: RoutedExperts) -> None: + super().process_weights_after_loading(layer) + assert isinstance(layer, DeepseekV4HeterogeneousSharedRoutedExperts) + layer.prepare_heterogeneous_shared_expert() + + +class DeepseekV4HeterogeneousSharedRoutedExperts(RoutedExperts): + def __init__( + self, + *args, + shared_expert: DeepseekV4MLP, + shared_expert_id: int, + **kwargs, + ): + self._shared_expert_ref = weakref.ref(shared_expert) + self.shared_expert_id = shared_expert_id + self._routed_quant_config: FusedMoEQuantConfig | None = None + super().__init__(*args, **kwargs) + if self.expert_map_manager.num_fused_shared_experts != 1: + raise ValueError("Heterogeneous fusion requires one appended expert.") + if self.shared_expert_id != self.global_num_experts: + raise ValueError("The shared expert ID must name the appended slot.") + self.register_buffer("shared_w1", None, persistent=False) + self.register_buffer("shared_w2", None, persistent=False) + self.register_buffer("shared_w1_scale", None, persistent=False) + self.register_buffer("shared_w2_scale", None, persistent=False) + + def _get_quant_method(self, prefix, quant_config, moe_config): + quant_method = super()._get_quant_method(prefix, quant_config, moe_config) + if not isinstance(quant_method, Mxfp4MoEMethod): + raise ValueError("Heterogeneous fusion requires MXFP4 routed experts.") + return DeepseekV4HeterogeneousMxfp4MoEMethod(moe_config) + + @torch.no_grad() + def prepare_heterogeneous_shared_expert(self) -> None: + shared_expert = self._shared_expert_ref() + if shared_expert is None: + raise RuntimeError("The native shared-expert module was released.") + + shared = _prepare_native_fp8_shared_expert( + shared_expert.gate_up_proj.weight, + shared_expert.down_proj.weight, + shared_expert.gate_up_proj.weight_scale_inv, + shared_expert.down_proj.weight_scale_inv, + self.moe_config.intermediate_size_per_partition, + ) + self.shared_w1 = rocm_aiter_ops.shuffle_weight_a16w4(shared[0], 16, True) + self.shared_w2 = rocm_aiter_ops.shuffle_weight_a16w4(shared[1], 16, False) + self.shared_w1_scale = rocm_aiter_ops.shuffle_scale_a16w4(shared[2], 1, True) + self.shared_w2_scale = rocm_aiter_ops.shuffle_scale(shared[3]) + + self._ensure_moe_quant_config_init() + quant_config = self.quant_method.moe_quant_config + if quant_config is None: + raise RuntimeError("The routed MXFP4 quantization config is unavailable.") + if quant_config.w1_scale is None or quant_config.w2_scale is None: + raise RuntimeError("The routed MXFP4 scales are unavailable.") + + def routed_scales(scale: torch.Tensor) -> torch.Tensor: + scale = scale.reshape(self.local_num_experts, -1, scale.shape[-1])[ + : self.shared_expert_id + ] + return scale.reshape(-1, scale.shape[-1]).contiguous() + + self._routed_quant_config = replace( + quant_config, + _w1=replace( + quant_config._w1, + scale=routed_scales(quant_config.w1_scale), + ), + _w2=replace( + quant_config._w2, + scale=routed_scales(quant_config.w2_scale), + ), + ) + + def forward_modular( + self, + x: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + shared_experts=None, + shared_experts_input: torch.Tensor | None = None, + ) -> torch.Tensor: + if shared_experts is not None or shared_experts_input is not None: + raise ValueError("The heterogeneous expert owns the shared path.") + _validate_heterogeneous_routes( + topk_weights, + topk_ids, + self.moe_config.experts_per_token, + ) + if any( + tensor is None + for tensor in ( + self.shared_w1, + self.shared_w2, + self.shared_w1_scale, + self.shared_w2_scale, + ) + ): + raise RuntimeError("Heterogeneous shared weights were not prepared.") + + if _use_heterogeneous_fhmoe(x.shape[0]): + self._ensure_moe_quant_config_init() + quant_config = self.quant_method.moe_quant_config + if quant_config is None: + raise RuntimeError("The routed MXFP4 config is unavailable.") + return rocm_aiter_fused_experts( + hidden_states=x, + w1=self.w13_weight, + w2=self.w2_weight, + topk_weights=topk_weights, + topk_ids=topk_ids, + moe_config=self.moe_config, + activation=self.activation, + expert_mask=self.expert_map, + quant_config=quant_config, + output_dtype=x.dtype, + moe_sorting_dispatch_policy=(rocm_aiter_ops.get_moe_dispatch_policy()), + shared_w1=self.shared_w1, + shared_w2=self.shared_w2, + shared_w1_scale=self.shared_w1_scale, + shared_w2_scale=self.shared_w2_scale, + shared_expert_id=self.shared_expert_id, + ) + + shared_expert = self._shared_expert_ref() + if shared_expert is None or self._routed_quant_config is None: + raise RuntimeError("The separate shared-expert fallback is unavailable.") + shared_out = shared_expert(x) + routed_w1 = self.w13_weight[: self.shared_expert_id] + routed_w2 = self.w2_weight[: self.shared_expert_id] + routed_w1.is_shuffled = True + routed_w2.is_shuffled = True + routed = rocm_aiter_fused_experts( + hidden_states=x, + w1=routed_w1, + w2=routed_w2, + topk_weights=topk_weights[:, :-1].contiguous(), + topk_ids=topk_ids[:, :-1].contiguous(), + moe_config=self.moe_config, + activation=self.activation, + expert_mask=self.expert_map, + quant_config=self._routed_quant_config, + output_dtype=x.dtype, + moe_sorting_dispatch_policy=rocm_aiter_ops.get_moe_dispatch_policy(), + ) + return routed + shared_out + + def _fuse_shared_experts_enabled(config) -> bool: return bool( getattr(config, "n_shared_experts", None) @@ -176,6 +515,7 @@ class DeepseekV4MoE(nn.Module): self, vllm_config: VllmConfig, prefix: str = "", + fuse_heterogeneous_shared_expert: bool = False, ): super().__init__() @@ -230,7 +570,12 @@ class DeepseekV4MoE(nn.Module): # TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1` # is checked to enable FSE for DeepSeek-v4, despite AITER not being used. # This should be cleaned up and use `resolve_layer_fused_shared_expert`. - fse_requested = _fuse_shared_experts_enabled(config) + self.fuse_heterogeneous_shared_expert = fuse_heterogeneous_shared_expert + fse_requested = ( + _fuse_shared_experts_enabled(config) + and not self.fuse_heterogeneous_shared_expert + ) + fse_compatible = False if fse_requested: fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible( quant_config, @@ -267,12 +612,15 @@ class DeepseekV4MoE(nn.Module): self.experts_start_idx = self.tp_rank * self.n_local_experts self.experts_end_idx = self.experts_start_idx + self.n_local_experts + fuse_shared_into_routed = ( + self.is_fused_shared_expert_enabled or self.fuse_heterogeneous_shared_expert + ) self.experts = FusedMoEFactory( - shared_experts=self.shared_experts, + shared_experts=(None if fuse_shared_into_routed else self.shared_experts), n_shared_experts=( - config.n_shared_experts if self.is_fused_shared_expert_enabled else None + config.n_shared_experts if fuse_shared_into_routed else None ), - fuse_shared_experts=self.is_fused_shared_expert_enabled, + fuse_shared_experts=fuse_shared_into_routed, gate=self.gate, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, @@ -287,6 +635,19 @@ class DeepseekV4MoE(nn.Module): hash_indices_table=self.gate.tid2eid, swiglu_limit=self.swiglu_limit, router_logits_dtype=torch.float32, + routed_experts_cls=( + DeepseekV4HeterogeneousSharedRoutedExperts + if self.fuse_heterogeneous_shared_expert + else None + ), + routed_experts_args=( + { + "shared_expert": self.shared_experts, + "shared_expert_id": config.n_routed_experts, + } + if self.fuse_heterogeneous_shared_expert + else None + ), ) def forward( @@ -316,6 +677,7 @@ class DeepseekV4DecoderLayer(nn.Module): prefix, topk_indices_buffer: torch.Tensor | None = None, aux_stream_list: list[torch.cuda.Stream] | None = None, + fuse_heterogeneous_shared_expert: bool = False, ): super().__init__() @@ -333,7 +695,11 @@ class DeepseekV4DecoderLayer(nn.Module): topk_indices_buffer=topk_indices_buffer, aux_stream_list=aux_stream_list, ) - self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn") + self.ffn = DeepseekV4MoE( + vllm_config, + prefix=f"{prefix}.ffn", + fuse_heterogeneous_shared_expert=fuse_heterogeneous_shared_expert, + ) self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) @@ -598,6 +964,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): else: self.embed_tokens = PPMissingLayer() + self.fuse_heterogeneous_shared_expert = _heterogeneous_shared_expert_enabled( + vllm_config + ) self.start_layer, self.end_layer, self.layers = make_layers( config.num_hidden_layers, lambda prefix: DeepseekV4DecoderLayer( @@ -605,6 +974,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): prefix=prefix, topk_indices_buffer=self.topk_indices_buffer, aux_stream_list=aux_stream_list, + fuse_heterogeneous_shared_expert=( + self.fuse_heterogeneous_shared_expert + ), ), prefix=f"{prefix}.layers", )