[ROCm][Perf][DeepSeek V4] Fuse native FP8 shared expert with MXFP4 routed experts (#53161)
Signed-off-by: fai <fangzhouai@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Cursor Agent <agent@cursor.com>
This commit is contained in:
co-authored by
OpenAI Codex
Cursor Agent
parent
294fbb4f59
commit
de69e821b7
@@ -2,8 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib
|
||||
import inspect
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -414,6 +416,343 @@ def test_deepseek_v4_shared_expert_fse_uses_mtp_quantization_config_prefix(
|
||||
assert reason is None
|
||||
|
||||
|
||||
def test_deepseek_v4_heterogeneous_fhmoe_keeps_native_intermediate_width() -> None:
|
||||
from vllm.models.deepseek_v4.amd.model import _prepare_native_fp8_shared_expert
|
||||
|
||||
hidden_size = 7168
|
||||
intermediate_size = 384
|
||||
w13 = torch.empty((2 * intermediate_size, hidden_size), dtype=torch.float8_e4m3fn)
|
||||
w2 = torch.empty((hidden_size, intermediate_size), dtype=torch.float8_e4m3fn)
|
||||
w13_scale_bytes = (torch.arange(6 * 56).reshape(6, 56).remainder(20) + 0x60).to(
|
||||
torch.uint8
|
||||
)
|
||||
w2_scale_bytes = (torch.arange(56 * 3).reshape(56, 3).remainder(30) + 0x50).to(
|
||||
torch.uint8
|
||||
)
|
||||
w13_scale = w13_scale_bytes.view(torch.float8_e8m0fnu)
|
||||
w2_scale = w2_scale_bytes.view(torch.float8_e8m0fnu)
|
||||
|
||||
prepared = _prepare_native_fp8_shared_expert(
|
||||
w13, w2, w13_scale, w2_scale, intermediate_size
|
||||
)
|
||||
|
||||
assert prepared[0].shape == (1, 768, 7168)
|
||||
assert prepared[1].shape == (1, 7168, 384)
|
||||
assert prepared[2].shape == (768, 224)
|
||||
assert prepared[3].shape == (7168, 16)
|
||||
expected_w13_scale = w13_scale_bytes.repeat_interleave(
|
||||
128, dim=0
|
||||
).repeat_interleave(4, dim=1)
|
||||
expected_w2_scale = w2_scale_bytes.repeat_interleave(128, dim=0).repeat_interleave(
|
||||
4, dim=1
|
||||
)
|
||||
assert torch.equal(prepared[2].view(torch.uint8), expected_w13_scale)
|
||||
assert torch.equal(prepared[3].view(torch.uint8)[:, :12], expected_w2_scale)
|
||||
assert torch.all(prepared[3].view(torch.uint8)[:, 12:] == 0x7F)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "supported_through", "expected"),
|
||||
[
|
||||
(0, 4096, False),
|
||||
(1, 4096, True),
|
||||
(1536, 4096, True),
|
||||
(2048, 4096, True),
|
||||
(2049, 4096, True),
|
||||
(4096, 4096, True),
|
||||
(4097, 4096, False),
|
||||
(1536, 0, False),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_heterogeneous_fhmoe_token_policy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
num_tokens: int,
|
||||
supported_through: int,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
from vllm.models.deepseek_v4.amd import model as deepseek_v4_model
|
||||
|
||||
checked_tokens: list[int] = []
|
||||
|
||||
def supports(num_tokens: int) -> bool:
|
||||
checked_tokens.append(num_tokens)
|
||||
return num_tokens <= supported_through
|
||||
|
||||
monkeypatch.setattr(
|
||||
deepseek_v4_model.rocm_aiter_ops,
|
||||
"fused_moe_supports_heterogeneous_shared_expert",
|
||||
supports,
|
||||
)
|
||||
|
||||
assert deepseek_v4_model._use_heterogeneous_fhmoe(num_tokens) is expected
|
||||
assert checked_tokens == ([] if num_tokens == 0 else [num_tokens])
|
||||
|
||||
|
||||
def _supported_fhmoe_signature(
|
||||
shared_w1=None,
|
||||
shared_w2=None,
|
||||
shared_w1_scale=None,
|
||||
shared_w2_scale=None,
|
||||
shared_expert_id=-1,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _incomplete_fhmoe_signature(
|
||||
shared_w1=None,
|
||||
shared_w2=None,
|
||||
shared_w1_scale=None,
|
||||
shared_w2_scale=None,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _install_fake_aiter_fhmoe(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
supports_dsv4_i384_fhmoe: object,
|
||||
fused_moe: object,
|
||||
) -> None:
|
||||
fake_aiter = ModuleType("aiter")
|
||||
fake_aiter.__path__ = []
|
||||
fake_fhmoe = ModuleType("aiter.fhmoe")
|
||||
if supports_dsv4_i384_fhmoe is not None:
|
||||
fake_fhmoe.__dict__["supports_dsv4_i384_fhmoe"] = supports_dsv4_i384_fhmoe
|
||||
fake_fused_moe = ModuleType("aiter.fused_moe")
|
||||
fake_fused_moe.__dict__["fused_moe"] = fused_moe
|
||||
fake_aiter.__dict__["fhmoe"] = fake_fhmoe
|
||||
fake_aiter.__dict__["fused_moe"] = fake_fused_moe
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setitem(sys.modules, "aiter.fhmoe", fake_fhmoe)
|
||||
monkeypatch.setitem(sys.modules, "aiter.fused_moe", fake_fused_moe)
|
||||
|
||||
|
||||
def _supports_fhmoe_through_2047(max_tokens: int) -> bool:
|
||||
return max_tokens <= 2047
|
||||
|
||||
|
||||
def _supports_fhmoe_through_2048(max_tokens: int) -> bool:
|
||||
return max_tokens <= 2048
|
||||
|
||||
|
||||
def _supports_fhmoe_through_4096(max_tokens: int) -> bool:
|
||||
return max_tokens <= 4096
|
||||
|
||||
|
||||
def _raises_fhmoe_config_error(max_tokens: int) -> bool:
|
||||
raise OSError
|
||||
|
||||
|
||||
def _returns_truthy_non_bool(max_tokens: int) -> int:
|
||||
return 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "capability", "fused_moe", "expected"),
|
||||
[
|
||||
(0, _supports_fhmoe_through_2048, _supported_fhmoe_signature, False),
|
||||
(True, _supports_fhmoe_through_2048, _supported_fhmoe_signature, False),
|
||||
(2048, None, _supported_fhmoe_signature, False),
|
||||
(2048, True, _supported_fhmoe_signature, False),
|
||||
(2048, _raises_fhmoe_config_error, _supported_fhmoe_signature, False),
|
||||
(2048, _returns_truthy_non_bool, _supported_fhmoe_signature, False),
|
||||
(2048, _supports_fhmoe_through_2047, _supported_fhmoe_signature, False),
|
||||
(2048, _supports_fhmoe_through_2048, _incomplete_fhmoe_signature, False),
|
||||
(2048, _supports_fhmoe_through_2048, _supported_fhmoe_signature, True),
|
||||
(2049, _supports_fhmoe_through_2048, _supported_fhmoe_signature, False),
|
||||
(4096, _supports_fhmoe_through_4096, _supported_fhmoe_signature, True),
|
||||
(4097, _supports_fhmoe_through_4096, _supported_fhmoe_signature, False),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_heterogeneous_fhmoe_aiter_capability(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
num_tokens: object,
|
||||
capability: object,
|
||||
fused_moe: object,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_install_fake_aiter_fhmoe(monkeypatch, capability, fused_moe)
|
||||
|
||||
assert (
|
||||
rocm_aiter_ops._probe_dsv4_i384_fhmoe_capability(cast(int, num_tokens))
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_v4_heterogeneous_fhmoe_aiter_capability_catches_import_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
fake_aiter = ModuleType("aiter")
|
||||
fake_aiter.__path__ = []
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setitem(sys.modules, "aiter.fhmoe", None)
|
||||
|
||||
assert not rocm_aiter_ops._probe_dsv4_i384_fhmoe_capability(2048)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error", [TypeError, ValueError])
|
||||
def test_deepseek_v4_heterogeneous_fhmoe_aiter_capability_catches_signature_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
error: type[Exception],
|
||||
) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_install_fake_aiter_fhmoe(
|
||||
monkeypatch, _supports_fhmoe_through_2048, _supported_fhmoe_signature
|
||||
)
|
||||
|
||||
def raise_signature_error(_: object) -> inspect.Signature:
|
||||
raise error
|
||||
|
||||
monkeypatch.setattr(inspect, "signature", raise_signature_error)
|
||||
|
||||
assert not rocm_aiter_ops._probe_dsv4_i384_fhmoe_capability(2048)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("setting", "value", "expected"),
|
||||
[
|
||||
("data_parallel_size", 1, True),
|
||||
("data_parallel_size", 2, False),
|
||||
("prefill_context_parallel_size", 2, False),
|
||||
("topk_method", "greedy", False),
|
||||
("fhmoe_supported", False, False),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_heterogeneous_fhmoe_compatibility_gates(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
setting: str,
|
||||
value: object,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
from vllm.models.deepseek_v4.amd import model as deepseek_v4_model
|
||||
|
||||
hf_config = SimpleNamespace(
|
||||
n_routed_experts=384,
|
||||
num_experts_per_tok=6,
|
||||
n_shared_experts=1,
|
||||
hidden_size=7168,
|
||||
moe_intermediate_size=3072,
|
||||
hidden_act="silu",
|
||||
expert_dtype="fp4",
|
||||
topk_method="noaux_tc",
|
||||
)
|
||||
parallel_config = SimpleNamespace(
|
||||
enable_expert_parallel=False,
|
||||
enable_eplb=False,
|
||||
tensor_parallel_size=8,
|
||||
data_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
)
|
||||
if setting == "topk_method":
|
||||
hf_config.topk_method = value
|
||||
elif setting != "fhmoe_supported":
|
||||
setattr(parallel_config, setting, value)
|
||||
|
||||
quant_config = SimpleNamespace(
|
||||
get_name=lambda: "deepseek_v4_fp8",
|
||||
moe_quant_algo="",
|
||||
weight_block_size=[128, 128],
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
is_scale_e8m0=True,
|
||||
ignored_layers=None,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=hf_config, dtype=torch.bfloat16),
|
||||
quant_config=quant_config,
|
||||
parallel_config=parallel_config,
|
||||
kernel_config=SimpleNamespace(moe_backend="aiter"),
|
||||
offload_config=None,
|
||||
)
|
||||
monkeypatch.setattr(deepseek_v4_model.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
deepseek_v4_model.envs,
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS",
|
||||
True,
|
||||
)
|
||||
monkeypatch.setattr(deepseek_v4_model, "on_gfx950", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
deepseek_v4_model.rocm_aiter_ops,
|
||||
"is_fusion_moe_shared_experts_enabled",
|
||||
lambda: True,
|
||||
)
|
||||
checked_tokens: list[int] = []
|
||||
fhmoe_supported = setting != "fhmoe_supported" or value is True
|
||||
|
||||
def supports(num_tokens: int) -> bool:
|
||||
checked_tokens.append(num_tokens)
|
||||
return fhmoe_supported
|
||||
|
||||
monkeypatch.setattr(
|
||||
deepseek_v4_model.rocm_aiter_ops,
|
||||
"fused_moe_supports_heterogeneous_shared_expert",
|
||||
supports,
|
||||
)
|
||||
|
||||
assert (
|
||||
deepseek_v4_model._heterogeneous_shared_expert_enabled(
|
||||
cast(VllmConfig, vllm_config)
|
||||
)
|
||||
is expected
|
||||
)
|
||||
assert checked_tokens == [1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("weights_shape", "ids_shape", "message"),
|
||||
[
|
||||
((2, 7, 1), (2, 7, 1), "equal two-dimensional shapes"),
|
||||
((2, 7), (3, 7), "equal two-dimensional shapes"),
|
||||
((2, 6), (2, 6), "exactly 7 columns"),
|
||||
((2, 8), (2, 8), "exactly 7 columns"),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_heterogeneous_fhmoe_rejects_invalid_routes(
|
||||
weights_shape: tuple[int, ...],
|
||||
ids_shape: tuple[int, ...],
|
||||
message: str,
|
||||
) -> None:
|
||||
from vllm.models.deepseek_v4.amd.model import _validate_heterogeneous_routes
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
_validate_heterogeneous_routes(
|
||||
torch.empty(weights_shape),
|
||||
torch.empty(ids_shape, dtype=torch.int64),
|
||||
experts_per_token=6,
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_v4_heterogeneous_fhmoe_accepts_appended_shared_route() -> None:
|
||||
from vllm.models.deepseek_v4.amd.model import _validate_heterogeneous_routes
|
||||
|
||||
_validate_heterogeneous_routes(
|
||||
torch.empty((2, 7)),
|
||||
torch.empty((2, 7), dtype=torch.int64),
|
||||
experts_per_token=6,
|
||||
)
|
||||
|
||||
|
||||
def test_aiter_fused_moe_validates_shared_expert_arguments() -> None:
|
||||
from vllm._aiter_ops import (
|
||||
_validate_rocm_aiter_fused_moe_shared_expert_args,
|
||||
)
|
||||
|
||||
validate = _validate_rocm_aiter_fused_moe_shared_expert_args
|
||||
shared_tensor = torch.empty(0)
|
||||
|
||||
assert not validate(None, None, None, None, -1)
|
||||
with pytest.raises(ValueError, match="requires shared weights and scales"):
|
||||
validate(None, None, None, None, 0)
|
||||
with pytest.raises(ValueError, match="both shared weights and scales"):
|
||||
validate(shared_tensor, None, None, None, 0)
|
||||
with pytest.raises(ValueError, match="non-negative shared expert ID"):
|
||||
validate(shared_tensor, shared_tensor, shared_tensor, shared_tensor, -1)
|
||||
assert validate(shared_tensor, shared_tensor, shared_tensor, shared_tensor, 0)
|
||||
|
||||
|
||||
def test_is_model_fused_shared_expert_compatible() -> None:
|
||||
class MoE(nn.Module):
|
||||
def __init__(self, enabled: bool) -> None:
|
||||
|
||||
@@ -211,6 +211,31 @@ def if_aiter_supported(func: Callable) -> Callable:
|
||||
return wrapper
|
||||
|
||||
|
||||
def _validate_rocm_aiter_fused_moe_shared_expert_args(
|
||||
shared_w1: torch.Tensor | None,
|
||||
shared_w2: torch.Tensor | None,
|
||||
shared_w1_scale: torch.Tensor | None,
|
||||
shared_w2_scale: torch.Tensor | None,
|
||||
shared_expert_id: int,
|
||||
) -> bool:
|
||||
shared_tensors = (shared_w1, shared_w2, shared_w1_scale, shared_w2_scale)
|
||||
has_shared_expert = any(tensor is not None for tensor in shared_tensors)
|
||||
if has_shared_expert:
|
||||
if not all(tensor is not None for tensor in shared_tensors):
|
||||
raise ValueError(
|
||||
"Heterogeneous fused MoE requires both shared weights and scales."
|
||||
)
|
||||
if shared_expert_id < 0:
|
||||
raise ValueError(
|
||||
"Heterogeneous fused MoE requires a non-negative shared expert ID."
|
||||
)
|
||||
elif shared_expert_id >= 0:
|
||||
raise ValueError(
|
||||
"A non-negative shared expert ID requires shared weights and scales."
|
||||
)
|
||||
return has_shared_expert
|
||||
|
||||
|
||||
def _rocm_aiter_fused_moe_impl(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
@@ -236,7 +261,20 @@ def _rocm_aiter_fused_moe_impl(
|
||||
swiglu_limit: float = 0.0,
|
||||
beta: float | None = None,
|
||||
linear_beta: float | None = None,
|
||||
shared_w1: torch.Tensor | None = None,
|
||||
shared_w2: torch.Tensor | None = None,
|
||||
shared_w1_scale: torch.Tensor | None = None,
|
||||
shared_w2_scale: torch.Tensor | None = None,
|
||||
shared_expert_id: int = -1,
|
||||
) -> torch.Tensor:
|
||||
has_shared_expert = _validate_rocm_aiter_fused_moe_shared_expert_args(
|
||||
shared_w1,
|
||||
shared_w2,
|
||||
shared_w1_scale,
|
||||
shared_w2_scale,
|
||||
shared_expert_id,
|
||||
)
|
||||
|
||||
from aiter import ActivationType, QuantType
|
||||
from aiter.fused_moe import fused_moe
|
||||
|
||||
@@ -253,6 +291,15 @@ def _rocm_aiter_fused_moe_impl(
|
||||
extra_kwargs["beta"] = beta
|
||||
extra_kwargs["linear_beta"] = linear_beta
|
||||
|
||||
if has_shared_expert:
|
||||
extra_kwargs.update(
|
||||
shared_w1=shared_w1,
|
||||
shared_w2=shared_w2,
|
||||
shared_w1_scale=shared_w1_scale,
|
||||
shared_w2_scale=shared_w2_scale,
|
||||
shared_expert_id=shared_expert_id,
|
||||
)
|
||||
|
||||
return fused_moe(
|
||||
hidden_states,
|
||||
w1,
|
||||
@@ -304,6 +351,11 @@ def _rocm_aiter_fused_moe_fake(
|
||||
swiglu_limit: float = 0.0,
|
||||
beta: float | None = None,
|
||||
linear_beta: float | None = None,
|
||||
shared_w1: torch.Tensor | None = None,
|
||||
shared_w2: torch.Tensor | None = None,
|
||||
shared_w1_scale: torch.Tensor | None = None,
|
||||
shared_w2_scale: torch.Tensor | None = None,
|
||||
shared_expert_id: int = -1,
|
||||
) -> torch.Tensor:
|
||||
if output_dtype is not None:
|
||||
return torch.empty_like(hidden_states, dtype=output_dtype)
|
||||
@@ -1997,6 +2049,44 @@ class rocm_aiter_ops:
|
||||
|
||||
return "gate_mode" in inspect.signature(fused_moe).parameters
|
||||
|
||||
@staticmethod
|
||||
def _probe_dsv4_i384_fhmoe_capability(num_tokens: int) -> bool:
|
||||
"""Probe AITER's CSV-backed DSV4 native-I384 FHMoE contract."""
|
||||
if type(num_tokens) is not int or num_tokens <= 0:
|
||||
return False
|
||||
|
||||
try:
|
||||
import inspect
|
||||
|
||||
import aiter.fhmoe as fhmoe
|
||||
from aiter.fused_moe import fused_moe
|
||||
|
||||
params = inspect.signature(fused_moe).parameters
|
||||
supports_dsv4_i384_fhmoe = getattr(fhmoe, "supports_dsv4_i384_fhmoe", None)
|
||||
if not callable(supports_dsv4_i384_fhmoe):
|
||||
return False
|
||||
supports_num_tokens = supports_dsv4_i384_fhmoe(num_tokens)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
return supports_num_tokens is True and all(
|
||||
name in params
|
||||
for name in (
|
||||
"shared_w1",
|
||||
"shared_w2",
|
||||
"shared_w1_scale",
|
||||
"shared_w2_scale",
|
||||
"shared_expert_id",
|
||||
)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
@functools.cache
|
||||
def fused_moe_supports_heterogeneous_shared_expert(cls, num_tokens: int) -> bool:
|
||||
"""Whether AITER has DSV4 native-I384 configs through the given M."""
|
||||
return cls._probe_dsv4_i384_fhmoe_capability(num_tokens)
|
||||
|
||||
@staticmethod
|
||||
def register_ops_once() -> None:
|
||||
global _OPS_REGISTERED
|
||||
@@ -2465,6 +2555,11 @@ class rocm_aiter_ops:
|
||||
swiglu_limit: float = 0.0,
|
||||
beta: float | None = None,
|
||||
linear_beta: float | None = None,
|
||||
shared_w1: torch.Tensor | None = None,
|
||||
shared_w2: torch.Tensor | None = None,
|
||||
shared_w1_scale: torch.Tensor | None = None,
|
||||
shared_w2_scale: torch.Tensor | None = None,
|
||||
shared_expert_id: int = -1,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.rocm_aiter_fused_moe(
|
||||
hidden_states,
|
||||
@@ -2491,6 +2586,11 @@ class rocm_aiter_ops:
|
||||
swiglu_limit,
|
||||
beta,
|
||||
linear_beta,
|
||||
shared_w1,
|
||||
shared_w2,
|
||||
shared_w1_scale,
|
||||
shared_w2_scale,
|
||||
shared_expert_id,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -3074,6 +3174,13 @@ class rocm_aiter_ops:
|
||||
|
||||
return shuffle_scale_a16w4(tensor, num_experts, gate_up)
|
||||
|
||||
@staticmethod
|
||||
def shuffle_scale(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Shuffle a non-gate/up E8M0 scale tensor for AITER."""
|
||||
from aiter.ops.shuffle import shuffle_scale
|
||||
|
||||
return shuffle_scale(tensor)
|
||||
|
||||
@staticmethod
|
||||
def shuffle_weights(
|
||||
*tensors: torch.Tensor, layout: tuple[int, int] = (16, 16)
|
||||
|
||||
@@ -250,6 +250,11 @@ def rocm_aiter_fused_experts(
|
||||
num_local_tokens: torch.Tensor | None = None,
|
||||
output_dtype: torch.dtype | None = None,
|
||||
moe_sorting_dispatch_policy: int = 0,
|
||||
shared_w1: torch.Tensor | None = None,
|
||||
shared_w2: torch.Tensor | None = None,
|
||||
shared_w1_scale: torch.Tensor | None = None,
|
||||
shared_w2_scale: torch.Tensor | None = None,
|
||||
shared_expert_id: int = -1,
|
||||
) -> torch.Tensor:
|
||||
"""ROCm AITER fused MoE expert computation."""
|
||||
if quant_config is None:
|
||||
@@ -419,8 +424,18 @@ def rocm_aiter_fused_experts(
|
||||
bias1=quant_config.w1_bias if quant_config.use_mxfp4_w4a16 else None,
|
||||
bias2=quant_config.w2_bias if quant_config.use_mxfp4_w4a16 else None,
|
||||
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
|
||||
swiglu_limit=(
|
||||
0.0
|
||||
if moe_config.swiglu_limit is None
|
||||
else float(moe_config.swiglu_limit)
|
||||
),
|
||||
beta=moe_config.activation_situ_beta,
|
||||
linear_beta=moe_config.activation_situ_linear_beta,
|
||||
shared_w1=shared_w1,
|
||||
shared_w2=shared_w2,
|
||||
shared_w1_scale=shared_w1_scale,
|
||||
shared_w2_scale=shared_w2_scale,
|
||||
shared_expert_id=shared_expert_id,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import typing
|
||||
import weakref
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import replace
|
||||
from itertools import islice
|
||||
|
||||
import regex as re
|
||||
@@ -20,9 +22,14 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
FusedMoEQuantConfig,
|
||||
GateLinear,
|
||||
RoutedExperts,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
rocm_aiter_fused_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
@@ -41,6 +48,7 @@ from vllm.model_executor.layers.mhc import (
|
||||
MHCPreOp,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
@@ -65,6 +73,7 @@ from vllm.model_executor.models.utils import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -163,6 +172,336 @@ class DeepseekV4MLP(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
def _use_heterogeneous_fhmoe(num_tokens: int) -> bool:
|
||||
return num_tokens > 0 and (
|
||||
rocm_aiter_ops.fused_moe_supports_heterogeneous_shared_expert(num_tokens)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def _validate_heterogeneous_routes(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
experts_per_token: int,
|
||||
) -> None:
|
||||
if (
|
||||
topk_weights.ndim != 2
|
||||
or topk_ids.ndim != 2
|
||||
or topk_weights.shape != topk_ids.shape
|
||||
):
|
||||
raise ValueError(
|
||||
"Heterogeneous fused MoE routes must have equal two-dimensional shapes."
|
||||
)
|
||||
expected_columns = experts_per_token + 1
|
||||
if topk_weights.shape[1] != expected_columns:
|
||||
raise ValueError(
|
||||
"Heterogeneous fused MoE routes must have exactly "
|
||||
f"{expected_columns} columns, got {topk_weights.shape[1]}."
|
||||
)
|
||||
|
||||
|
||||
def _heterogeneous_shared_expert_enabled(vllm_config: VllmConfig) -> bool:
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
reasons: list[str] = []
|
||||
|
||||
if not rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
|
||||
return False
|
||||
if not on_gfx950():
|
||||
reasons.append("the device is not gfx950")
|
||||
if parallel_config.enable_expert_parallel:
|
||||
reasons.append("expert parallelism is enabled")
|
||||
if parallel_config.enable_eplb:
|
||||
reasons.append("EPLB is enabled")
|
||||
if parallel_config.tensor_parallel_size != 8:
|
||||
reasons.append("tensor parallelism is not 8")
|
||||
if parallel_config.data_parallel_size != 1:
|
||||
reasons.append("data parallelism is enabled")
|
||||
if parallel_config.prefill_context_parallel_size != 1:
|
||||
reasons.append("prefill context parallelism is enabled")
|
||||
if vllm_config.kernel_config.moe_backend != "aiter":
|
||||
reasons.append("the MoE backend is not AITER")
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
reasons.append("the model dtype is not BF16")
|
||||
if not rocm_aiter_ops.fused_moe_supports_heterogeneous_shared_expert(1):
|
||||
reasons.append(
|
||||
"AITER lacks CSV-backed DSV4 I384 heterogeneous shared-expert support"
|
||||
)
|
||||
|
||||
expected_model = {
|
||||
"n_routed_experts": 384,
|
||||
"num_experts_per_tok": 6,
|
||||
"n_shared_experts": 1,
|
||||
"hidden_size": 7168,
|
||||
"moe_intermediate_size": 3072,
|
||||
"hidden_act": "silu",
|
||||
"expert_dtype": "fp4",
|
||||
"topk_method": "noaux_tc",
|
||||
}
|
||||
for name, expected in expected_model.items():
|
||||
if getattr(config, name, None) != expected:
|
||||
reasons.append(f"{name} is not {expected!r}")
|
||||
|
||||
if quant_config is None or quant_config.get_name() != "deepseek_v4_fp8":
|
||||
reasons.append("the DeepSeek V4 FP8 quantization config is not active")
|
||||
else:
|
||||
if getattr(quant_config, "moe_quant_algo", "").upper() == "NVFP4":
|
||||
reasons.append("routed experts use NVFP4")
|
||||
if getattr(quant_config, "weight_block_size", None) != [128, 128]:
|
||||
reasons.append("shared experts are not block-128 FP8")
|
||||
if not getattr(quant_config, "is_checkpoint_fp8_serialized", False):
|
||||
reasons.append("shared experts are not serialized as FP8")
|
||||
if not getattr(quant_config, "is_scale_e8m0", False):
|
||||
reasons.append("shared-expert scales are not E8M0")
|
||||
if getattr(quant_config, "ignored_layers", None):
|
||||
reasons.append("the quantization config has ignored layers")
|
||||
|
||||
offload_config = getattr(vllm_config, "offload_config", None)
|
||||
if offload_config is not None and (
|
||||
getattr(getattr(offload_config, "uva", None), "cpu_offload_gb", 0) > 0
|
||||
or getattr(
|
||||
getattr(offload_config, "prefetch", None),
|
||||
"offload_group_size",
|
||||
0,
|
||||
)
|
||||
> 0
|
||||
):
|
||||
reasons.append("weight offloading is enabled")
|
||||
|
||||
if reasons:
|
||||
logger.debug_once(
|
||||
"DeepSeek V4 heterogeneous shared-expert fusion is unavailable: %s. "
|
||||
"Continuing with the existing shared-expert path.",
|
||||
"; ".join(reasons),
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info_once(
|
||||
"Using AITER DeepSeek V4 heterogeneous MoE for CSV-covered MoE input "
|
||||
"rows (M) per invocation (native intermediate width 384)."
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def _prepare_native_fp8_shared_expert(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
intermediate_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Expand native block-128 E8M0 scales to FHMoE's 1x32 layout."""
|
||||
if w13.dtype != torch.float8_e4m3fn or w2.dtype != torch.float8_e4m3fn:
|
||||
raise ValueError("Heterogeneous shared-expert weights must be FP8 E4M3.")
|
||||
hidden_size = w13.shape[1]
|
||||
if w13.shape != (2 * intermediate_size, hidden_size):
|
||||
raise ValueError(f"Unexpected shared W13 shape {tuple(w13.shape)}.")
|
||||
if w2.shape != (hidden_size, intermediate_size):
|
||||
raise ValueError(f"Unexpected shared W2 shape {tuple(w2.shape)}.")
|
||||
|
||||
def scale_bytes(scale: torch.Tensor) -> torch.Tensor:
|
||||
if scale.dtype == torch.float8_e8m0fnu:
|
||||
return scale.contiguous().view(torch.uint8)
|
||||
if scale.dtype == torch.uint8:
|
||||
return scale.contiguous()
|
||||
if scale.dtype == torch.float32:
|
||||
bits = scale.contiguous().view(torch.int32)
|
||||
exponent = (bits >> 23).to(torch.uint8)
|
||||
if not torch.equal(bits, exponent.to(torch.int32) << 23):
|
||||
raise ValueError("FP32 shared-expert scales are not exact E8M0.")
|
||||
return exponent
|
||||
raise ValueError(f"Unsupported shared-expert scale dtype {scale.dtype}.")
|
||||
|
||||
w13_scale = scale_bytes(w13_scale)
|
||||
w2_scale = scale_bytes(w2_scale)
|
||||
expected_w13_scale = (2 * intermediate_size // 128, hidden_size // 128)
|
||||
expected_w2_scale = (hidden_size // 128, intermediate_size // 128)
|
||||
if tuple(w13_scale.shape) != expected_w13_scale:
|
||||
raise ValueError(
|
||||
f"Expected shared W13 scale shape {expected_w13_scale}, got "
|
||||
f"{tuple(w13_scale.shape)}."
|
||||
)
|
||||
if tuple(w2_scale.shape) != expected_w2_scale:
|
||||
raise ValueError(
|
||||
f"Expected shared W2 scale shape {expected_w2_scale}, got "
|
||||
f"{tuple(w2_scale.shape)}."
|
||||
)
|
||||
|
||||
w13_scale = w13_scale.repeat_interleave(128, dim=0).repeat_interleave(4, dim=1)
|
||||
w2_scale = w2_scale.repeat_interleave(128, dim=0).repeat_interleave(4, dim=1)
|
||||
|
||||
scale_cols = ((w2_scale.shape[1] + 7) // 8) * 8
|
||||
if scale_cols != w2_scale.shape[1]:
|
||||
padded_w2_scale = torch.full(
|
||||
(w2_scale.shape[0], scale_cols),
|
||||
0x7F,
|
||||
dtype=torch.uint8,
|
||||
device=w2_scale.device,
|
||||
)
|
||||
padded_w2_scale[:, : w2_scale.shape[1]].copy_(w2_scale)
|
||||
w2_scale = padded_w2_scale
|
||||
|
||||
return (
|
||||
w13.unsqueeze(0),
|
||||
w2.unsqueeze(0),
|
||||
w13_scale.view(torch.float8_e8m0fnu),
|
||||
w2_scale.view(torch.float8_e8m0fnu),
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4HeterogeneousMxfp4MoEMethod(Mxfp4MoEMethod):
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
assert isinstance(layer, DeepseekV4HeterogeneousSharedRoutedExperts)
|
||||
layer.prepare_heterogeneous_shared_expert()
|
||||
|
||||
|
||||
class DeepseekV4HeterogeneousSharedRoutedExperts(RoutedExperts):
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
shared_expert: DeepseekV4MLP,
|
||||
shared_expert_id: int,
|
||||
**kwargs,
|
||||
):
|
||||
self._shared_expert_ref = weakref.ref(shared_expert)
|
||||
self.shared_expert_id = shared_expert_id
|
||||
self._routed_quant_config: FusedMoEQuantConfig | None = None
|
||||
super().__init__(*args, **kwargs)
|
||||
if self.expert_map_manager.num_fused_shared_experts != 1:
|
||||
raise ValueError("Heterogeneous fusion requires one appended expert.")
|
||||
if self.shared_expert_id != self.global_num_experts:
|
||||
raise ValueError("The shared expert ID must name the appended slot.")
|
||||
self.register_buffer("shared_w1", None, persistent=False)
|
||||
self.register_buffer("shared_w2", None, persistent=False)
|
||||
self.register_buffer("shared_w1_scale", None, persistent=False)
|
||||
self.register_buffer("shared_w2_scale", None, persistent=False)
|
||||
|
||||
def _get_quant_method(self, prefix, quant_config, moe_config):
|
||||
quant_method = super()._get_quant_method(prefix, quant_config, moe_config)
|
||||
if not isinstance(quant_method, Mxfp4MoEMethod):
|
||||
raise ValueError("Heterogeneous fusion requires MXFP4 routed experts.")
|
||||
return DeepseekV4HeterogeneousMxfp4MoEMethod(moe_config)
|
||||
|
||||
@torch.no_grad()
|
||||
def prepare_heterogeneous_shared_expert(self) -> None:
|
||||
shared_expert = self._shared_expert_ref()
|
||||
if shared_expert is None:
|
||||
raise RuntimeError("The native shared-expert module was released.")
|
||||
|
||||
shared = _prepare_native_fp8_shared_expert(
|
||||
shared_expert.gate_up_proj.weight,
|
||||
shared_expert.down_proj.weight,
|
||||
shared_expert.gate_up_proj.weight_scale_inv,
|
||||
shared_expert.down_proj.weight_scale_inv,
|
||||
self.moe_config.intermediate_size_per_partition,
|
||||
)
|
||||
self.shared_w1 = rocm_aiter_ops.shuffle_weight_a16w4(shared[0], 16, True)
|
||||
self.shared_w2 = rocm_aiter_ops.shuffle_weight_a16w4(shared[1], 16, False)
|
||||
self.shared_w1_scale = rocm_aiter_ops.shuffle_scale_a16w4(shared[2], 1, True)
|
||||
self.shared_w2_scale = rocm_aiter_ops.shuffle_scale(shared[3])
|
||||
|
||||
self._ensure_moe_quant_config_init()
|
||||
quant_config = self.quant_method.moe_quant_config
|
||||
if quant_config is None:
|
||||
raise RuntimeError("The routed MXFP4 quantization config is unavailable.")
|
||||
if quant_config.w1_scale is None or quant_config.w2_scale is None:
|
||||
raise RuntimeError("The routed MXFP4 scales are unavailable.")
|
||||
|
||||
def routed_scales(scale: torch.Tensor) -> torch.Tensor:
|
||||
scale = scale.reshape(self.local_num_experts, -1, scale.shape[-1])[
|
||||
: self.shared_expert_id
|
||||
]
|
||||
return scale.reshape(-1, scale.shape[-1]).contiguous()
|
||||
|
||||
self._routed_quant_config = replace(
|
||||
quant_config,
|
||||
_w1=replace(
|
||||
quant_config._w1,
|
||||
scale=routed_scales(quant_config.w1_scale),
|
||||
),
|
||||
_w2=replace(
|
||||
quant_config._w2,
|
||||
scale=routed_scales(quant_config.w2_scale),
|
||||
),
|
||||
)
|
||||
|
||||
def forward_modular(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts=None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if shared_experts is not None or shared_experts_input is not None:
|
||||
raise ValueError("The heterogeneous expert owns the shared path.")
|
||||
_validate_heterogeneous_routes(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
self.moe_config.experts_per_token,
|
||||
)
|
||||
if any(
|
||||
tensor is None
|
||||
for tensor in (
|
||||
self.shared_w1,
|
||||
self.shared_w2,
|
||||
self.shared_w1_scale,
|
||||
self.shared_w2_scale,
|
||||
)
|
||||
):
|
||||
raise RuntimeError("Heterogeneous shared weights were not prepared.")
|
||||
|
||||
if _use_heterogeneous_fhmoe(x.shape[0]):
|
||||
self._ensure_moe_quant_config_init()
|
||||
quant_config = self.quant_method.moe_quant_config
|
||||
if quant_config is None:
|
||||
raise RuntimeError("The routed MXFP4 config is unavailable.")
|
||||
return rocm_aiter_fused_experts(
|
||||
hidden_states=x,
|
||||
w1=self.w13_weight,
|
||||
w2=self.w2_weight,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
moe_config=self.moe_config,
|
||||
activation=self.activation,
|
||||
expert_mask=self.expert_map,
|
||||
quant_config=quant_config,
|
||||
output_dtype=x.dtype,
|
||||
moe_sorting_dispatch_policy=(rocm_aiter_ops.get_moe_dispatch_policy()),
|
||||
shared_w1=self.shared_w1,
|
||||
shared_w2=self.shared_w2,
|
||||
shared_w1_scale=self.shared_w1_scale,
|
||||
shared_w2_scale=self.shared_w2_scale,
|
||||
shared_expert_id=self.shared_expert_id,
|
||||
)
|
||||
|
||||
shared_expert = self._shared_expert_ref()
|
||||
if shared_expert is None or self._routed_quant_config is None:
|
||||
raise RuntimeError("The separate shared-expert fallback is unavailable.")
|
||||
shared_out = shared_expert(x)
|
||||
routed_w1 = self.w13_weight[: self.shared_expert_id]
|
||||
routed_w2 = self.w2_weight[: self.shared_expert_id]
|
||||
routed_w1.is_shuffled = True
|
||||
routed_w2.is_shuffled = True
|
||||
routed = rocm_aiter_fused_experts(
|
||||
hidden_states=x,
|
||||
w1=routed_w1,
|
||||
w2=routed_w2,
|
||||
topk_weights=topk_weights[:, :-1].contiguous(),
|
||||
topk_ids=topk_ids[:, :-1].contiguous(),
|
||||
moe_config=self.moe_config,
|
||||
activation=self.activation,
|
||||
expert_mask=self.expert_map,
|
||||
quant_config=self._routed_quant_config,
|
||||
output_dtype=x.dtype,
|
||||
moe_sorting_dispatch_policy=rocm_aiter_ops.get_moe_dispatch_policy(),
|
||||
)
|
||||
return routed + shared_out
|
||||
|
||||
|
||||
def _fuse_shared_experts_enabled(config) -> bool:
|
||||
return bool(
|
||||
getattr(config, "n_shared_experts", None)
|
||||
@@ -176,6 +515,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
fuse_heterogeneous_shared_expert: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -230,7 +570,12 @@ class DeepseekV4MoE(nn.Module):
|
||||
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
|
||||
# is checked to enable FSE for DeepSeek-v4, despite AITER not being used.
|
||||
# This should be cleaned up and use `resolve_layer_fused_shared_expert`.
|
||||
fse_requested = _fuse_shared_experts_enabled(config)
|
||||
self.fuse_heterogeneous_shared_expert = fuse_heterogeneous_shared_expert
|
||||
fse_requested = (
|
||||
_fuse_shared_experts_enabled(config)
|
||||
and not self.fuse_heterogeneous_shared_expert
|
||||
)
|
||||
fse_compatible = False
|
||||
if fse_requested:
|
||||
fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
@@ -267,12 +612,15 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.experts_start_idx = self.tp_rank * self.n_local_experts
|
||||
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
|
||||
|
||||
fuse_shared_into_routed = (
|
||||
self.is_fused_shared_expert_enabled or self.fuse_heterogeneous_shared_expert
|
||||
)
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
shared_experts=(None if fuse_shared_into_routed else self.shared_experts),
|
||||
n_shared_experts=(
|
||||
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
|
||||
config.n_shared_experts if fuse_shared_into_routed else None
|
||||
),
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
fuse_shared_experts=fuse_shared_into_routed,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
@@ -287,6 +635,19 @@ class DeepseekV4MoE(nn.Module):
|
||||
hash_indices_table=self.gate.tid2eid,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
router_logits_dtype=torch.float32,
|
||||
routed_experts_cls=(
|
||||
DeepseekV4HeterogeneousSharedRoutedExperts
|
||||
if self.fuse_heterogeneous_shared_expert
|
||||
else None
|
||||
),
|
||||
routed_experts_args=(
|
||||
{
|
||||
"shared_expert": self.shared_experts,
|
||||
"shared_expert_id": config.n_routed_experts,
|
||||
}
|
||||
if self.fuse_heterogeneous_shared_expert
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -316,6 +677,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
prefix,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
aux_stream_list: list[torch.cuda.Stream] | None = None,
|
||||
fuse_heterogeneous_shared_expert: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -333,7 +695,11 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
aux_stream_list=aux_stream_list,
|
||||
)
|
||||
self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn")
|
||||
self.ffn = DeepseekV4MoE(
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.ffn",
|
||||
fuse_heterogeneous_shared_expert=fuse_heterogeneous_shared_expert,
|
||||
)
|
||||
|
||||
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
@@ -598,6 +964,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.fuse_heterogeneous_shared_expert = _heterogeneous_shared_expert_enabled(
|
||||
vllm_config
|
||||
)
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: DeepseekV4DecoderLayer(
|
||||
@@ -605,6 +974,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
aux_stream_list=aux_stream_list,
|
||||
fuse_heterogeneous_shared_expert=(
|
||||
self.fuse_heterogeneous_shared_expert
|
||||
),
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user