[MOE] Standardize and abstract fused shared expert optimization selection (#51695)
Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
@@ -0,0 +1,700 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import vllm.config as vllm_config_module
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import utils as fused_moe_utils
|
||||
from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config
|
||||
from vllm.models.minimax_m3.amd import model as minimax_m3_model
|
||||
from vllm.transformers_utils.configs.minimax_m3 import MiniMaxM3TextConfig
|
||||
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
|
||||
|
||||
_QUARK_FSE_CONFIG: dict[str, Any] = {
|
||||
"global_quant_config": {
|
||||
"input_tensors": {
|
||||
"dtype": "fp4",
|
||||
"is_dynamic": True,
|
||||
"qscheme": "per_group",
|
||||
"ch_axis": -1,
|
||||
"group_size": 32,
|
||||
"block_size": None,
|
||||
"symmetric": None,
|
||||
"round_method": "half_even",
|
||||
"scale_type": "float",
|
||||
"scale_format": "e8m0",
|
||||
"scale_calculation_mode": "even",
|
||||
"mx_element_dtype": None,
|
||||
"observer_cls": "PerBlockMXObserver",
|
||||
"is_scale_quant": False,
|
||||
"enable_buffer_reuse": False,
|
||||
"max_input_numel": 4194304,
|
||||
},
|
||||
"output_tensors": None,
|
||||
"weight": {
|
||||
"dtype": "fp4",
|
||||
"is_dynamic": False,
|
||||
"qscheme": "per_group",
|
||||
"ch_axis": -1,
|
||||
"group_size": 32,
|
||||
"block_size": None,
|
||||
"symmetric": None,
|
||||
"round_method": "half_even",
|
||||
"scale_type": "float",
|
||||
"scale_format": "e8m0",
|
||||
"scale_calculation_mode": "even",
|
||||
"mx_element_dtype": None,
|
||||
"observer_cls": "PerBlockMXObserver",
|
||||
"is_scale_quant": False,
|
||||
"enable_buffer_reuse": False,
|
||||
"max_input_numel": 4194304,
|
||||
},
|
||||
"bias": None,
|
||||
"target_device": None,
|
||||
},
|
||||
"algo_config": None,
|
||||
"softmax_quant_spec": None,
|
||||
"quant_method": "quark",
|
||||
"layer_type_quant_config": {},
|
||||
"layer_quant_config": {},
|
||||
"kv_cache_quant_config": {},
|
||||
"kv_cache_post_rope": False,
|
||||
"quant_mode": "eager_mode",
|
||||
"version": "0.12+9d3d471cdf1",
|
||||
"export": {
|
||||
"kv_cache_group": [],
|
||||
"min_kv_scale": 0.0,
|
||||
"pack_method": "reorder",
|
||||
"weight_format": "real_quantized",
|
||||
"weight_merge_groups": None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_deepseek_v4_quark_config(exclude: list[str]) -> dict[str, Any]:
|
||||
"""Return the DeepSeek-V4-Pro-MXFP4 FSE quantization layout."""
|
||||
# Mimics https://huggingface.co/amd/DeepSeek-V4-Pro-MXFP4.
|
||||
quantization_config: dict[str, Any] = deepcopy(_QUARK_FSE_CONFIG)
|
||||
quantization_config["exclude"] = exclude
|
||||
mxfp4_config = cast(
|
||||
dict[str, Any], deepcopy(quantization_config["global_quant_config"])
|
||||
)
|
||||
fp8_config = deepcopy(mxfp4_config)
|
||||
fp8_config["input_tensors"].update(
|
||||
{
|
||||
"dtype": "fp8_e4m3",
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
"scale_type": None,
|
||||
"scale_format": None,
|
||||
"scale_calculation_mode": None,
|
||||
"observer_cls": "PerGroupMinMaxObserver",
|
||||
}
|
||||
)
|
||||
fp8_config["weight"].update(
|
||||
{
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"ch_axis": None,
|
||||
"group_size": None,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
"scale_type": "float8_e8m0fnu",
|
||||
"scale_format": None,
|
||||
"scale_calculation_mode": None,
|
||||
"observer_cls": "PerBlock2DMinMaxObserver",
|
||||
}
|
||||
)
|
||||
quantization_config["layer_quant_config"] = {
|
||||
r"re:mtp\.0\.ffn\.experts\.\d+\.w1": mxfp4_config,
|
||||
r"re:mtp\.0\.ffn\.shared_experts\.w1": fp8_config,
|
||||
}
|
||||
return quantization_config
|
||||
|
||||
|
||||
def get_fse_test_model_config(
|
||||
model_type: str,
|
||||
quantization_config: dict[str, Any],
|
||||
) -> tuple[object, type[nn.Module]]:
|
||||
if model_type == "minimax_m3":
|
||||
config = MiniMaxM3TextConfig(
|
||||
hidden_size=128,
|
||||
intermediate_size=32,
|
||||
dense_intermediate_size=32,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=1,
|
||||
num_key_value_heads=1,
|
||||
head_dim=128,
|
||||
num_local_experts=2,
|
||||
num_experts_per_tok=1,
|
||||
moe_layer_freq=[0, 1],
|
||||
sparse_attention_config=None,
|
||||
rotary_dim=64,
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
return config, minimax_m3_model.MiniMaxM3Model
|
||||
if model_type == "deepseek_v4":
|
||||
from vllm.models.deepseek_v4.amd.model import DeepseekV4Model
|
||||
|
||||
config = SimpleNamespace(
|
||||
vocab_size=256,
|
||||
hidden_size=128,
|
||||
num_hidden_layers=1,
|
||||
num_attention_heads=1,
|
||||
head_dim=128,
|
||||
max_position_embeddings=128,
|
||||
sliding_window=None,
|
||||
compress_ratios=[1],
|
||||
rope_theta=10000.0,
|
||||
compress_rope_theta=10000.0,
|
||||
rope_parameters={"rope_type": "default"},
|
||||
rms_norm_eps=1e-6,
|
||||
hidden_act="silu",
|
||||
q_lora_rank=0,
|
||||
o_lora_rank=0,
|
||||
o_groups=1,
|
||||
qk_rope_head_dim=64,
|
||||
index_head_dim=64,
|
||||
index_n_heads=1,
|
||||
index_topk=1,
|
||||
n_routed_experts=2,
|
||||
n_shared_experts=1,
|
||||
num_experts_per_tok=1,
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
moe_intermediate_size=32,
|
||||
norm_topk_prob=False,
|
||||
num_hash_layers=0,
|
||||
swiglu_limit=0.0,
|
||||
hc_eps=1e-6,
|
||||
hc_mult=1,
|
||||
hc_sinkhorn_iters=1,
|
||||
expert_dtype="fp4",
|
||||
num_nextn_predict_layers=1,
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
return config, DeepseekV4Model
|
||||
if model_type == "qwen3_5":
|
||||
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
|
||||
|
||||
config = Qwen3_5MoeTextConfig(
|
||||
vocab_size=256,
|
||||
hidden_size=128,
|
||||
num_hidden_layers=1,
|
||||
num_attention_heads=1,
|
||||
num_key_value_heads=1,
|
||||
head_dim=128,
|
||||
linear_key_head_dim=64,
|
||||
linear_value_head_dim=64,
|
||||
linear_num_key_heads=1,
|
||||
linear_num_value_heads=1,
|
||||
moe_intermediate_size=32,
|
||||
shared_expert_intermediate_size=32,
|
||||
num_experts_per_tok=1,
|
||||
num_experts=2,
|
||||
layer_types=["full_attention"],
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
return config, Qwen3_5Model
|
||||
if model_type == "deepseek_v2":
|
||||
from transformers import DeepseekV2Config
|
||||
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV2Model
|
||||
|
||||
config = DeepseekV2Config(
|
||||
vocab_size=256,
|
||||
hidden_size=128,
|
||||
intermediate_size=32,
|
||||
num_hidden_layers=1,
|
||||
num_attention_heads=1,
|
||||
num_key_value_heads=1,
|
||||
head_dim=128,
|
||||
first_k_dense_replace=0,
|
||||
n_routed_experts=2,
|
||||
n_shared_experts=1,
|
||||
num_experts_per_tok=1,
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
moe_intermediate_size=32,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=0,
|
||||
qk_nope_head_dim=0,
|
||||
qk_rope_head_dim=0,
|
||||
v_head_dim=0,
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
return config, DeepseekV2Model
|
||||
if model_type == "glm4_moe":
|
||||
from transformers.models.glm4_moe import Glm4MoeConfig
|
||||
|
||||
from vllm.model_executor.models.glm4_moe import Glm4MoeModel
|
||||
|
||||
config = Glm4MoeConfig(
|
||||
vocab_size=256,
|
||||
hidden_size=128,
|
||||
intermediate_size=32,
|
||||
num_hidden_layers=1,
|
||||
num_attention_heads=1,
|
||||
num_key_value_heads=1,
|
||||
head_dim=128,
|
||||
moe_intermediate_size=32,
|
||||
n_routed_experts=2,
|
||||
n_shared_experts=1,
|
||||
num_experts_per_tok=1,
|
||||
first_k_dense_replace=0,
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
return config, Glm4MoeModel
|
||||
raise ValueError(f"Unsupported FSE test model: {model_type}")
|
||||
|
||||
|
||||
def test_determine_expert_counts_fuse_shared_experts_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
common_args = (8, 0, 2)
|
||||
assert determine_expert_counts(*common_args, True)[2] == 2
|
||||
assert determine_expert_counts(*common_args, False)[2] == 0
|
||||
|
||||
|
||||
def test_resolve_layer_fused_shared_expert_skips_compatibility_when_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
fused_moe_utils.rocm_aiter_ops,
|
||||
"is_fusion_moe_shared_experts_enabled",
|
||||
lambda: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
fused_moe_utils,
|
||||
"is_shared_expert_quant_fse_compatible",
|
||||
lambda *_: pytest.fail(
|
||||
"compatibility must not be checked when FSE is disabled"
|
||||
),
|
||||
)
|
||||
|
||||
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
|
||||
object(), "model.layers.0.mlp"
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
fused_moe_utils.rocm_aiter_ops,
|
||||
"is_fusion_moe_shared_experts_enabled",
|
||||
lambda: None,
|
||||
)
|
||||
|
||||
assert (
|
||||
fused_moe_utils.resolve_layer_fused_shared_expert(
|
||||
object(), "model.layers.0.mlp"
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_layer_fused_shared_expert_passes_module_prefixes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
quant_config = object()
|
||||
monkeypatch.setattr(
|
||||
fused_moe_utils.rocm_aiter_ops,
|
||||
"is_fusion_moe_shared_experts_enabled",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
calls: list[tuple[object, str, str]] = []
|
||||
|
||||
def check_compatibility(
|
||||
config: object, expert_prefix: str, shared_expert_prefix: str
|
||||
) -> tuple[bool, str | None]:
|
||||
calls.append((config, expert_prefix, shared_expert_prefix))
|
||||
return True, None
|
||||
|
||||
monkeypatch.setattr(
|
||||
fused_moe_utils, "is_shared_expert_quant_fse_compatible", check_compatibility
|
||||
)
|
||||
|
||||
assert fused_moe_utils.resolve_layer_fused_shared_expert(
|
||||
quant_config,
|
||||
"model.layers.0.mlp",
|
||||
shared_expert_name="shared_expert",
|
||||
)
|
||||
assert calls == [
|
||||
(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.experts",
|
||||
"model.layers.0.mlp.shared_expert",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_resolve_layer_fused_shared_expert_rejects_incompatible_quantization(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
fused_moe_utils.rocm_aiter_ops,
|
||||
"is_fusion_moe_shared_experts_enabled",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
fused_moe_utils,
|
||||
"is_shared_expert_quant_fse_compatible",
|
||||
lambda *_: (False, "shared experts are excluded"),
|
||||
)
|
||||
|
||||
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
|
||||
object(), "model.layers.0.mlp"
|
||||
)
|
||||
assert "shared experts are excluded" in caplog.text
|
||||
|
||||
|
||||
def test_deepseek_v4_shared_expert_fse_uses_mtp_quantization_config_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class DeepseekV4Config:
|
||||
expert_dtype = "fp4"
|
||||
|
||||
def _is_quark_mxfp4_ocp(self, hf_config: object) -> bool:
|
||||
return True
|
||||
|
||||
hf_config = SimpleNamespace(
|
||||
num_hidden_layers=2,
|
||||
quantization_config={
|
||||
"layer_quant_config": {
|
||||
r"re:mtp\.0\.ffn\.shared_experts\.w1": {"weight": {"dtype": "fp4"}}
|
||||
},
|
||||
"global_quant_config": {"weight": {"dtype": "fp8"}},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deepseek_v4_quant_config, "DeepseekV4FP8Config", DeepseekV4Config
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
vllm_config_module,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=SimpleNamespace(hf_config=hf_config)),
|
||||
)
|
||||
|
||||
compatible, reason = is_shared_expert_quant_fse_compatible(
|
||||
DeepseekV4Config(),
|
||||
"model.layers.2.ffn.experts",
|
||||
"model.layers.2.ffn.shared_experts",
|
||||
)
|
||||
|
||||
assert compatible
|
||||
assert reason is None
|
||||
|
||||
|
||||
def test_is_model_fused_shared_expert_compatible() -> None:
|
||||
class MoE(nn.Module):
|
||||
def __init__(self, enabled: bool) -> None:
|
||||
super().__init__()
|
||||
self.is_fused_shared_expert_enabled = enabled
|
||||
|
||||
class Layer(nn.Module):
|
||||
def __init__(self, enabled: bool) -> None:
|
||||
super().__init__()
|
||||
self.mlp = MoE(enabled)
|
||||
|
||||
enabled_layers = nn.ModuleList([Layer(True)])
|
||||
disabled_layers = nn.ModuleList([Layer(False)])
|
||||
mixed_layers = nn.ModuleList([Layer(True), Layer(False)])
|
||||
empty_layers = nn.ModuleList()
|
||||
pipeline_layers = nn.ModuleList([Layer(True), PPMissingLayer()])
|
||||
|
||||
assert fused_moe_utils.is_model_fused_shared_expert_compatible(
|
||||
enabled_layers, MoE, "mlp"
|
||||
)
|
||||
assert not fused_moe_utils.is_model_fused_shared_expert_compatible(
|
||||
disabled_layers, MoE, "mlp"
|
||||
)
|
||||
assert not fused_moe_utils.is_model_fused_shared_expert_compatible(
|
||||
empty_layers, MoE, "mlp"
|
||||
)
|
||||
assert fused_moe_utils.is_model_fused_shared_expert_compatible(
|
||||
pipeline_layers, MoE, "mlp"
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="1 enabled and 1 disabled layers"):
|
||||
fused_moe_utils.is_model_fused_shared_expert_compatible(
|
||||
mixed_layers, MoE, "mlp"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_type",
|
||||
["minimax_m3", "deepseek_v4", "qwen3_5", "glm4_moe", "deepseek_v2"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("use_fse", "exclude"),
|
||||
[
|
||||
(False, []),
|
||||
(True, []),
|
||||
(True, ["*.shared_experts.*"]),
|
||||
],
|
||||
)
|
||||
def test_models_fse_init(
|
||||
model_type: str,
|
||||
use_fse: bool,
|
||||
exclude: list[str],
|
||||
dist_init: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Model construction resolves FSE consistently with Quark quantization."""
|
||||
|
||||
quantization_config: dict[str, Any] = (
|
||||
get_deepseek_v4_quark_config(["layers.0.ffn.shared_experts"] if exclude else [])
|
||||
if model_type == "deepseek_v4"
|
||||
else {**_QUARK_FSE_CONFIG, "exclude": exclude}
|
||||
)
|
||||
|
||||
config, model_constructor = get_fse_test_model_config(
|
||||
model_type, quantization_config
|
||||
)
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
hf_config=config,
|
||||
hf_text_config=config,
|
||||
dtype=torch.bfloat16,
|
||||
max_model_len=128,
|
||||
is_diffusion=False,
|
||||
head_dtype=None,
|
||||
is_mm_prefix_lm=False,
|
||||
multimodal_config=None,
|
||||
quantization_config=None,
|
||||
)
|
||||
vllm_config.parallel_config.enable_expert_parallel = False
|
||||
if model_type == "deepseek_v4":
|
||||
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
|
||||
|
||||
vllm_config.cache_config.cache_dtype = "fp8_ds_mla"
|
||||
vllm_config.quant_config = DeepseekV4FP8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
weight_block_size=[128, 128],
|
||||
)
|
||||
else:
|
||||
vllm_config.quant_config = QuarkConfig(quantization_config)
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if model_type == "deepseek_v4":
|
||||
from vllm.models.deepseek_v4.amd import model as deepseek_v4_model
|
||||
|
||||
warning_logger = deepseek_v4_model.logger
|
||||
elif model_type == "minimax_m3":
|
||||
warning_logger = minimax_m3_model.logger
|
||||
else:
|
||||
warning_logger = fused_moe_utils.logger
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", str(use_fse))
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", str(use_fse))
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", str(use_fse))
|
||||
mp.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.experts."
|
||||
"ocp_mx_emulation_moe.has_quark",
|
||||
lambda: True,
|
||||
)
|
||||
importlib.reload(envs)
|
||||
mp.setattr(envs, "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", use_fse)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
aiter_fse_enabled = bool(rocm_aiter_ops.is_fusion_moe_shared_experts_enabled())
|
||||
# These AMD-specific models currently use the raw environment flag.
|
||||
# and do not rely on `rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()`.
|
||||
fse_enabled = use_fse and (
|
||||
aiter_fse_enabled or model_type in {"deepseek_v4", "minimax_m3"}
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(warning_logger, "warning") as warning,
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
model = model_constructor(vllm_config=vllm_config)
|
||||
mtp = None
|
||||
if model_type == "deepseek_v4" and use_fse and not exclude:
|
||||
from vllm.models.deepseek_v4.amd.mtp import (
|
||||
DeepSeekV4MTP,
|
||||
)
|
||||
|
||||
vllm_config.speculative_config = SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(hf_config=config)
|
||||
)
|
||||
mtp = DeepSeekV4MTP(vllm_config=vllm_config)
|
||||
assert model.is_fused_shared_expert_enabled is (fse_enabled and not exclude)
|
||||
|
||||
# The dummy quant config here uses mixed mxfp4/fp8 for experts/shared_expert
|
||||
# so should just raise a warning.
|
||||
if mtp is not None:
|
||||
assert not mtp.model.layers[
|
||||
"1"
|
||||
].mtp_block.ffn.is_fused_shared_expert_enabled
|
||||
warning.assert_called_once()
|
||||
assert (
|
||||
"DeepSeek-V4 shared experts at mtp.0.ffn.shared_experts"
|
||||
in (warning.call_args.args[1])
|
||||
)
|
||||
if aiter_fse_enabled and exclude:
|
||||
warning.assert_called_once()
|
||||
assert (
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled"
|
||||
in (warning.call_args.args[0])
|
||||
)
|
||||
assert "excludes shared experts" in warning.call_args.args[1]
|
||||
|
||||
importlib.reload(envs)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("exclude", "expected"),
|
||||
[([], True), (["*.shared_expert.*"], False)],
|
||||
)
|
||||
def test_quark_shared_expert_fse_compatibility(
|
||||
exclude: list[str], expected: bool
|
||||
) -> None:
|
||||
compatible, reason = is_shared_expert_quant_fse_compatible(
|
||||
QuarkConfig(
|
||||
{
|
||||
"exclude": exclude,
|
||||
"global_quant_config": {},
|
||||
"layer_quant_config": {},
|
||||
}
|
||||
),
|
||||
"model.layers.0.mlp.experts",
|
||||
"model.layers.0.mlp.shared_expert",
|
||||
)
|
||||
|
||||
assert compatible is expected
|
||||
if expected:
|
||||
assert reason is None
|
||||
else:
|
||||
assert (
|
||||
reason
|
||||
== "Quark excludes shared experts at model.layers.0.mlp.shared_expert"
|
||||
)
|
||||
|
||||
|
||||
def test_quark_shared_expert_fse_requires_matching_layer_quant_configs() -> None:
|
||||
global_quant_config = {"weight": {"dtype": "fp4"}}
|
||||
quant_config = QuarkConfig(
|
||||
{
|
||||
"exclude": [],
|
||||
"global_quant_config": global_quant_config,
|
||||
"layer_quant_config": {
|
||||
"model.layers.0.mlp.shared_expert.gate_up_proj": {
|
||||
"weight": {"dtype": "fp8"}
|
||||
},
|
||||
"model.layers.0.mlp.shared_expert.down_proj": {
|
||||
"weight": {"dtype": "fp8"}
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
compatible, reason = is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.experts",
|
||||
"model.layers.0.mlp.shared_expert",
|
||||
)
|
||||
|
||||
assert not compatible
|
||||
assert reason == (
|
||||
"Quark uses different quantization configurations for routed and "
|
||||
"shared experts at model.layers.0.mlp.shared_expert"
|
||||
)
|
||||
|
||||
|
||||
def test_quark_shared_expert_fse_rejects_partial_packed_projection_override() -> None:
|
||||
fp4_config = {"weight": {"dtype": "fp4"}}
|
||||
fp8_config = {"weight": {"dtype": "fp8"}}
|
||||
quant_config = QuarkConfig(
|
||||
{
|
||||
"exclude": [],
|
||||
"global_quant_config": fp4_config,
|
||||
"layer_quant_config": {
|
||||
"model.layers.0.mlp.experts": fp8_config,
|
||||
"model.layers.0.mlp.shared_expert.gate_proj": fp8_config,
|
||||
"model.layers.0.mlp.shared_expert.down_proj": fp8_config,
|
||||
},
|
||||
}
|
||||
)
|
||||
quant_config.packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]}
|
||||
|
||||
compatible, reason = is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.experts",
|
||||
"model.layers.0.mlp.shared_expert",
|
||||
)
|
||||
|
||||
assert not compatible
|
||||
assert reason == (
|
||||
"Quark uses different quantization configurations for routed and "
|
||||
"shared experts at model.layers.0.mlp.shared_expert"
|
||||
)
|
||||
|
||||
|
||||
def test_quark_layer_config_from_name_checks_packed_projections() -> None:
|
||||
quant_config = QuarkConfig(
|
||||
{
|
||||
"global_quant_config": {"weight": {"dtype": "fp4"}},
|
||||
"layer_quant_config": {
|
||||
"model.layers.0.mlp.shared_expert.w1": {"weight": {"dtype": "fp4"}},
|
||||
"model.layers.0.mlp.shared_expert.w3": {"weight": {"dtype": "fp8"}},
|
||||
},
|
||||
}
|
||||
)
|
||||
quant_config.packed_modules_mapping = {"gate_up_proj": ["w1", "w3"]}
|
||||
|
||||
with pytest.raises(ValueError, match="requires all to use the same scheme"):
|
||||
quant_config.get_layer_quant_config_from_name(
|
||||
"model.layers.0.mlp.shared_expert.gate_up_proj"
|
||||
)
|
||||
|
||||
|
||||
def test_quark_packed_layer_config_must_match_global_config() -> None:
|
||||
quant_config = QuarkConfig(
|
||||
{
|
||||
"global_quant_config": {"weight": {"dtype": "fp4"}},
|
||||
"layer_type_quant_config": {},
|
||||
"layer_quant_config": {
|
||||
"model.layers.0.mlp.shared_expert.w1": {"weight": {"dtype": "fp8"}},
|
||||
},
|
||||
}
|
||||
)
|
||||
quant_config.packed_modules_mapping = {"gate_up_proj": ["w1", "w3"]}
|
||||
|
||||
with pytest.raises(ValueError, match="requires all to use the same scheme"):
|
||||
quant_config._find_matched_config(
|
||||
"model.layers.0.mlp.shared_expert.gate_up_proj", nn.Module()
|
||||
)
|
||||
|
||||
|
||||
def test_non_quark_shared_expert_fse_is_incompatible() -> None:
|
||||
compatible, reason = is_shared_expert_quant_fse_compatible(
|
||||
object(),
|
||||
"model.layers.0.mlp.experts",
|
||||
"model.layers.0.mlp.shared_experts",
|
||||
)
|
||||
|
||||
assert not compatible
|
||||
assert reason == (
|
||||
"shared-expert FSE quantization compatibility is not implemented for object"
|
||||
)
|
||||
@@ -6,7 +6,6 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import ParallelConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
@@ -74,23 +73,13 @@ def determine_expert_counts(
|
||||
num_experts: int,
|
||||
num_redundant_experts: int,
|
||||
n_shared_experts: int | None,
|
||||
is_act_and_mul: bool,
|
||||
fuse_shared_experts: bool,
|
||||
) -> tuple[int, int, int]:
|
||||
global_num_experts = num_experts + num_redundant_experts
|
||||
logical_num_experts = num_experts
|
||||
# Shared-expert fusion: append the shared expert(s) as routed-expert slots
|
||||
# so they run in the same grouped GEMM. Gated by
|
||||
# VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: either the native aiter fused-MoE
|
||||
# path (env + master switch, via is_fusion_moe_shared_experts_enabled) or the
|
||||
# backend-neutral router-append path (env alone, independent of the master
|
||||
# switch; e.g. the MM3 triton/flydsl mxfp8 MoE). Gated activations only.
|
||||
fuse_shared_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
or envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
) and is_act_and_mul
|
||||
|
||||
num_fused_shared_experts = (
|
||||
n_shared_experts if n_shared_experts is not None and fuse_shared_enabled else 0
|
||||
n_shared_experts if n_shared_experts is not None and fuse_shared_experts else 0
|
||||
)
|
||||
|
||||
return global_num_experts, logical_num_experts, num_fused_shared_experts
|
||||
@@ -132,6 +121,7 @@ def FusedMoEFactory(
|
||||
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
|
||||
is_fused_checkpoint_transposed: bool = False,
|
||||
n_shared_experts: int | None = None,
|
||||
fuse_shared_experts: bool = False,
|
||||
router_logits_dtype: torch.dtype | None = None,
|
||||
gate: torch.nn.Module | None = None,
|
||||
shared_experts: torch.nn.Module | None = None,
|
||||
@@ -198,6 +188,7 @@ def FusedMoEFactory(
|
||||
block scales use transposed storage.
|
||||
n_shared_experts: Number of shared experts to fuse into the routed
|
||||
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
|
||||
fuse_shared_experts: Whether to enable shared-expert fusion.
|
||||
router_logits_dtype: Data type for router logits buffers
|
||||
gate: Pre-configured gate module
|
||||
shared_experts: Pre-configured shared experts module
|
||||
@@ -244,7 +235,7 @@ def FusedMoEFactory(
|
||||
num_experts,
|
||||
num_redundant_experts,
|
||||
n_shared_experts,
|
||||
is_act_and_mul,
|
||||
fuse_shared_experts,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -1,15 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
from collections.abc import Iterable
|
||||
from math import prod
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
@@ -34,16 +40,95 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
per_tensor_dequantize,
|
||||
)
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def resolve_layer_fused_shared_expert(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
prefix: str,
|
||||
shared_expert_name: str = "shared_experts",
|
||||
) -> bool:
|
||||
"""Resolve whether AITER fused shared-expert execution is enabled.
|
||||
|
||||
Args:
|
||||
quant_config: Model quantization configuration.
|
||||
prefix: MoE module prefix.
|
||||
shared_expert_name: Shared-expert module name under ``prefix``.
|
||||
|
||||
Returns:
|
||||
Whether AITER fused shared experts are enabled.
|
||||
|
||||
Raises:
|
||||
ValueError: If requested shared-expert fusion is quantization-incompatible.
|
||||
"""
|
||||
# NOTE: is_fusion_moe_shared_experts_enabled is decorated with @if_aiter_supported
|
||||
# that returns None if AITER is not available.
|
||||
fse_requested = bool(rocm_aiter_ops.is_fusion_moe_shared_experts_enabled())
|
||||
fse_compatible, fse_reason = (
|
||||
is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
f"{prefix}.experts",
|
||||
f"{prefix}.{shared_expert_name}",
|
||||
)
|
||||
if fse_requested
|
||||
else (True, None)
|
||||
)
|
||||
is_fused_shared_expert_enabled = fse_requested and fse_compatible
|
||||
if fse_requested and not is_fused_shared_expert_enabled:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
"cannot be enabled: %s.",
|
||||
fse_reason,
|
||||
)
|
||||
return is_fused_shared_expert_enabled
|
||||
|
||||
|
||||
def is_model_fused_shared_expert_compatible(
|
||||
layers: nn.ModuleList | Iterable[nn.Module],
|
||||
moe_cls: type[nn.Module],
|
||||
moe_name: str,
|
||||
) -> bool:
|
||||
"""Resolve one fused-shared-expert state for a model's MoE layers."""
|
||||
|
||||
def get_moe_layer(layer: nn.Module) -> nn.Module | None:
|
||||
for name in moe_name.split("."):
|
||||
layer = getattr(layer, name, None)
|
||||
if layer is None:
|
||||
return None
|
||||
return layer
|
||||
|
||||
moe_layers = (
|
||||
moe_layer
|
||||
for layer in layers
|
||||
if not isinstance(layer, PPMissingLayer)
|
||||
and (moe_layer := get_moe_layer(layer)) is not None
|
||||
and isinstance(moe_layer, moe_cls)
|
||||
)
|
||||
|
||||
enabled = [
|
||||
getattr(layer, "is_fused_shared_expert_enabled", False) for layer in moe_layers
|
||||
]
|
||||
enabled_count = sum(enabled)
|
||||
disabled_count = len(enabled) - enabled_count
|
||||
if enabled_count > 0 and disabled_count > 0:
|
||||
raise NotImplementedError(
|
||||
"Fused shared experts must be enabled for all MoE layers; found "
|
||||
f"{enabled_count} enabled and {disabled_count} disabled layers. "
|
||||
"Per-layer fused shared experts is not yet supported. Please open "
|
||||
"an issue."
|
||||
)
|
||||
return enabled_count > 0 and disabled_count == 0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _count_expert_num_tokens(
|
||||
topk_ids_ptr,
|
||||
|
||||
@@ -576,63 +576,91 @@ class QuarkConfig(QuantizationConfig):
|
||||
|
||||
return True
|
||||
|
||||
def _find_matched_config(
|
||||
self, layer_name: str, module: torch.nn.Module
|
||||
) -> dict[str, Any]:
|
||||
def get_layer_quant_config_from_name(
|
||||
self, layer_name: str
|
||||
) -> dict[str, Any] | None:
|
||||
proj_name = layer_name.split(".")[-1]
|
||||
if proj_name in self.packed_modules_mapping:
|
||||
shard_proj_names = self.packed_modules_mapping[proj_name]
|
||||
|
||||
# Convert fused_name --> [shard_names]
|
||||
shard_names = [
|
||||
layer_name.replace(proj_name, shard_proj_name)
|
||||
for shard_proj_name in shard_proj_names
|
||||
]
|
||||
|
||||
shard_configs = []
|
||||
for shard_name in shard_names:
|
||||
if shard_name == layer_name:
|
||||
config = cast(
|
||||
dict[str, Any], self.quant_config.get("global_quant_config")
|
||||
)
|
||||
for shard_proj_name in shard_proj_names:
|
||||
shard_name = layer_name.replace(proj_name, shard_proj_name)
|
||||
if shard_name != layer_name:
|
||||
config = self.get_layer_quant_config_from_name(shard_name)
|
||||
else:
|
||||
config = self._find_matched_config(shard_name, module)
|
||||
config = None
|
||||
shard_configs.append(config)
|
||||
|
||||
if not all(
|
||||
deep_compare(q_config, shard_configs[0]) for q_config in shard_configs
|
||||
matched_configs = [config for config in shard_configs if config is not None]
|
||||
if matched_configs and not all(
|
||||
deep_compare(config, matched_configs[0]) for config in matched_configs
|
||||
):
|
||||
raise ValueError(
|
||||
f"Found a different quantization configuration for "
|
||||
f"{shard_proj_names} in {layer_name}. vLLM "
|
||||
"requires all to use the same scheme."
|
||||
)
|
||||
return shard_configs[0]
|
||||
if matched_configs:
|
||||
return matched_configs[0]
|
||||
return None
|
||||
else:
|
||||
layer_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("layer_quant_config")
|
||||
dict[str, Any], self.quant_config.get("layer_quant_config") or {}
|
||||
)
|
||||
|
||||
def _matches_pattern(layer_name, pattern):
|
||||
if "*" not in pattern:
|
||||
return layer_name in pattern
|
||||
return fnmatch.fnmatch(layer_name, pattern)
|
||||
|
||||
for name_pattern, config in layer_quant_config.items():
|
||||
if _matches_pattern(layer_name, name_pattern):
|
||||
if "*" not in name_pattern:
|
||||
matches = layer_name in name_pattern
|
||||
else:
|
||||
matches = fnmatch.fnmatch(layer_name, name_pattern)
|
||||
if matches:
|
||||
return config
|
||||
return None
|
||||
|
||||
layer_type = cast(str, type(module))
|
||||
layer_type_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("layer_type_quant_config")
|
||||
)
|
||||
if layer_type in layer_type_quant_config:
|
||||
return layer_type_quant_config[layer_type]
|
||||
def _find_matched_config(
|
||||
self, layer_name: str, module: torch.nn.Module
|
||||
) -> dict[str, Any]:
|
||||
# Priority order:
|
||||
# 1. layer_quant_config,
|
||||
# 2. layer_type_quant_config,
|
||||
# 3. global_quant_config.
|
||||
|
||||
global_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("global_quant_config")
|
||||
)
|
||||
return global_quant_config
|
||||
layer_type = cast(str, type(module))
|
||||
layer_type_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("layer_type_quant_config")
|
||||
)
|
||||
global_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("global_quant_config")
|
||||
)
|
||||
fallback_config = layer_type_quant_config.get(layer_type, global_quant_config)
|
||||
|
||||
proj_name = layer_name.split(".")[-1]
|
||||
if proj_name in self.packed_modules_mapping:
|
||||
shard_proj_names = self.packed_modules_mapping[proj_name]
|
||||
shard_configs = []
|
||||
for shard_proj_name in shard_proj_names:
|
||||
shard_name = layer_name.replace(proj_name, shard_proj_name)
|
||||
if shard_name == layer_name:
|
||||
config = fallback_config
|
||||
else:
|
||||
config = self.get_layer_quant_config_from_name(shard_name)
|
||||
if config is None:
|
||||
config = fallback_config
|
||||
shard_configs.append(config)
|
||||
|
||||
if not all(
|
||||
deep_compare(config, shard_configs[0]) for config in shard_configs
|
||||
):
|
||||
raise ValueError(
|
||||
f"Found a different quantization configuration for "
|
||||
f"{shard_proj_names} in {layer_name}. vLLM requires all "
|
||||
"to use the same scheme."
|
||||
)
|
||||
return shard_configs[0]
|
||||
else:
|
||||
layer_quant_config = self.get_layer_quant_config_from_name(layer_name)
|
||||
if layer_quant_config is not None:
|
||||
return layer_quant_config
|
||||
return fallback_config
|
||||
|
||||
def _get_scheme_from_config(
|
||||
self, config: dict[str, Any], dynamic_mxfp4_quant: bool = False
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
def is_shared_expert_quant_fse_compatible(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
expert_prefix: str,
|
||||
shared_expert_prefix: str,
|
||||
projection_names: list[str] | None = None,
|
||||
) -> tuple[bool, str | None]:
|
||||
"""Check whether quantization permits fused shared-expert execution.
|
||||
|
||||
Args:
|
||||
quant_config: Model quantization configuration.
|
||||
expert_prefix: Routed-expert module prefix.
|
||||
shared_expert_prefix: Shared-expert module prefix.
|
||||
projection_names: Shared-expert projection names.
|
||||
|
||||
Returns:
|
||||
A compatibility flag and, when incompatible, the reason.
|
||||
"""
|
||||
if projection_names is None:
|
||||
projection_names = ["gate_up_proj", "down_proj"]
|
||||
|
||||
if quant_config is None:
|
||||
return True, None
|
||||
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
|
||||
|
||||
if isinstance(quant_config, DeepseekV4FP8Config):
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
if quant_config.expert_dtype != "fp4":
|
||||
return False, "DeepSeek-V4 routed experts are not MXFP4"
|
||||
|
||||
hf_config = get_current_vllm_config().model_config.hf_config
|
||||
|
||||
# TODO: This is adapted from former `_shared_experts_are_fp4`, and
|
||||
# needs to be cleaned up this . There should not be Quark-specific
|
||||
# logic in DeepseekV4FP8Config.
|
||||
quantization_config = getattr(hf_config, "quantization_config", None)
|
||||
if quantization_config is None:
|
||||
return False, "DeepSeek-V4 has no quantization configuration"
|
||||
|
||||
if not quant_config._is_quark_mxfp4_ocp(quantization_config):
|
||||
return False, "DeepSeek-v4 FSE is only implemented/tested with Quark MXFP4"
|
||||
|
||||
layer_idx = extract_layer_index(shared_expert_prefix)
|
||||
if layer_idx >= hf_config.num_hidden_layers:
|
||||
shared_expert_prefix = (
|
||||
f"mtp.{layer_idx - hf_config.num_hidden_layers}.ffn.shared_experts"
|
||||
)
|
||||
else:
|
||||
shared_expert_prefix = f"layers.{layer_idx}.ffn.shared_experts"
|
||||
|
||||
if any(
|
||||
entry.startswith(shared_expert_prefix)
|
||||
for entry in quantization_config.get("exclude") or []
|
||||
if isinstance(entry, str)
|
||||
):
|
||||
return (
|
||||
False,
|
||||
f"DeepSeek-V4 excludes shared experts at {shared_expert_prefix}",
|
||||
)
|
||||
|
||||
shared_expert_weight_name = f"{shared_expert_prefix}.w1"
|
||||
layer_quant_config = quantization_config.get("layer_quant_config") or {}
|
||||
layer_config = layer_quant_config.get(shared_expert_weight_name)
|
||||
if layer_config is None:
|
||||
layer_config = next(
|
||||
(
|
||||
config
|
||||
for pattern, config in layer_quant_config.items()
|
||||
if isinstance(pattern, str)
|
||||
and pattern.startswith("re:")
|
||||
and re.fullmatch(
|
||||
pattern.removeprefix("re:"), shared_expert_weight_name
|
||||
)
|
||||
),
|
||||
None,
|
||||
)
|
||||
shared_weight_config = (
|
||||
layer_config or quantization_config.get("global_quant_config") or {}
|
||||
).get("weight") or {}
|
||||
if shared_weight_config.get("dtype") == "fp4":
|
||||
return True, None
|
||||
return (
|
||||
False,
|
||||
f"DeepSeek-V4 shared experts at {shared_expert_prefix} are not MXFP4",
|
||||
)
|
||||
|
||||
if isinstance(quant_config, QuarkConfig):
|
||||
# TODO: layer_type_quant_config is not taken into account here.
|
||||
assert "exclude" in quant_config.quant_config
|
||||
assert "global_quant_config" in quant_config.quant_config
|
||||
|
||||
is_compatible = not any(
|
||||
"shared_expert" in str(entry)
|
||||
for entry in quant_config.quant_config["exclude"]
|
||||
)
|
||||
if not is_compatible:
|
||||
return False, f"Quark excludes shared experts at {shared_expert_prefix}"
|
||||
|
||||
global_quant_config = quant_config.quant_config["global_quant_config"]
|
||||
|
||||
def get_projection_quant_configs(layer_name: str) -> list[object]:
|
||||
module_prefix, _, projection_name = layer_name.rpartition(".")
|
||||
packed_projection_names = quant_config.packed_modules_mapping.get(
|
||||
projection_name, [projection_name]
|
||||
)
|
||||
return [
|
||||
quant_config.get_layer_quant_config_from_name(
|
||||
f"{module_prefix}.{packed_projection_name}"
|
||||
)
|
||||
or global_quant_config
|
||||
for packed_projection_name in packed_projection_names
|
||||
]
|
||||
|
||||
expert_quant_config = (
|
||||
quant_config.get_layer_quant_config_from_name(expert_prefix)
|
||||
or global_quant_config
|
||||
)
|
||||
shared_expert_quant_configs = [
|
||||
config
|
||||
for projection_name in projection_names
|
||||
for config in get_projection_quant_configs(
|
||||
f"{shared_expert_prefix}.{projection_name}"
|
||||
)
|
||||
]
|
||||
if all(config == expert_quant_config for config in shared_expert_quant_configs):
|
||||
return True, None
|
||||
return (
|
||||
False,
|
||||
"Quark uses different quantization configurations for routed and "
|
||||
f"shared experts at {shared_expert_prefix}",
|
||||
)
|
||||
|
||||
# TODO: Extend FSE support detection to other quantization methods. Typically,
|
||||
# one would check that the experts and shared_experts use the same
|
||||
# quantization config. This may be refactored as part of QuantizationConfig later.
|
||||
|
||||
return (
|
||||
False,
|
||||
"shared-expert FSE quantization compatibility is not implemented for "
|
||||
f"{type(quant_config).__name__}",
|
||||
)
|
||||
@@ -46,6 +46,10 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
resolve_layer_fused_shared_expert,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -148,10 +152,14 @@ class AXK1MoE(nn.Module):
|
||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||
|
||||
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
||||
self.is_fusion_moe_shared_experts_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
|
||||
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if config.n_shared_experts is not None:
|
||||
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
|
||||
quant_config, prefix
|
||||
)
|
||||
|
||||
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
|
||||
self.shared_experts = None
|
||||
else:
|
||||
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
|
||||
@@ -189,8 +197,9 @@ class AXK1MoE(nn.Module):
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
n_shared_experts=config.n_shared_experts
|
||||
if self.is_fusion_moe_shared_experts_enabled
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else None,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -725,6 +734,12 @@ class AXK1Model(nn.Module):
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
AXK1MoE,
|
||||
"mlp",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
@@ -789,9 +804,6 @@ class AXK1Model(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping: list[tuple[str, str, int | str]] = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
@@ -822,7 +834,7 @@ class AXK1Model(nn.Module):
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
(self.config.n_shared_experts or 0)
|
||||
if rocm_aiter_moe_shared_expert_enabled
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
@@ -839,7 +851,7 @@ class AXK1Model(nn.Module):
|
||||
continue # skip spec decode layers for main model
|
||||
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
|
||||
@@ -7,13 +7,15 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -255,6 +257,11 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
self.moe_mlp_layers.append(layer.mlp)
|
||||
self.moe_layers.append(layer.mlp.experts)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.model.layers.values(),
|
||||
DeepseekV2MoE,
|
||||
"mtp_block.mlp",
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
@@ -285,9 +292,6 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
@@ -310,7 +314,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if rocm_aiter_moe_shared_expert_enabled
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
)
|
||||
@@ -326,7 +330,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
if spec_layer is None:
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
|
||||
@@ -54,6 +54,10 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
resolve_layer_fused_shared_expert,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -329,9 +333,13 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||
|
||||
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
||||
self.is_fusion_moe_shared_experts_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if config.n_shared_experts is not None:
|
||||
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
|
||||
quant_config, prefix
|
||||
)
|
||||
|
||||
if (
|
||||
self.is_rocm_aiter_moe_enabled
|
||||
and self.gate.e_score_correction_bias is not None
|
||||
@@ -340,7 +348,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
# Accumulates in fp32; avoids bf16->fp32 cast.
|
||||
self.gate.set_out_dtype(self.gate.weight.dtype)
|
||||
|
||||
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
|
||||
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
|
||||
self.shared_experts = None
|
||||
else:
|
||||
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
|
||||
@@ -377,8 +385,9 @@ class DeepseekV2MoE(nn.Module):
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
reduce_results=reduce_results,
|
||||
n_shared_experts=config.n_shared_experts
|
||||
if self.is_fusion_moe_shared_experts_enabled
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else None,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
router_logits_dtype=self.gate.out_dtype,
|
||||
)
|
||||
|
||||
@@ -1401,6 +1410,12 @@ class DeepseekV2Model(nn.Module):
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
DeepseekV2MoE,
|
||||
"mlp",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(self.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
@@ -1511,9 +1526,6 @@ class DeepseekV2Model(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping: list[tuple[str, str, int | str]] = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
@@ -1554,7 +1566,7 @@ class DeepseekV2Model(nn.Module):
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if rocm_aiter_moe_shared_expert_enabled
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
@@ -1582,7 +1594,7 @@ class DeepseekV2Model(nn.Module):
|
||||
continue # this layer has no indexer; drop its checkpoint weights
|
||||
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
|
||||
if _try_load_fp8_indexer_wk(
|
||||
|
||||
@@ -39,12 +39,15 @@ from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
resolve_layer_fused_shared_expert,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -72,8 +75,6 @@ from .utils import (
|
||||
skip_spec_layers,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Glm4MoeMLP(nn.Module):
|
||||
def __init__(
|
||||
@@ -159,14 +160,15 @@ class Glm4MoE(nn.Module):
|
||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||
|
||||
# AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py
|
||||
# pattern (see Glm4MoE / MoERunner wiring there).
|
||||
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
||||
self.is_fusion_moe_shared_experts_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
|
||||
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if config.n_shared_experts is not None:
|
||||
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
|
||||
quant_config, prefix
|
||||
)
|
||||
|
||||
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
|
||||
self.shared_experts = None
|
||||
else:
|
||||
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
|
||||
@@ -200,10 +202,9 @@ class Glm4MoE(nn.Module):
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
router_logits_dtype=torch.float32,
|
||||
n_shared_experts=(
|
||||
config.n_shared_experts
|
||||
if self.is_fusion_moe_shared_experts_enabled
|
||||
else None
|
||||
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
|
||||
),
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -443,6 +444,12 @@ class Glm4MoeModel(nn.Module):
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
Glm4MoE,
|
||||
"mlp",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
@@ -488,6 +495,7 @@ class Glm4MoeModel(nn.Module):
|
||||
skip_spec_layers(weights, self.config),
|
||||
n_routed_experts=self.config.n_routed_experts,
|
||||
n_shared_experts=self.config.n_shared_experts or 1,
|
||||
enabled=self.is_fused_shared_expert_enabled,
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -34,7 +34,6 @@ from torch import nn
|
||||
if TYPE_CHECKING:
|
||||
from transformers.models.glm4_moe_lite import Glm4MoeLiteConfig
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
@@ -44,6 +43,9 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
@@ -254,6 +256,12 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
Glm4MoeLite,
|
||||
"mlp",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
@@ -320,9 +328,6 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
@@ -345,7 +350,7 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if rocm_aiter_moe_shared_expert_enabled
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
)
|
||||
@@ -361,7 +366,7 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
continue # skip spec decode layers for main model
|
||||
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
|
||||
@@ -30,12 +30,14 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -224,6 +226,11 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
||||
self.moe_mlp_layers.append(layer.mlp)
|
||||
self.moe_layers.append(layer.mlp.experts)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.model.layers.values(),
|
||||
Glm4MoeLite,
|
||||
"mtp_block.mlp",
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
@@ -250,9 +257,6 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
@@ -268,7 +272,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if rocm_aiter_moe_shared_expert_enabled
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
)
|
||||
@@ -282,7 +286,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
||||
if spec_layer is None:
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
|
||||
@@ -31,12 +31,14 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -212,6 +214,11 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
|
||||
self.moe_mlp_layers.append(layer.mlp)
|
||||
self.moe_layers.append(layer.mlp.experts)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.model.layers.values(),
|
||||
Glm4MoE,
|
||||
"mtp_block.mlp",
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
@@ -238,10 +245,6 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# FSE weight loading mirrors glm4_moe.py / deepseek_mtp.py.
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
@@ -254,7 +257,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
num_experts = self.config.n_routed_experts
|
||||
if rocm_aiter_moe_shared_expert_enabled and self.config.n_shared_experts:
|
||||
if self.is_fused_shared_expert_enabled and self.config.n_shared_experts:
|
||||
num_experts += self.config.n_shared_experts
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
@@ -279,7 +282,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
|
||||
@@ -29,13 +29,15 @@ from collections.abc import Iterable
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
@@ -78,7 +80,6 @@ from .qwen3_next import (
|
||||
Qwen3NextModel,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
QwenNextMixtureOfExperts,
|
||||
_is_shared_expert_fse_compatible,
|
||||
)
|
||||
from .qwen3_vl import (
|
||||
Qwen3_VisionTransformer,
|
||||
@@ -256,6 +257,11 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
"mlp",
|
||||
)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
@@ -274,8 +280,7 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
if "moe" in self.config.model_type:
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
and _is_shared_expert_fse_compatible(self.quant_config),
|
||||
enabled=self.is_fused_shared_expert_enabled,
|
||||
n_routed_experts=self.config.num_experts,
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
|
||||
@@ -7,11 +7,13 @@ from collections.abc import Iterable
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
@@ -25,8 +27,8 @@ from vllm.model_executor.models.qwen3_5 import (
|
||||
Qwen3_5RMSNorm,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_next import (
|
||||
Qwen3NextSparseMoeBlock,
|
||||
QwenNextMixtureOfExperts,
|
||||
_is_shared_expert_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -122,6 +124,11 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
||||
for idx in range(self.num_mtp_layers)
|
||||
)
|
||||
vllm_config.quant_config = original_quant
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
"mlp",
|
||||
)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
@@ -185,10 +192,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
and _is_shared_expert_fse_compatible(
|
||||
get_current_vllm_config().quant_config
|
||||
),
|
||||
enabled=self.is_fused_shared_expert_enabled,
|
||||
n_routed_experts=getattr(self.config, "num_experts", 0),
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
|
||||
@@ -8,7 +8,6 @@ from itertools import islice
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
@@ -18,9 +17,12 @@ from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
resolve_layer_fused_shared_expert,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
|
||||
from vllm.model_executor.layers.layernorm import (
|
||||
GemmaRMSNorm as Qwen3NextRMSNorm,
|
||||
@@ -73,8 +75,6 @@ from .utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
KVCache = tuple[torch.Tensor, torch.Tensor]
|
||||
|
||||
|
||||
@@ -90,26 +90,6 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _is_shared_expert_fse_compatible(quant_config) -> bool:
|
||||
"""Check if shared expert can be fused with routed experts.
|
||||
|
||||
FSE requires that shared and routed expert weights use the same
|
||||
quantization format. Returns False when the shared expert is
|
||||
excluded from quantization (e.g. float32 shared in an MXFP4 model)
|
||||
or has a different quant spec than routed experts.
|
||||
"""
|
||||
if quant_config is None:
|
||||
return True
|
||||
# Quark stores its full config dict in quant_config.quant_config
|
||||
raw_config = getattr(quant_config, "quant_config", None)
|
||||
if not isinstance(raw_config, dict):
|
||||
return True
|
||||
exclude = raw_config.get("exclude", [])
|
||||
if not exclude:
|
||||
return True
|
||||
return not any("shared_expert." in str(e) for e in exclude)
|
||||
|
||||
|
||||
class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
@@ -157,15 +137,18 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
prefix=f"{prefix}.shared_expert_gate",
|
||||
)
|
||||
|
||||
_fse_requested = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
_fse_enabled = _fse_requested and _is_shared_expert_fse_compatible(quant_config)
|
||||
if _fse_requested and not _fse_enabled:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
"shared expert has a different quantization spec than routed "
|
||||
"experts. Falling back to non-fused shared expert path."
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
|
||||
quant_config,
|
||||
prefix,
|
||||
shared_expert_name="shared_expert",
|
||||
)
|
||||
if _fse_enabled or config.shared_expert_intermediate_size <= 0:
|
||||
|
||||
if (
|
||||
self.is_fused_shared_expert_enabled
|
||||
or config.shared_expert_intermediate_size <= 0
|
||||
):
|
||||
self.shared_expert = None
|
||||
else:
|
||||
self.shared_expert = Qwen3NextMLP(
|
||||
@@ -193,6 +176,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
n_shared_experts=1 if self.shared_expert is None else None,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_gate=self.shared_expert_gate
|
||||
if self.shared_expert is None
|
||||
else None,
|
||||
@@ -599,6 +583,11 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
"mlp",
|
||||
)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
@@ -678,6 +667,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
enabled=self.is_fused_shared_expert_enabled,
|
||||
n_routed_experts=getattr(self.config, "num_experts", 0),
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
|
||||
@@ -11,6 +11,9 @@ from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
@@ -21,6 +24,7 @@ from vllm.model_executor.models.qwen3_next import (
|
||||
Qwen3NextDecoderLayer,
|
||||
Qwen3NextModel,
|
||||
Qwen3NextRMSNorm,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
QwenNextMixtureOfExperts,
|
||||
)
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
@@ -91,6 +95,11 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
|
||||
for idx in range(self.num_mtp_layers)
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
"mlp",
|
||||
)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
@@ -155,6 +164,7 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
enabled=self.is_fused_shared_expert_enabled,
|
||||
n_routed_experts=self.config.num_experts,
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
|
||||
@@ -8,12 +8,14 @@ from collections.abc import Callable, Iterable
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
@@ -167,6 +169,11 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.set_moe_parameters()
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.model.layers.values(),
|
||||
DeepseekV2MoE,
|
||||
"mtp_block.mlp",
|
||||
)
|
||||
|
||||
def set_moe_parameters(self):
|
||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||
@@ -231,9 +238,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
return name
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
@@ -250,7 +254,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if rocm_aiter_moe_shared_expert_enabled
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
)
|
||||
@@ -266,7 +270,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
if spec_layer is None:
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_embed_norm import (
|
||||
@@ -344,9 +343,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
return name
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
@@ -360,12 +356,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if rocm_aiter_moe_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
|
||||
pp_missing_layer_names = get_pp_missing_layer_names(self)
|
||||
@@ -378,9 +369,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
if _try_load_fp8_indexer_wk(
|
||||
@@ -398,8 +386,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
continue
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if (
|
||||
param_name == "fused_qkv_a_proj"
|
||||
@@ -415,32 +401,10 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
break
|
||||
else:
|
||||
num_chunks = 1
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
|
||||
split_dim = (
|
||||
1
|
||||
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
|
||||
else 0
|
||||
)
|
||||
total = loaded_weight.shape[split_dim]
|
||||
assert total % num_chunks == 0
|
||||
chunk_size = total // num_chunks
|
||||
|
||||
for j in range(num_chunks):
|
||||
chunk_name = name
|
||||
weight_to_load = loaded_weight
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
|
||||
if loaded_weight.ndim == 1:
|
||||
weight_to_load = loaded_weight[chunk_slice]
|
||||
elif split_dim == 0:
|
||||
weight_to_load = loaded_weight[chunk_slice, :]
|
||||
else:
|
||||
weight_to_load = loaded_weight[:, chunk_slice]
|
||||
chunk_name = name.replace(
|
||||
"mlp.shared_experts",
|
||||
f"mlp.experts.{self.config.n_routed_experts + j}",
|
||||
)
|
||||
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
@@ -462,10 +426,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
if not is_fusion_moe_shared_experts_layer:
|
||||
name = name_mapped
|
||||
else:
|
||||
loaded_params.add(name_mapped)
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
@@ -485,8 +446,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
if not is_fusion_moe_shared_experts_layer:
|
||||
loaded_params.add(name)
|
||||
loaded_params.add(name)
|
||||
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
|
||||
@@ -16,12 +16,16 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -37,6 +41,9 @@ from vllm.model_executor.layers.mhc import (
|
||||
MHCPreOp,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
@@ -60,6 +67,8 @@ from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DeepseekV4MLP(nn.Module):
|
||||
def __init__(
|
||||
@@ -154,48 +163,11 @@ class DeepseekV4MLP(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
def _shared_experts_are_fp4(config, layer_idx: int | None = None) -> bool:
|
||||
"""Whether the shared experts are MXFP4 and thus fusable.
|
||||
|
||||
``layer_idx=None`` resolves the model-wide default (global scheme), used by
|
||||
the main-model weight loader / mapper callers that operate per-model.
|
||||
"""
|
||||
quant_cfg = getattr(config, "quantization_config", None)
|
||||
if quant_cfg is None:
|
||||
return False
|
||||
if layer_idx is None:
|
||||
base = None
|
||||
elif layer_idx >= config.num_hidden_layers:
|
||||
base = f"mtp.{layer_idx - config.num_hidden_layers}.ffn.shared_experts"
|
||||
else:
|
||||
base = f"layers.{layer_idx}.ffn.shared_experts"
|
||||
if base and any(e.startswith(base) for e in (quant_cfg.get("exclude") or [])):
|
||||
return False
|
||||
entry = (
|
||||
(quant_cfg.get("layer_quant_config") or {}).get(f"{base}.w1") if base else None
|
||||
)
|
||||
if entry is None:
|
||||
entry = quant_cfg.get("global_quant_config")
|
||||
return ((entry or {}).get("weight") or {}).get("dtype") == "fp4"
|
||||
|
||||
|
||||
def _fuse_shared_experts_enabled(config, prefix: str = "") -> bool:
|
||||
"""Whether to fuse the shared expert into the routed MXFP4 grouped GEMM.
|
||||
|
||||
Fusion fuses the shared expert into the routed experts' MXFP4 grouped GEMM,
|
||||
so it only applies where the shared expert is the same precision as the
|
||||
routed experts. Some layers may carry a shared expert in a different quantization
|
||||
than the routed experts; when so, it runs as its own linear and must not be fused.
|
||||
"""
|
||||
if not (
|
||||
current_platform.is_rocm()
|
||||
and getattr(config, "n_shared_experts", None)
|
||||
def _fuse_shared_experts_enabled(config) -> bool:
|
||||
return bool(
|
||||
getattr(config, "n_shared_experts", None)
|
||||
and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
and not get_current_vllm_config().parallel_config.enable_expert_parallel
|
||||
):
|
||||
return False
|
||||
return _shared_experts_are_fp4(
|
||||
config, extract_layer_index(prefix) if prefix else None
|
||||
)
|
||||
|
||||
|
||||
@@ -255,9 +227,25 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
self.n_shared_experts = config.n_shared_experts
|
||||
|
||||
self.fuse_shared_experts = _fuse_shared_experts_enabled(config, prefix)
|
||||
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
|
||||
# is checked to enable FSE for DeepSeek-v4, despite AITER not being used.
|
||||
# This should be cleaned up and use `resolve_layer_fused_shared_expert`.
|
||||
fse_requested = _fuse_shared_experts_enabled(config)
|
||||
if fse_requested:
|
||||
fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
f"{prefix}.experts",
|
||||
f"{prefix}.shared_experts",
|
||||
)
|
||||
if not fse_compatible:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
"cannot be enabled: %s.",
|
||||
fse_reason,
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = fse_requested and fse_compatible
|
||||
|
||||
if config.n_shared_experts is None or self.fuse_shared_experts:
|
||||
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
|
||||
self.shared_experts = None
|
||||
else:
|
||||
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
|
||||
@@ -282,8 +270,9 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
n_shared_experts=(
|
||||
config.n_shared_experts if self.fuse_shared_experts else None
|
||||
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
|
||||
),
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
@@ -581,6 +570,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
DeepseekV4MoE,
|
||||
"ffn",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps)
|
||||
@@ -763,7 +757,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# diverge from how the module was built if per-layer quantization ever
|
||||
# mixes fused and non-fused layers.
|
||||
fuse_by_layer = {
|
||||
extract_layer_index(mod_name): mod.fuse_shared_experts
|
||||
extract_layer_index(mod_name): mod.is_fused_shared_expert_enabled
|
||||
for mod_name, mod in self.named_modules()
|
||||
if isinstance(mod, DeepseekV4MoE)
|
||||
}
|
||||
@@ -874,7 +868,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# redirected shared-expert weights route through the expert loader.
|
||||
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
|
||||
num_experts = self.config.n_routed_experts + (
|
||||
n_shared if _fuse_shared_experts_enabled(self.config) else 0
|
||||
n_shared if self.is_fused_shared_expert_enabled else 0
|
||||
)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
@@ -956,15 +950,14 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
expert_dtype = getattr(config, "expert_dtype", "fp4")
|
||||
fuse_shared_experts = _fuse_shared_experts_enabled(config)
|
||||
if expert_dtype != "fp4" or fuse_shared_experts:
|
||||
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
|
||||
expert_dtype, fuse_shared_experts=fuse_shared_experts
|
||||
)
|
||||
|
||||
self.model = self.model_cls(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
if expert_dtype != "fp4" or self.model.is_fused_shared_expert_enabled:
|
||||
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
|
||||
expert_dtype,
|
||||
fuse_shared_experts=self.model.is_fused_shared_expert_enabled,
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.config import (
|
||||
)
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention.attention import set_default_quant_scales
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
@@ -45,6 +46,9 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
MinimaxM3QKVParallelLinearWithIndexer,
|
||||
@@ -53,6 +57,9 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -108,23 +115,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
|
||||
|
||||
def _fuse_shared_experts_enabled(config: PretrainedConfig) -> bool:
|
||||
"""Whether to fuse the shared expert with routed experts.
|
||||
|
||||
ROCm only. Opt-in via ``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS`` (the
|
||||
router-append fusion runs on both aiter and non-aiter MoE);
|
||||
it is disabled under expert parallelism (the shared slot is appended to
|
||||
the routed top-k, which the EP expert-mapping path does not handle).
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return bool(
|
||||
current_platform.is_rocm()
|
||||
and getattr(config, "n_shared_experts", None)
|
||||
and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
and not get_current_vllm_config().parallel_config.enable_expert_parallel
|
||||
)
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]:
|
||||
@@ -303,10 +294,12 @@ class MiniMaxM3MLP(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool:
|
||||
def _aiter_moe_fused_shared_experts_enabled(
|
||||
is_fused_shared_expert_enabled: bool,
|
||||
) -> bool:
|
||||
"""Whether the fused shared expert routes through aiter's grouped top-k MoE.
|
||||
|
||||
A strict sub-case of :func:`_fuse_shared_experts_enabled`: shared-expert
|
||||
A strict sub-case of `is_fused_shared_expert_enabled`: shared-expert
|
||||
fusion must already be opted in (``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS``)
|
||||
and allowed (not under expert parallelism). When additionally on gfx950 with
|
||||
an active aiter MoE backend, the shared expert is appended inside aiter's
|
||||
@@ -314,11 +307,13 @@ def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool:
|
||||
vLLM router's torch concat. Otherwise FSE still runs via the vLLM top-k bias
|
||||
router.
|
||||
"""
|
||||
if not _fuse_shared_experts_enabled(config):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950() and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
return (
|
||||
on_gfx950()
|
||||
and is_fused_shared_expert_enabled
|
||||
and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3MoE(nn.Module):
|
||||
@@ -371,11 +366,46 @@ class MiniMaxM3MoE(nn.Module):
|
||||
# MoE call as the last expert slot, so we don't build a separate module.
|
||||
# On gfx950 with aiter MoE the append is fused inside aiter's grouped
|
||||
# top-k kernel; otherwise it goes through the vLLM top-k bias router.
|
||||
self.fuse_shared_experts = _fuse_shared_experts_enabled(config)
|
||||
self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(config)
|
||||
# It is disabled under expert parallelism (the shared slot is appended to
|
||||
# the routed top-k, which the EP expert-mapping path does not handle).
|
||||
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
|
||||
# is checked to enable FSE for MiniMax-M3, despite AITER not being used.
|
||||
# This should be cleaned up and use `resolve_layer_fused_shared_expert`.
|
||||
fse_requested = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if (
|
||||
fse_requested
|
||||
and bool(getattr(config, "n_shared_experts", None))
|
||||
and not get_current_vllm_config().parallel_config.enable_expert_parallel
|
||||
):
|
||||
fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
f"{prefix}.experts",
|
||||
f"{prefix}.shared_experts",
|
||||
)
|
||||
if not fse_compatible:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
"cannot be enabled: %s.",
|
||||
fse_reason,
|
||||
)
|
||||
else:
|
||||
self.is_fused_shared_expert_enabled = True
|
||||
|
||||
# When additionally on gfx950 with an active aiter MoE backend, the shared
|
||||
# expert is appended inside aiter's an active aiter MoE backend, the shared
|
||||
# expert is appended inside aiter's biased grouped top-k kernel
|
||||
# (``num_fused_shared_experts``) instead of the vLLM router's torch concat.
|
||||
# Otherwise FSE still runs via the vLLM top-k bias router.
|
||||
# TODO: `on_gfx950()` check here should not be MiniMax-M3 specific, and
|
||||
# the check should be done on resolved MOE backend directly.
|
||||
self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(
|
||||
self.is_fused_shared_expert_enabled
|
||||
)
|
||||
|
||||
self.shared_experts: MiniMaxM3MLP | None = None
|
||||
if self.n_shared_experts and not self.fuse_shared_experts:
|
||||
if self.n_shared_experts and not self.is_fused_shared_expert_enabled:
|
||||
self.shared_experts = MiniMaxM3MLP(
|
||||
config=config,
|
||||
intermediate_size=config.intermediate_size * self.n_shared_experts,
|
||||
@@ -412,8 +442,9 @@ class MiniMaxM3MoE(nn.Module):
|
||||
router_logits_dtype=self.gate.out_dtype,
|
||||
shared_experts=self.shared_experts,
|
||||
n_shared_experts=(
|
||||
self.n_shared_experts if self.fuse_shared_experts else None
|
||||
self.n_shared_experts if self.is_fused_shared_expert_enabled else None
|
||||
),
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
@@ -1101,6 +1132,11 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
MiniMaxM3MoE,
|
||||
"block_sparse_moe",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -1155,7 +1191,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
# expert, include the appended slot (id == num_local_experts).
|
||||
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
|
||||
num_experts = self.config.num_local_experts + (
|
||||
n_shared if _fuse_shared_experts_enabled(self.config) else 0
|
||||
n_shared if self.is_fused_shared_expert_enabled else 0
|
||||
)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
@@ -1187,8 +1223,6 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
_fuse_shared = _fuse_shared_experts_enabled(self.config)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
@@ -1206,7 +1240,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
# down->w2) so it loads via the routed expert loader. Runs before the
|
||||
# stacked/dense mappings so shared_experts.gate_proj/up_proj are not
|
||||
# captured by the dense gate_up_proj mapping.
|
||||
if _fuse_shared and ".shared_experts." in name:
|
||||
if self.is_fused_shared_expert_enabled and ".shared_experts." in name:
|
||||
sid = self.config.num_local_experts
|
||||
name = name.replace(".shared_experts.gate_proj.", f".experts.{sid}.w1.")
|
||||
name = name.replace(".shared_experts.up_proj.", f".experts.{sid}.w3.")
|
||||
|
||||
Reference in New Issue
Block a user