[MOE] Standardize and abstract fused shared expert optimization selection (#51695)
Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
@@ -6,7 +6,6 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import ParallelConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
@@ -74,23 +73,13 @@ def determine_expert_counts(
|
||||
num_experts: int,
|
||||
num_redundant_experts: int,
|
||||
n_shared_experts: int | None,
|
||||
is_act_and_mul: bool,
|
||||
fuse_shared_experts: bool,
|
||||
) -> tuple[int, int, int]:
|
||||
global_num_experts = num_experts + num_redundant_experts
|
||||
logical_num_experts = num_experts
|
||||
# Shared-expert fusion: append the shared expert(s) as routed-expert slots
|
||||
# so they run in the same grouped GEMM. Gated by
|
||||
# VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: either the native aiter fused-MoE
|
||||
# path (env + master switch, via is_fusion_moe_shared_experts_enabled) or the
|
||||
# backend-neutral router-append path (env alone, independent of the master
|
||||
# switch; e.g. the MM3 triton/flydsl mxfp8 MoE). Gated activations only.
|
||||
fuse_shared_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
or envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
) and is_act_and_mul
|
||||
|
||||
num_fused_shared_experts = (
|
||||
n_shared_experts if n_shared_experts is not None and fuse_shared_enabled else 0
|
||||
n_shared_experts if n_shared_experts is not None and fuse_shared_experts else 0
|
||||
)
|
||||
|
||||
return global_num_experts, logical_num_experts, num_fused_shared_experts
|
||||
@@ -132,6 +121,7 @@ def FusedMoEFactory(
|
||||
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
|
||||
is_fused_checkpoint_transposed: bool = False,
|
||||
n_shared_experts: int | None = None,
|
||||
fuse_shared_experts: bool = False,
|
||||
router_logits_dtype: torch.dtype | None = None,
|
||||
gate: torch.nn.Module | None = None,
|
||||
shared_experts: torch.nn.Module | None = None,
|
||||
@@ -198,6 +188,7 @@ def FusedMoEFactory(
|
||||
block scales use transposed storage.
|
||||
n_shared_experts: Number of shared experts to fuse into the routed
|
||||
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
|
||||
fuse_shared_experts: Whether to enable shared-expert fusion.
|
||||
router_logits_dtype: Data type for router logits buffers
|
||||
gate: Pre-configured gate module
|
||||
shared_experts: Pre-configured shared experts module
|
||||
@@ -244,7 +235,7 @@ def FusedMoEFactory(
|
||||
num_experts,
|
||||
num_redundant_experts,
|
||||
n_shared_experts,
|
||||
is_act_and_mul,
|
||||
fuse_shared_experts,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -1,15 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
from collections.abc import Iterable
|
||||
from math import prod
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
@@ -34,16 +40,95 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
per_tensor_dequantize,
|
||||
)
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def resolve_layer_fused_shared_expert(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
prefix: str,
|
||||
shared_expert_name: str = "shared_experts",
|
||||
) -> bool:
|
||||
"""Resolve whether AITER fused shared-expert execution is enabled.
|
||||
|
||||
Args:
|
||||
quant_config: Model quantization configuration.
|
||||
prefix: MoE module prefix.
|
||||
shared_expert_name: Shared-expert module name under ``prefix``.
|
||||
|
||||
Returns:
|
||||
Whether AITER fused shared experts are enabled.
|
||||
|
||||
Raises:
|
||||
ValueError: If requested shared-expert fusion is quantization-incompatible.
|
||||
"""
|
||||
# NOTE: is_fusion_moe_shared_experts_enabled is decorated with @if_aiter_supported
|
||||
# that returns None if AITER is not available.
|
||||
fse_requested = bool(rocm_aiter_ops.is_fusion_moe_shared_experts_enabled())
|
||||
fse_compatible, fse_reason = (
|
||||
is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
f"{prefix}.experts",
|
||||
f"{prefix}.{shared_expert_name}",
|
||||
)
|
||||
if fse_requested
|
||||
else (True, None)
|
||||
)
|
||||
is_fused_shared_expert_enabled = fse_requested and fse_compatible
|
||||
if fse_requested and not is_fused_shared_expert_enabled:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
"cannot be enabled: %s.",
|
||||
fse_reason,
|
||||
)
|
||||
return is_fused_shared_expert_enabled
|
||||
|
||||
|
||||
def is_model_fused_shared_expert_compatible(
|
||||
layers: nn.ModuleList | Iterable[nn.Module],
|
||||
moe_cls: type[nn.Module],
|
||||
moe_name: str,
|
||||
) -> bool:
|
||||
"""Resolve one fused-shared-expert state for a model's MoE layers."""
|
||||
|
||||
def get_moe_layer(layer: nn.Module) -> nn.Module | None:
|
||||
for name in moe_name.split("."):
|
||||
layer = getattr(layer, name, None)
|
||||
if layer is None:
|
||||
return None
|
||||
return layer
|
||||
|
||||
moe_layers = (
|
||||
moe_layer
|
||||
for layer in layers
|
||||
if not isinstance(layer, PPMissingLayer)
|
||||
and (moe_layer := get_moe_layer(layer)) is not None
|
||||
and isinstance(moe_layer, moe_cls)
|
||||
)
|
||||
|
||||
enabled = [
|
||||
getattr(layer, "is_fused_shared_expert_enabled", False) for layer in moe_layers
|
||||
]
|
||||
enabled_count = sum(enabled)
|
||||
disabled_count = len(enabled) - enabled_count
|
||||
if enabled_count > 0 and disabled_count > 0:
|
||||
raise NotImplementedError(
|
||||
"Fused shared experts must be enabled for all MoE layers; found "
|
||||
f"{enabled_count} enabled and {disabled_count} disabled layers. "
|
||||
"Per-layer fused shared experts is not yet supported. Please open "
|
||||
"an issue."
|
||||
)
|
||||
return enabled_count > 0 and disabled_count == 0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _count_expert_num_tokens(
|
||||
topk_ids_ptr,
|
||||
|
||||
@@ -576,63 +576,91 @@ class QuarkConfig(QuantizationConfig):
|
||||
|
||||
return True
|
||||
|
||||
def _find_matched_config(
|
||||
self, layer_name: str, module: torch.nn.Module
|
||||
) -> dict[str, Any]:
|
||||
def get_layer_quant_config_from_name(
|
||||
self, layer_name: str
|
||||
) -> dict[str, Any] | None:
|
||||
proj_name = layer_name.split(".")[-1]
|
||||
if proj_name in self.packed_modules_mapping:
|
||||
shard_proj_names = self.packed_modules_mapping[proj_name]
|
||||
|
||||
# Convert fused_name --> [shard_names]
|
||||
shard_names = [
|
||||
layer_name.replace(proj_name, shard_proj_name)
|
||||
for shard_proj_name in shard_proj_names
|
||||
]
|
||||
|
||||
shard_configs = []
|
||||
for shard_name in shard_names:
|
||||
if shard_name == layer_name:
|
||||
config = cast(
|
||||
dict[str, Any], self.quant_config.get("global_quant_config")
|
||||
)
|
||||
for shard_proj_name in shard_proj_names:
|
||||
shard_name = layer_name.replace(proj_name, shard_proj_name)
|
||||
if shard_name != layer_name:
|
||||
config = self.get_layer_quant_config_from_name(shard_name)
|
||||
else:
|
||||
config = self._find_matched_config(shard_name, module)
|
||||
config = None
|
||||
shard_configs.append(config)
|
||||
|
||||
if not all(
|
||||
deep_compare(q_config, shard_configs[0]) for q_config in shard_configs
|
||||
matched_configs = [config for config in shard_configs if config is not None]
|
||||
if matched_configs and not all(
|
||||
deep_compare(config, matched_configs[0]) for config in matched_configs
|
||||
):
|
||||
raise ValueError(
|
||||
f"Found a different quantization configuration for "
|
||||
f"{shard_proj_names} in {layer_name}. vLLM "
|
||||
"requires all to use the same scheme."
|
||||
)
|
||||
return shard_configs[0]
|
||||
if matched_configs:
|
||||
return matched_configs[0]
|
||||
return None
|
||||
else:
|
||||
layer_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("layer_quant_config")
|
||||
dict[str, Any], self.quant_config.get("layer_quant_config") or {}
|
||||
)
|
||||
|
||||
def _matches_pattern(layer_name, pattern):
|
||||
if "*" not in pattern:
|
||||
return layer_name in pattern
|
||||
return fnmatch.fnmatch(layer_name, pattern)
|
||||
|
||||
for name_pattern, config in layer_quant_config.items():
|
||||
if _matches_pattern(layer_name, name_pattern):
|
||||
if "*" not in name_pattern:
|
||||
matches = layer_name in name_pattern
|
||||
else:
|
||||
matches = fnmatch.fnmatch(layer_name, name_pattern)
|
||||
if matches:
|
||||
return config
|
||||
return None
|
||||
|
||||
layer_type = cast(str, type(module))
|
||||
layer_type_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("layer_type_quant_config")
|
||||
)
|
||||
if layer_type in layer_type_quant_config:
|
||||
return layer_type_quant_config[layer_type]
|
||||
def _find_matched_config(
|
||||
self, layer_name: str, module: torch.nn.Module
|
||||
) -> dict[str, Any]:
|
||||
# Priority order:
|
||||
# 1. layer_quant_config,
|
||||
# 2. layer_type_quant_config,
|
||||
# 3. global_quant_config.
|
||||
|
||||
global_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("global_quant_config")
|
||||
)
|
||||
return global_quant_config
|
||||
layer_type = cast(str, type(module))
|
||||
layer_type_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("layer_type_quant_config")
|
||||
)
|
||||
global_quant_config = cast(
|
||||
dict[str, Any], self.quant_config.get("global_quant_config")
|
||||
)
|
||||
fallback_config = layer_type_quant_config.get(layer_type, global_quant_config)
|
||||
|
||||
proj_name = layer_name.split(".")[-1]
|
||||
if proj_name in self.packed_modules_mapping:
|
||||
shard_proj_names = self.packed_modules_mapping[proj_name]
|
||||
shard_configs = []
|
||||
for shard_proj_name in shard_proj_names:
|
||||
shard_name = layer_name.replace(proj_name, shard_proj_name)
|
||||
if shard_name == layer_name:
|
||||
config = fallback_config
|
||||
else:
|
||||
config = self.get_layer_quant_config_from_name(shard_name)
|
||||
if config is None:
|
||||
config = fallback_config
|
||||
shard_configs.append(config)
|
||||
|
||||
if not all(
|
||||
deep_compare(config, shard_configs[0]) for config in shard_configs
|
||||
):
|
||||
raise ValueError(
|
||||
f"Found a different quantization configuration for "
|
||||
f"{shard_proj_names} in {layer_name}. vLLM requires all "
|
||||
"to use the same scheme."
|
||||
)
|
||||
return shard_configs[0]
|
||||
else:
|
||||
layer_quant_config = self.get_layer_quant_config_from_name(layer_name)
|
||||
if layer_quant_config is not None:
|
||||
return layer_quant_config
|
||||
return fallback_config
|
||||
|
||||
def _get_scheme_from_config(
|
||||
self, config: dict[str, Any], dynamic_mxfp4_quant: bool = False
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
def is_shared_expert_quant_fse_compatible(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
expert_prefix: str,
|
||||
shared_expert_prefix: str,
|
||||
projection_names: list[str] | None = None,
|
||||
) -> tuple[bool, str | None]:
|
||||
"""Check whether quantization permits fused shared-expert execution.
|
||||
|
||||
Args:
|
||||
quant_config: Model quantization configuration.
|
||||
expert_prefix: Routed-expert module prefix.
|
||||
shared_expert_prefix: Shared-expert module prefix.
|
||||
projection_names: Shared-expert projection names.
|
||||
|
||||
Returns:
|
||||
A compatibility flag and, when incompatible, the reason.
|
||||
"""
|
||||
if projection_names is None:
|
||||
projection_names = ["gate_up_proj", "down_proj"]
|
||||
|
||||
if quant_config is None:
|
||||
return True, None
|
||||
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
|
||||
|
||||
if isinstance(quant_config, DeepseekV4FP8Config):
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
if quant_config.expert_dtype != "fp4":
|
||||
return False, "DeepSeek-V4 routed experts are not MXFP4"
|
||||
|
||||
hf_config = get_current_vllm_config().model_config.hf_config
|
||||
|
||||
# TODO: This is adapted from former `_shared_experts_are_fp4`, and
|
||||
# needs to be cleaned up this . There should not be Quark-specific
|
||||
# logic in DeepseekV4FP8Config.
|
||||
quantization_config = getattr(hf_config, "quantization_config", None)
|
||||
if quantization_config is None:
|
||||
return False, "DeepSeek-V4 has no quantization configuration"
|
||||
|
||||
if not quant_config._is_quark_mxfp4_ocp(quantization_config):
|
||||
return False, "DeepSeek-v4 FSE is only implemented/tested with Quark MXFP4"
|
||||
|
||||
layer_idx = extract_layer_index(shared_expert_prefix)
|
||||
if layer_idx >= hf_config.num_hidden_layers:
|
||||
shared_expert_prefix = (
|
||||
f"mtp.{layer_idx - hf_config.num_hidden_layers}.ffn.shared_experts"
|
||||
)
|
||||
else:
|
||||
shared_expert_prefix = f"layers.{layer_idx}.ffn.shared_experts"
|
||||
|
||||
if any(
|
||||
entry.startswith(shared_expert_prefix)
|
||||
for entry in quantization_config.get("exclude") or []
|
||||
if isinstance(entry, str)
|
||||
):
|
||||
return (
|
||||
False,
|
||||
f"DeepSeek-V4 excludes shared experts at {shared_expert_prefix}",
|
||||
)
|
||||
|
||||
shared_expert_weight_name = f"{shared_expert_prefix}.w1"
|
||||
layer_quant_config = quantization_config.get("layer_quant_config") or {}
|
||||
layer_config = layer_quant_config.get(shared_expert_weight_name)
|
||||
if layer_config is None:
|
||||
layer_config = next(
|
||||
(
|
||||
config
|
||||
for pattern, config in layer_quant_config.items()
|
||||
if isinstance(pattern, str)
|
||||
and pattern.startswith("re:")
|
||||
and re.fullmatch(
|
||||
pattern.removeprefix("re:"), shared_expert_weight_name
|
||||
)
|
||||
),
|
||||
None,
|
||||
)
|
||||
shared_weight_config = (
|
||||
layer_config or quantization_config.get("global_quant_config") or {}
|
||||
).get("weight") or {}
|
||||
if shared_weight_config.get("dtype") == "fp4":
|
||||
return True, None
|
||||
return (
|
||||
False,
|
||||
f"DeepSeek-V4 shared experts at {shared_expert_prefix} are not MXFP4",
|
||||
)
|
||||
|
||||
if isinstance(quant_config, QuarkConfig):
|
||||
# TODO: layer_type_quant_config is not taken into account here.
|
||||
assert "exclude" in quant_config.quant_config
|
||||
assert "global_quant_config" in quant_config.quant_config
|
||||
|
||||
is_compatible = not any(
|
||||
"shared_expert" in str(entry)
|
||||
for entry in quant_config.quant_config["exclude"]
|
||||
)
|
||||
if not is_compatible:
|
||||
return False, f"Quark excludes shared experts at {shared_expert_prefix}"
|
||||
|
||||
global_quant_config = quant_config.quant_config["global_quant_config"]
|
||||
|
||||
def get_projection_quant_configs(layer_name: str) -> list[object]:
|
||||
module_prefix, _, projection_name = layer_name.rpartition(".")
|
||||
packed_projection_names = quant_config.packed_modules_mapping.get(
|
||||
projection_name, [projection_name]
|
||||
)
|
||||
return [
|
||||
quant_config.get_layer_quant_config_from_name(
|
||||
f"{module_prefix}.{packed_projection_name}"
|
||||
)
|
||||
or global_quant_config
|
||||
for packed_projection_name in packed_projection_names
|
||||
]
|
||||
|
||||
expert_quant_config = (
|
||||
quant_config.get_layer_quant_config_from_name(expert_prefix)
|
||||
or global_quant_config
|
||||
)
|
||||
shared_expert_quant_configs = [
|
||||
config
|
||||
for projection_name in projection_names
|
||||
for config in get_projection_quant_configs(
|
||||
f"{shared_expert_prefix}.{projection_name}"
|
||||
)
|
||||
]
|
||||
if all(config == expert_quant_config for config in shared_expert_quant_configs):
|
||||
return True, None
|
||||
return (
|
||||
False,
|
||||
"Quark uses different quantization configurations for routed and "
|
||||
f"shared experts at {shared_expert_prefix}",
|
||||
)
|
||||
|
||||
# TODO: Extend FSE support detection to other quantization methods. Typically,
|
||||
# one would check that the experts and shared_experts use the same
|
||||
# quantization config. This may be refactored as part of QuantizationConfig later.
|
||||
|
||||
return (
|
||||
False,
|
||||
"shared-expert FSE quantization compatibility is not implemented for "
|
||||
f"{type(quant_config).__name__}",
|
||||
)
|
||||
Reference in New Issue
Block a user