[MOE] Standardize and abstract fused shared expert optimization selection (#51695)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-18 12:56:56 -04:00
committed by GitHub
parent f9f066d195
commit 88b2bff2c6
20 changed files with 1288 additions and 281 deletions
@@ -576,63 +576,91 @@ class QuarkConfig(QuantizationConfig):
return True
def _find_matched_config(
self, layer_name: str, module: torch.nn.Module
) -> dict[str, Any]:
def get_layer_quant_config_from_name(
self, layer_name: str
) -> dict[str, Any] | None:
proj_name = layer_name.split(".")[-1]
if proj_name in self.packed_modules_mapping:
shard_proj_names = self.packed_modules_mapping[proj_name]
# Convert fused_name --> [shard_names]
shard_names = [
layer_name.replace(proj_name, shard_proj_name)
for shard_proj_name in shard_proj_names
]
shard_configs = []
for shard_name in shard_names:
if shard_name == layer_name:
config = cast(
dict[str, Any], self.quant_config.get("global_quant_config")
)
for shard_proj_name in shard_proj_names:
shard_name = layer_name.replace(proj_name, shard_proj_name)
if shard_name != layer_name:
config = self.get_layer_quant_config_from_name(shard_name)
else:
config = self._find_matched_config(shard_name, module)
config = None
shard_configs.append(config)
if not all(
deep_compare(q_config, shard_configs[0]) for q_config in shard_configs
matched_configs = [config for config in shard_configs if config is not None]
if matched_configs and not all(
deep_compare(config, matched_configs[0]) for config in matched_configs
):
raise ValueError(
f"Found a different quantization configuration for "
f"{shard_proj_names} in {layer_name}. vLLM "
"requires all to use the same scheme."
)
return shard_configs[0]
if matched_configs:
return matched_configs[0]
return None
else:
layer_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_quant_config")
dict[str, Any], self.quant_config.get("layer_quant_config") or {}
)
def _matches_pattern(layer_name, pattern):
if "*" not in pattern:
return layer_name in pattern
return fnmatch.fnmatch(layer_name, pattern)
for name_pattern, config in layer_quant_config.items():
if _matches_pattern(layer_name, name_pattern):
if "*" not in name_pattern:
matches = layer_name in name_pattern
else:
matches = fnmatch.fnmatch(layer_name, name_pattern)
if matches:
return config
return None
layer_type = cast(str, type(module))
layer_type_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_type_quant_config")
)
if layer_type in layer_type_quant_config:
return layer_type_quant_config[layer_type]
def _find_matched_config(
self, layer_name: str, module: torch.nn.Module
) -> dict[str, Any]:
# Priority order:
# 1. layer_quant_config,
# 2. layer_type_quant_config,
# 3. global_quant_config.
global_quant_config = cast(
dict[str, Any], self.quant_config.get("global_quant_config")
)
return global_quant_config
layer_type = cast(str, type(module))
layer_type_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_type_quant_config")
)
global_quant_config = cast(
dict[str, Any], self.quant_config.get("global_quant_config")
)
fallback_config = layer_type_quant_config.get(layer_type, global_quant_config)
proj_name = layer_name.split(".")[-1]
if proj_name in self.packed_modules_mapping:
shard_proj_names = self.packed_modules_mapping[proj_name]
shard_configs = []
for shard_proj_name in shard_proj_names:
shard_name = layer_name.replace(proj_name, shard_proj_name)
if shard_name == layer_name:
config = fallback_config
else:
config = self.get_layer_quant_config_from_name(shard_name)
if config is None:
config = fallback_config
shard_configs.append(config)
if not all(
deep_compare(config, shard_configs[0]) for config in shard_configs
):
raise ValueError(
f"Found a different quantization configuration for "
f"{shard_proj_names} in {layer_name}. vLLM requires all "
"to use the same scheme."
)
return shard_configs[0]
else:
layer_quant_config = self.get_layer_quant_config_from_name(layer_name)
if layer_quant_config is not None:
return layer_quant_config
return fallback_config
def _get_scheme_from_config(
self, config: dict[str, Any], dynamic_mxfp4_quant: bool = False
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
import regex as re
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
def is_shared_expert_quant_fse_compatible(
quant_config: "QuantizationConfig | None",
expert_prefix: str,
shared_expert_prefix: str,
projection_names: list[str] | None = None,
) -> tuple[bool, str | None]:
"""Check whether quantization permits fused shared-expert execution.
Args:
quant_config: Model quantization configuration.
expert_prefix: Routed-expert module prefix.
shared_expert_prefix: Shared-expert module prefix.
projection_names: Shared-expert projection names.
Returns:
A compatibility flag and, when incompatible, the reason.
"""
if projection_names is None:
projection_names = ["gate_up_proj", "down_proj"]
if quant_config is None:
return True, None
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
if isinstance(quant_config, DeepseekV4FP8Config):
from vllm.config import get_current_vllm_config
from vllm.model_executor.models.utils import extract_layer_index
if quant_config.expert_dtype != "fp4":
return False, "DeepSeek-V4 routed experts are not MXFP4"
hf_config = get_current_vllm_config().model_config.hf_config
# TODO: This is adapted from former `_shared_experts_are_fp4`, and
# needs to be cleaned up this . There should not be Quark-specific
# logic in DeepseekV4FP8Config.
quantization_config = getattr(hf_config, "quantization_config", None)
if quantization_config is None:
return False, "DeepSeek-V4 has no quantization configuration"
if not quant_config._is_quark_mxfp4_ocp(quantization_config):
return False, "DeepSeek-v4 FSE is only implemented/tested with Quark MXFP4"
layer_idx = extract_layer_index(shared_expert_prefix)
if layer_idx >= hf_config.num_hidden_layers:
shared_expert_prefix = (
f"mtp.{layer_idx - hf_config.num_hidden_layers}.ffn.shared_experts"
)
else:
shared_expert_prefix = f"layers.{layer_idx}.ffn.shared_experts"
if any(
entry.startswith(shared_expert_prefix)
for entry in quantization_config.get("exclude") or []
if isinstance(entry, str)
):
return (
False,
f"DeepSeek-V4 excludes shared experts at {shared_expert_prefix}",
)
shared_expert_weight_name = f"{shared_expert_prefix}.w1"
layer_quant_config = quantization_config.get("layer_quant_config") or {}
layer_config = layer_quant_config.get(shared_expert_weight_name)
if layer_config is None:
layer_config = next(
(
config
for pattern, config in layer_quant_config.items()
if isinstance(pattern, str)
and pattern.startswith("re:")
and re.fullmatch(
pattern.removeprefix("re:"), shared_expert_weight_name
)
),
None,
)
shared_weight_config = (
layer_config or quantization_config.get("global_quant_config") or {}
).get("weight") or {}
if shared_weight_config.get("dtype") == "fp4":
return True, None
return (
False,
f"DeepSeek-V4 shared experts at {shared_expert_prefix} are not MXFP4",
)
if isinstance(quant_config, QuarkConfig):
# TODO: layer_type_quant_config is not taken into account here.
assert "exclude" in quant_config.quant_config
assert "global_quant_config" in quant_config.quant_config
is_compatible = not any(
"shared_expert" in str(entry)
for entry in quant_config.quant_config["exclude"]
)
if not is_compatible:
return False, f"Quark excludes shared experts at {shared_expert_prefix}"
global_quant_config = quant_config.quant_config["global_quant_config"]
def get_projection_quant_configs(layer_name: str) -> list[object]:
module_prefix, _, projection_name = layer_name.rpartition(".")
packed_projection_names = quant_config.packed_modules_mapping.get(
projection_name, [projection_name]
)
return [
quant_config.get_layer_quant_config_from_name(
f"{module_prefix}.{packed_projection_name}"
)
or global_quant_config
for packed_projection_name in packed_projection_names
]
expert_quant_config = (
quant_config.get_layer_quant_config_from_name(expert_prefix)
or global_quant_config
)
shared_expert_quant_configs = [
config
for projection_name in projection_names
for config in get_projection_quant_configs(
f"{shared_expert_prefix}.{projection_name}"
)
]
if all(config == expert_quant_config for config in shared_expert_quant_configs):
return True, None
return (
False,
"Quark uses different quantization configurations for routed and "
f"shared experts at {shared_expert_prefix}",
)
# TODO: Extend FSE support detection to other quantization methods. Typically,
# one would check that the experts and shared_experts use the same
# quantization config. This may be refactored as part of QuantizationConfig later.
return (
False,
"shared-expert FSE quantization compatibility is not implemented for "
f"{type(quant_config).__name__}",
)