[Quantization] Enable shared expert fusion compatibility with online shared_expert quantization (showcase: along Quark MXFP4 routed experts) (#55686)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: fxmarty-amd <felmarty@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Misha Goin
parent
e2414ec1db
commit
9f3d70e081
@@ -1293,6 +1293,7 @@ class FusedMoEConfig:
|
||||
has_bias: bool = False
|
||||
is_lora_enabled: bool = False
|
||||
has_hash_routing: bool = False
|
||||
shared_expert_prefix: str | None = None
|
||||
|
||||
# When True, the MoE skips its final cross-rank all-reduce (and the separate
|
||||
# shared-expert reduce), returning the partial per-rank sum. The caller is
|
||||
|
||||
@@ -122,6 +122,7 @@ def FusedMoEFactory(
|
||||
is_fused_checkpoint_transposed: bool = False,
|
||||
n_shared_experts: int | None = None,
|
||||
fuse_shared_experts: bool = False,
|
||||
shared_expert_prefix: str | None = None,
|
||||
router_logits_dtype: torch.dtype | None = None,
|
||||
gate: torch.nn.Module | None = None,
|
||||
shared_experts: torch.nn.Module | None = None,
|
||||
@@ -197,6 +198,7 @@ def FusedMoEFactory(
|
||||
n_shared_experts: Number of shared experts to fuse into the routed
|
||||
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
|
||||
fuse_shared_experts: Whether to enable shared-expert fusion.
|
||||
shared_expert_prefix: Checkpoint prefix for the fused shared expert.
|
||||
router_logits_dtype: Data type for router logits buffers
|
||||
gate: Pre-configured gate module
|
||||
shared_experts: Pre-configured shared experts module
|
||||
@@ -360,6 +362,7 @@ def FusedMoEFactory(
|
||||
elastic_ep_max_dp_size=vllm_config.parallel_config.elastic_ep_max_dp_size,
|
||||
has_bias=has_bias,
|
||||
is_lora_enabled=vllm_config.lora_config is not None,
|
||||
shared_expert_prefix=shared_expert_prefix,
|
||||
activation=moe_activation,
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=router.routing_method_type, # Not ideal
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
@@ -153,6 +154,7 @@ class RoutedExperts(PluggableLayer):
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
# End random parameters
|
||||
self._loaded_expert_biases: set[str] = set()
|
||||
self._fused_shared_expert_quantizer = self._get_fused_shared_expert_quantizer()
|
||||
|
||||
self.quant_method = self._get_quant_method(
|
||||
self.layer_name,
|
||||
@@ -630,6 +632,31 @@ class RoutedExperts(PluggableLayer):
|
||||
# _to_scalar's reshape(()) would reject the size-2 weight_shape.
|
||||
param_data[expert_id] = loaded_weight
|
||||
|
||||
def _get_fused_shared_expert_quantizer(
|
||||
self,
|
||||
) -> Callable[[torch.Tensor], tuple[torch.Tensor, torch.Tensor]] | None:
|
||||
"""Return the quantizer for an online-quantized fused shared expert."""
|
||||
online_config = getattr(self.quant_config, "online_quantization_config", None)
|
||||
prefix = self.moe_config.shared_expert_prefix
|
||||
if (
|
||||
online_config is None
|
||||
or prefix is None
|
||||
or self.expert_map_manager.num_fused_shared_experts == 0
|
||||
):
|
||||
return None
|
||||
# FSE compatibility checks require every shared-expert projection to
|
||||
# use the routed experts' weight key.
|
||||
resolved = online_config.resolve_quant_method_cls(
|
||||
LinearBase, f"{prefix}.down_proj"
|
||||
)
|
||||
if resolved is None:
|
||||
return None
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
ONLINE_SHARED_EXPERT_QUANTIZERS,
|
||||
)
|
||||
|
||||
return ONLINE_SHARED_EXPERT_QUANTIZERS[resolved[3].weight]
|
||||
|
||||
@overload
|
||||
def weight_loader(
|
||||
self,
|
||||
@@ -687,6 +714,35 @@ class RoutedExperts(PluggableLayer):
|
||||
return False if return_success else None
|
||||
# Hereafter, `expert_id` is local physical id
|
||||
|
||||
# A full-precision fused shared expert is quantized into the checkpoint
|
||||
# layout, then loaded like a pre-quantized expert weight and scale.
|
||||
if (
|
||||
self._fused_shared_expert_quantizer is not None
|
||||
and self.moe_config.num_logical_experts
|
||||
<= global_expert_id
|
||||
< self.moe_config.num_logical_experts
|
||||
+ self.expert_map_manager.num_fused_shared_experts
|
||||
and weight_name.endswith("_weight")
|
||||
and loaded_weight.is_floating_point()
|
||||
and loaded_weight.dtype != param.dtype
|
||||
):
|
||||
weight, weight_scale = self._fused_shared_expert_quantizer(
|
||||
loaded_weight.to(self.moe_config.device)
|
||||
)
|
||||
stem = "w2" if shard_id == "w2" else "w13"
|
||||
self.weight_loader(
|
||||
getattr(self, f"{stem}_weight_scale"),
|
||||
weight_scale,
|
||||
f"{weight_name}_scale",
|
||||
shard_id,
|
||||
global_expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
loaded = self.weight_loader(
|
||||
param, weight, weight_name, shard_id, global_expert_id, True
|
||||
)
|
||||
return loaded if return_success else None
|
||||
|
||||
# is_transposed: if the dim to shard the weight
|
||||
# should be flipped. Required by GPTQ/AWQ (K-first format).
|
||||
# should be whatever dimension intermediate_size_per_partition is
|
||||
|
||||
@@ -84,6 +84,40 @@ def resolve_layer_fused_shared_expert(
|
||||
else (True, None)
|
||||
)
|
||||
is_fused_shared_expert_enabled = fse_requested and fse_compatible
|
||||
|
||||
# In online shared_expert quantization is used, register it to
|
||||
# `online_quantization_config.quantized_layers` ahead of weight loading.
|
||||
if is_fused_shared_expert_enabled and quant_config is not None:
|
||||
online_quantization_config = quant_config.online_quantization_config
|
||||
if online_quantization_config is not None:
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
for projection_name in ("gate_up_proj", "down_proj"):
|
||||
projection_prefix = f"{prefix}.{shared_expert_name}.{projection_name}"
|
||||
quant_method_metadata = (
|
||||
online_quantization_config.resolve_quant_method_cls(
|
||||
LinearBase, projection_prefix
|
||||
)
|
||||
)
|
||||
|
||||
if quant_method_metadata is None:
|
||||
continue
|
||||
|
||||
source, quant_key_str, target_pattern, _, quant_method_cls = (
|
||||
quant_method_metadata
|
||||
)
|
||||
|
||||
if quant_method_cls in (None, UnquantizedLinearMethod):
|
||||
continue
|
||||
|
||||
online_quantization_config.quantized_layers[projection_prefix] = (
|
||||
source.value,
|
||||
quant_key_str,
|
||||
target_pattern,
|
||||
)
|
||||
if fse_requested and not is_fused_shared_expert_enabled:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
|
||||
Reference in New Issue
Block a user