diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index 2d3e48d435..00d3cf29c8 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -13,6 +13,11 @@ from vllm._aiter_ops import ( is_aiter_found_and_supported, rocm_aiter_ops, ) +from vllm.model_executor.layers.fused_moe.experts.ocp_mx_emulation_moe import ( + activation_quant_dtype, +) +from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input +from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import OCP_MX_Scheme from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer @@ -1859,3 +1864,146 @@ def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons( with pytest.raises(NotImplementedError, match="Unsupported reasons"): mxfp4_oracle.select_mxfp4_moe_backend(moe_config) + + +# Every activation-quantizing OCP MX scheme must map to a `quant_dtype` that +# `moe_kernel_quantize_input` actually dispatches on. Its final `else` returns +# the activation untouched, so a name it does not know (e.g. "mxfp6" instead of +# "mxfp6_e3m2") silently skips the fake-quantization the emulation exists for. +@pytest.mark.skipif(not ROCM_AVAILABLE, reason="emulation backend targets ROCm") +@pytest.mark.parametrize("ocp_mx_scheme", list(OCP_MX_Scheme)) +def test_emulation_activation_quant_dtype_is_dispatchable(ocp_mx_scheme): + quant_dtype = activation_quant_dtype(ocp_mx_scheme) + + if "_a_" not in ocp_mx_scheme.value: + assert quant_dtype is None, "weight-only schemes must not quantize activations" + return + + a = torch.randn(64, 128, dtype=torch.bfloat16, device="cuda") + a_scale = torch.ones(1, dtype=torch.float32, device="cuda") + out, _ = moe_kernel_quantize_input( + a, a_scale, quant_dtype, False, None, quantization_emulation=True + ) + assert not torch.equal(out, a), ( + f"{ocp_mx_scheme.value} -> quant_dtype={quant_dtype!r} left the activation" + " unquantized; moe_kernel_quantize_input does not dispatch on it" + ) + + +@pytest.mark.skipif(not ROCM_AVAILABLE, reason="emulation backend targets ROCm") +@torch.inference_mode() +def test_emulation_a_mxfp6_moe_forward_quantizes_activations(): + """The same property observed through a full MoE forward. + + `w_mxfp4_a_mxfp6_e3m2` and the weight-only `w_mxfp4` differ only in whether + activations are fake-quantized, so with identical weights, activations and + routing their layer outputs must differ. When the emulation selects a + `quant_dtype` `moe_kernel_quantize_input` does not dispatch on, the QDQ is + skipped and the two outputs come out bit-identical. + """ + from vllm.config import VllmConfig, set_current_vllm_config + from vllm.model_executor.layers.fused_moe import FusedMoEConfig + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEParallelConfig, + RoutingMethodType, + mxfp4_w4a16_moe_quant_config, + ocp_mx_moe_quant_config, + ) + from vllm.model_executor.layers.fused_moe.experts.ocp_mx_emulation_moe import ( + OCP_MXQuantizationEmulationTritonExperts, + ) + from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( + Mxfp4MoeBackend, + make_mxfp4_moe_kernel, + ) + from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + mxfp4_quantize, + ) + from vllm.v1.worker.workspace import init_workspace_manager + + init_workspace_manager(torch.accelerator.current_device_index()) + + num_experts, topk = 8, 2 + hidden_size, intermediate_size, num_tokens = 256, 256, 64 + dtype, device = torch.bfloat16, "cuda:0" + + torch.manual_seed(0) + w13, w13_scale = mxfp4_quantize( + torch.randn( + num_experts, 2 * intermediate_size, hidden_size, dtype=dtype, device=device + ) + / 8 + ) + w2, w2_scale = mxfp4_quantize( + torch.randn( + num_experts, hidden_size, intermediate_size, dtype=dtype, device=device + ) + / 8 + ) + w13, w2 = w13.contiguous(), w2.contiguous() + w13_scale, w2_scale = w13_scale.contiguous(), w2_scale.contiguous() + + torch.manual_seed(1) + hidden_states = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device) + topk_weights, topk_ids = torch.topk( + torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device), + k=topk, + dim=-1, + ) + topk_weights = torch.softmax(topk_weights, dim=-1).to(dtype) + topk_ids = topk_ids.to(torch.int32) + + def run(quant_config): + moe_config = FusedMoEConfig( + num_experts=num_experts, + experts_per_token=topk, + hidden_dim=hidden_size, + intermediate_size=intermediate_size, + num_local_experts=num_experts, + num_logical_experts=num_experts, + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + activation=MoEActivation.SILU, + in_dtype=dtype, + device=device, + routing_method=RoutingMethodType.Renormalize, + ) + with set_current_vllm_config(VllmConfig()): + kernel = make_mxfp4_moe_kernel( + moe_quant_config=quant_config, + moe_config=moe_config, + mxfp4_backend=Mxfp4MoeBackend.EMULATION, + experts_cls=OCP_MXQuantizationEmulationTritonExperts, + routing_tables=None, + ) + return kernel.apply( + hidden_states=hidden_states, + w1=w13, + w2=w2, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=MoEActivation.SILU, + global_num_experts=num_experts, + expert_map=None, + apply_router_weight_on_input=False, + ) + + # A fresh quant config per run: the experts nulls the weight scales on it. + a_mxfp6 = ocp_mx_moe_quant_config( + quant_dtype="mxfp6_e3m2", + weight_dtype="mxfp4", + w1_scale=w13_scale, + w2_scale=w2_scale, + ) + assert a_mxfp6.ocp_mx_scheme == OCP_MX_Scheme.w_mxfp4_a_mxfp6_e3m2 + out_a_mxfp6 = run(a_mxfp6) + + weight_only = mxfp4_w4a16_moe_quant_config(w1_scale=w13_scale, w2_scale=w2_scale) + assert weight_only.ocp_mx_scheme == OCP_MX_Scheme.w_mxfp4 + out_weight_only = run(weight_only) + + max_diff = (out_a_mxfp6.float() - out_weight_only.float()).abs().max().item() + assert max_diff > 0.0, ( + "w_mxfp4_a_mxfp6_e3m2 output is bit-identical to weight-only w_mxfp4:" + " the emulation never fake-quantized the activations" + ) diff --git a/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py b/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py index 9107a52ed8..2c91e22d91 100644 --- a/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py @@ -35,6 +35,52 @@ from vllm.utils.import_utils import has_quark logger = init_logger(__name__) +def activation_quant_dtype( + ocp_mx_scheme: OCP_MX_Scheme | str, +) -> torch.dtype | str | None: + """Activation dtype `moe_kernel_quantize_input` should fake-quantize to. + + Args: + ocp_mx_scheme: The OCP MX scheme the emulated experts run. Accepts the + enum member or its string value. + + Returns: + A `quant_dtype` `moe_kernel_quantize_input` dispatches on, or None for + weight-only schemes, which leave activations untouched. + + Raises: + NotImplementedError: If the scheme has no emulated activation dtype. + """ + if ocp_mx_scheme in { + OCP_MX_Scheme.w_mxfp4, + OCP_MX_Scheme.w_mxfp6_e3m2, + OCP_MX_Scheme.w_mxfp6_e2m3, + }: + return None + elif ocp_mx_scheme == OCP_MX_Scheme.w_mxfp4_a_mxfp4: + return "mxfp4" + elif ocp_mx_scheme in { + OCP_MX_Scheme.w_mxfp4_a_mxfp6_e3m2, + OCP_MX_Scheme.w_mxfp6_e3m2_a_mxfp6_e3m2, + }: + return "mxfp6_e3m2" + elif ocp_mx_scheme in { + OCP_MX_Scheme.w_mxfp4_a_mxfp6_e2m3, + OCP_MX_Scheme.w_mxfp6_e2m3_a_mxfp6_e2m3, + }: + return "mxfp6_e2m3" + elif ocp_mx_scheme in { + OCP_MX_Scheme.w_mxfp4_a_fp8, + OCP_MX_Scheme.w_mxfp6_e3m2_a_fp8, + OCP_MX_Scheme.w_mxfp6_e2m3_a_fp8, + }: + return current_platform.fp8_dtype() + raise NotImplementedError( + f"No emulated activation dtype for OCP MX scheme {ocp_mx_scheme}." + " Please open an issue." + ) + + class OCP_MXQuantizationEmulationTritonExperts(TritonExperts): """ Extension of TritonExperts to support emulated OCP MX MoE experts. @@ -72,29 +118,7 @@ class OCP_MXQuantizationEmulationTritonExperts(TritonExperts): self.quantization_emulation = True - if self.ocp_mx_scheme in { - OCP_MX_Scheme.w_mxfp4, - OCP_MX_Scheme.w_mxfp6_e3m2, - OCP_MX_Scheme.w_mxfp6_e2m3, - }: - # Weight-only schemes leave activations unquantized. - self._quant_dtype = None - elif self.ocp_mx_scheme in { - OCP_MX_Scheme.w_mxfp4_a_mxfp4, - }: - self._quant_dtype = "mxfp4" - elif self.ocp_mx_scheme in [ - OCP_MX_Scheme.w_mxfp4_a_mxfp6_e3m2, - OCP_MX_Scheme.w_mxfp4_a_mxfp6_e2m3, - OCP_MX_Scheme.w_mxfp6_e3m2_a_mxfp6_e3m2, - OCP_MX_Scheme.w_mxfp6_e2m3_a_mxfp6_e2m3, - ]: - self._quant_dtype = "mxfp6" - elif self.ocp_mx_scheme in [ - OCP_MX_Scheme.w_mxfp4_a_fp8, - OCP_MX_Scheme.w_mxfp6_e3m2_a_fp8, - ]: - self._quant_dtype = current_platform.fp8_dtype() + self._quant_dtype = activation_quant_dtype(self.ocp_mx_scheme) @staticmethod def is_supported_config(