[Bugfix][Quantization] Fix OCP MX MoE emulation silently skipping mxfp6 activation QDQ (#52704)

Signed-off-by: xuebwang-amd <xuebwang@amd.com>
This commit is contained in:
xuebwang-amd
2026-08-19 11:29:29 -05:00
committed by GitHub
parent 583a00257d
commit c676232313
2 changed files with 195 additions and 23 deletions
+148
View File
@@ -13,6 +13,11 @@ from vllm._aiter_ops import (
is_aiter_found_and_supported,
rocm_aiter_ops,
)
from vllm.model_executor.layers.fused_moe.experts.ocp_mx_emulation_moe import (
activation_quant_dtype,
)
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import OCP_MX_Scheme
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
@@ -1859,3 +1864,146 @@ def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons(
with pytest.raises(NotImplementedError, match="Unsupported reasons"):
mxfp4_oracle.select_mxfp4_moe_backend(moe_config)
# Every activation-quantizing OCP MX scheme must map to a `quant_dtype` that
# `moe_kernel_quantize_input` actually dispatches on. Its final `else` returns
# the activation untouched, so a name it does not know (e.g. "mxfp6" instead of
# "mxfp6_e3m2") silently skips the fake-quantization the emulation exists for.
@pytest.mark.skipif(not ROCM_AVAILABLE, reason="emulation backend targets ROCm")
@pytest.mark.parametrize("ocp_mx_scheme", list(OCP_MX_Scheme))
def test_emulation_activation_quant_dtype_is_dispatchable(ocp_mx_scheme):
quant_dtype = activation_quant_dtype(ocp_mx_scheme)
if "_a_" not in ocp_mx_scheme.value:
assert quant_dtype is None, "weight-only schemes must not quantize activations"
return
a = torch.randn(64, 128, dtype=torch.bfloat16, device="cuda")
a_scale = torch.ones(1, dtype=torch.float32, device="cuda")
out, _ = moe_kernel_quantize_input(
a, a_scale, quant_dtype, False, None, quantization_emulation=True
)
assert not torch.equal(out, a), (
f"{ocp_mx_scheme.value} -> quant_dtype={quant_dtype!r} left the activation"
" unquantized; moe_kernel_quantize_input does not dispatch on it"
)
@pytest.mark.skipif(not ROCM_AVAILABLE, reason="emulation backend targets ROCm")
@torch.inference_mode()
def test_emulation_a_mxfp6_moe_forward_quantizes_activations():
"""The same property observed through a full MoE forward.
`w_mxfp4_a_mxfp6_e3m2` and the weight-only `w_mxfp4` differ only in whether
activations are fake-quantized, so with identical weights, activations and
routing their layer outputs must differ. When the emulation selects a
`quant_dtype` `moe_kernel_quantize_input` does not dispatch on, the QDQ is
skipped and the two outputs come out bit-identical.
"""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
RoutingMethodType,
mxfp4_w4a16_moe_quant_config,
ocp_mx_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.ocp_mx_emulation_moe import (
OCP_MXQuantizationEmulationTritonExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
make_mxfp4_moe_kernel,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
mxfp4_quantize,
)
from vllm.v1.worker.workspace import init_workspace_manager
init_workspace_manager(torch.accelerator.current_device_index())
num_experts, topk = 8, 2
hidden_size, intermediate_size, num_tokens = 256, 256, 64
dtype, device = torch.bfloat16, "cuda:0"
torch.manual_seed(0)
w13, w13_scale = mxfp4_quantize(
torch.randn(
num_experts, 2 * intermediate_size, hidden_size, dtype=dtype, device=device
)
/ 8
)
w2, w2_scale = mxfp4_quantize(
torch.randn(
num_experts, hidden_size, intermediate_size, dtype=dtype, device=device
)
/ 8
)
w13, w2 = w13.contiguous(), w2.contiguous()
w13_scale, w2_scale = w13_scale.contiguous(), w2_scale.contiguous()
torch.manual_seed(1)
hidden_states = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
topk_weights, topk_ids = torch.topk(
torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device),
k=topk,
dim=-1,
)
topk_weights = torch.softmax(topk_weights, dim=-1).to(dtype)
topk_ids = topk_ids.to(torch.int32)
def run(quant_config):
moe_config = FusedMoEConfig(
num_experts=num_experts,
experts_per_token=topk,
hidden_dim=hidden_size,
intermediate_size=intermediate_size,
num_local_experts=num_experts,
num_logical_experts=num_experts,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SILU,
in_dtype=dtype,
device=device,
routing_method=RoutingMethodType.Renormalize,
)
with set_current_vllm_config(VllmConfig()):
kernel = make_mxfp4_moe_kernel(
moe_quant_config=quant_config,
moe_config=moe_config,
mxfp4_backend=Mxfp4MoeBackend.EMULATION,
experts_cls=OCP_MXQuantizationEmulationTritonExperts,
routing_tables=None,
)
return kernel.apply(
hidden_states=hidden_states,
w1=w13,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
)
# A fresh quant config per run: the experts nulls the weight scales on it.
a_mxfp6 = ocp_mx_moe_quant_config(
quant_dtype="mxfp6_e3m2",
weight_dtype="mxfp4",
w1_scale=w13_scale,
w2_scale=w2_scale,
)
assert a_mxfp6.ocp_mx_scheme == OCP_MX_Scheme.w_mxfp4_a_mxfp6_e3m2
out_a_mxfp6 = run(a_mxfp6)
weight_only = mxfp4_w4a16_moe_quant_config(w1_scale=w13_scale, w2_scale=w2_scale)
assert weight_only.ocp_mx_scheme == OCP_MX_Scheme.w_mxfp4
out_weight_only = run(weight_only)
max_diff = (out_a_mxfp6.float() - out_weight_only.float()).abs().max().item()
assert max_diff > 0.0, (
"w_mxfp4_a_mxfp6_e3m2 output is bit-identical to weight-only w_mxfp4:"
" the emulation never fake-quantized the activations"
)
@@ -35,6 +35,52 @@ from vllm.utils.import_utils import has_quark
logger = init_logger(__name__)
def activation_quant_dtype(
ocp_mx_scheme: OCP_MX_Scheme | str,
) -> torch.dtype | str | None:
"""Activation dtype `moe_kernel_quantize_input` should fake-quantize to.
Args:
ocp_mx_scheme: The OCP MX scheme the emulated experts run. Accepts the
enum member or its string value.
Returns:
A `quant_dtype` `moe_kernel_quantize_input` dispatches on, or None for
weight-only schemes, which leave activations untouched.
Raises:
NotImplementedError: If the scheme has no emulated activation dtype.
"""
if ocp_mx_scheme in {
OCP_MX_Scheme.w_mxfp4,
OCP_MX_Scheme.w_mxfp6_e3m2,
OCP_MX_Scheme.w_mxfp6_e2m3,
}:
return None
elif ocp_mx_scheme == OCP_MX_Scheme.w_mxfp4_a_mxfp4:
return "mxfp4"
elif ocp_mx_scheme in {
OCP_MX_Scheme.w_mxfp4_a_mxfp6_e3m2,
OCP_MX_Scheme.w_mxfp6_e3m2_a_mxfp6_e3m2,
}:
return "mxfp6_e3m2"
elif ocp_mx_scheme in {
OCP_MX_Scheme.w_mxfp4_a_mxfp6_e2m3,
OCP_MX_Scheme.w_mxfp6_e2m3_a_mxfp6_e2m3,
}:
return "mxfp6_e2m3"
elif ocp_mx_scheme in {
OCP_MX_Scheme.w_mxfp4_a_fp8,
OCP_MX_Scheme.w_mxfp6_e3m2_a_fp8,
OCP_MX_Scheme.w_mxfp6_e2m3_a_fp8,
}:
return current_platform.fp8_dtype()
raise NotImplementedError(
f"No emulated activation dtype for OCP MX scheme {ocp_mx_scheme}."
" Please open an issue."
)
class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
"""
Extension of TritonExperts to support emulated OCP MX MoE experts.
@@ -72,29 +118,7 @@ class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
self.quantization_emulation = True
if self.ocp_mx_scheme in {
OCP_MX_Scheme.w_mxfp4,
OCP_MX_Scheme.w_mxfp6_e3m2,
OCP_MX_Scheme.w_mxfp6_e2m3,
}:
# Weight-only schemes leave activations unquantized.
self._quant_dtype = None
elif self.ocp_mx_scheme in {
OCP_MX_Scheme.w_mxfp4_a_mxfp4,
}:
self._quant_dtype = "mxfp4"
elif self.ocp_mx_scheme in [
OCP_MX_Scheme.w_mxfp4_a_mxfp6_e3m2,
OCP_MX_Scheme.w_mxfp4_a_mxfp6_e2m3,
OCP_MX_Scheme.w_mxfp6_e3m2_a_mxfp6_e3m2,
OCP_MX_Scheme.w_mxfp6_e2m3_a_mxfp6_e2m3,
]:
self._quant_dtype = "mxfp6"
elif self.ocp_mx_scheme in [
OCP_MX_Scheme.w_mxfp4_a_fp8,
OCP_MX_Scheme.w_mxfp6_e3m2_a_fp8,
]:
self._quant_dtype = current_platform.fp8_dtype()
self._quant_dtype = activation_quant_dtype(self.ocp_mx_scheme)
@staticmethod
def is_supported_config(