[Bugfix][Quantization] Fix OCP MX MoE emulation silently skipping mxfp6 activation QDQ (#52704)
Signed-off-by: xuebwang-amd <xuebwang@amd.com>
This commit is contained in:
@@ -13,6 +13,11 @@ from vllm._aiter_ops import (
|
||||
is_aiter_found_and_supported,
|
||||
rocm_aiter_ops,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.ocp_mx_emulation_moe import (
|
||||
activation_quant_dtype,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import OCP_MX_Scheme
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
@@ -1859,3 +1864,146 @@ def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons(
|
||||
|
||||
with pytest.raises(NotImplementedError, match="Unsupported reasons"):
|
||||
mxfp4_oracle.select_mxfp4_moe_backend(moe_config)
|
||||
|
||||
|
||||
# Every activation-quantizing OCP MX scheme must map to a `quant_dtype` that
|
||||
# `moe_kernel_quantize_input` actually dispatches on. Its final `else` returns
|
||||
# the activation untouched, so a name it does not know (e.g. "mxfp6" instead of
|
||||
# "mxfp6_e3m2") silently skips the fake-quantization the emulation exists for.
|
||||
@pytest.mark.skipif(not ROCM_AVAILABLE, reason="emulation backend targets ROCm")
|
||||
@pytest.mark.parametrize("ocp_mx_scheme", list(OCP_MX_Scheme))
|
||||
def test_emulation_activation_quant_dtype_is_dispatchable(ocp_mx_scheme):
|
||||
quant_dtype = activation_quant_dtype(ocp_mx_scheme)
|
||||
|
||||
if "_a_" not in ocp_mx_scheme.value:
|
||||
assert quant_dtype is None, "weight-only schemes must not quantize activations"
|
||||
return
|
||||
|
||||
a = torch.randn(64, 128, dtype=torch.bfloat16, device="cuda")
|
||||
a_scale = torch.ones(1, dtype=torch.float32, device="cuda")
|
||||
out, _ = moe_kernel_quantize_input(
|
||||
a, a_scale, quant_dtype, False, None, quantization_emulation=True
|
||||
)
|
||||
assert not torch.equal(out, a), (
|
||||
f"{ocp_mx_scheme.value} -> quant_dtype={quant_dtype!r} left the activation"
|
||||
" unquantized; moe_kernel_quantize_input does not dispatch on it"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ROCM_AVAILABLE, reason="emulation backend targets ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_emulation_a_mxfp6_moe_forward_quantizes_activations():
|
||||
"""The same property observed through a full MoE forward.
|
||||
|
||||
`w_mxfp4_a_mxfp6_e3m2` and the weight-only `w_mxfp4` differ only in whether
|
||||
activations are fake-quantized, so with identical weights, activations and
|
||||
routing their layer outputs must differ. When the emulation selects a
|
||||
`quant_dtype` `moe_kernel_quantize_input` does not dispatch on, the QDQ is
|
||||
skipped and the two outputs come out bit-identical.
|
||||
"""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
ocp_mx_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.ocp_mx_emulation_moe import (
|
||||
OCP_MXQuantizationEmulationTritonExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
make_mxfp4_moe_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
mxfp4_quantize,
|
||||
)
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
init_workspace_manager(torch.accelerator.current_device_index())
|
||||
|
||||
num_experts, topk = 8, 2
|
||||
hidden_size, intermediate_size, num_tokens = 256, 256, 64
|
||||
dtype, device = torch.bfloat16, "cuda:0"
|
||||
|
||||
torch.manual_seed(0)
|
||||
w13, w13_scale = mxfp4_quantize(
|
||||
torch.randn(
|
||||
num_experts, 2 * intermediate_size, hidden_size, dtype=dtype, device=device
|
||||
)
|
||||
/ 8
|
||||
)
|
||||
w2, w2_scale = mxfp4_quantize(
|
||||
torch.randn(
|
||||
num_experts, hidden_size, intermediate_size, dtype=dtype, device=device
|
||||
)
|
||||
/ 8
|
||||
)
|
||||
w13, w2 = w13.contiguous(), w2.contiguous()
|
||||
w13_scale, w2_scale = w13_scale.contiguous(), w2_scale.contiguous()
|
||||
|
||||
torch.manual_seed(1)
|
||||
hidden_states = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
topk_weights, topk_ids = torch.topk(
|
||||
torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device),
|
||||
k=topk,
|
||||
dim=-1,
|
||||
)
|
||||
topk_weights = torch.softmax(topk_weights, dim=-1).to(dtype)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
def run(quant_config):
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=dtype,
|
||||
device=device,
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=quant_config,
|
||||
moe_config=moe_config,
|
||||
mxfp4_backend=Mxfp4MoeBackend.EMULATION,
|
||||
experts_cls=OCP_MXQuantizationEmulationTritonExperts,
|
||||
routing_tables=None,
|
||||
)
|
||||
return kernel.apply(
|
||||
hidden_states=hidden_states,
|
||||
w1=w13,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
# A fresh quant config per run: the experts nulls the weight scales on it.
|
||||
a_mxfp6 = ocp_mx_moe_quant_config(
|
||||
quant_dtype="mxfp6_e3m2",
|
||||
weight_dtype="mxfp4",
|
||||
w1_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
)
|
||||
assert a_mxfp6.ocp_mx_scheme == OCP_MX_Scheme.w_mxfp4_a_mxfp6_e3m2
|
||||
out_a_mxfp6 = run(a_mxfp6)
|
||||
|
||||
weight_only = mxfp4_w4a16_moe_quant_config(w1_scale=w13_scale, w2_scale=w2_scale)
|
||||
assert weight_only.ocp_mx_scheme == OCP_MX_Scheme.w_mxfp4
|
||||
out_weight_only = run(weight_only)
|
||||
|
||||
max_diff = (out_a_mxfp6.float() - out_weight_only.float()).abs().max().item()
|
||||
assert max_diff > 0.0, (
|
||||
"w_mxfp4_a_mxfp6_e3m2 output is bit-identical to weight-only w_mxfp4:"
|
||||
" the emulation never fake-quantized the activations"
|
||||
)
|
||||
|
||||
@@ -35,6 +35,52 @@ from vllm.utils.import_utils import has_quark
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def activation_quant_dtype(
|
||||
ocp_mx_scheme: OCP_MX_Scheme | str,
|
||||
) -> torch.dtype | str | None:
|
||||
"""Activation dtype `moe_kernel_quantize_input` should fake-quantize to.
|
||||
|
||||
Args:
|
||||
ocp_mx_scheme: The OCP MX scheme the emulated experts run. Accepts the
|
||||
enum member or its string value.
|
||||
|
||||
Returns:
|
||||
A `quant_dtype` `moe_kernel_quantize_input` dispatches on, or None for
|
||||
weight-only schemes, which leave activations untouched.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If the scheme has no emulated activation dtype.
|
||||
"""
|
||||
if ocp_mx_scheme in {
|
||||
OCP_MX_Scheme.w_mxfp4,
|
||||
OCP_MX_Scheme.w_mxfp6_e3m2,
|
||||
OCP_MX_Scheme.w_mxfp6_e2m3,
|
||||
}:
|
||||
return None
|
||||
elif ocp_mx_scheme == OCP_MX_Scheme.w_mxfp4_a_mxfp4:
|
||||
return "mxfp4"
|
||||
elif ocp_mx_scheme in {
|
||||
OCP_MX_Scheme.w_mxfp4_a_mxfp6_e3m2,
|
||||
OCP_MX_Scheme.w_mxfp6_e3m2_a_mxfp6_e3m2,
|
||||
}:
|
||||
return "mxfp6_e3m2"
|
||||
elif ocp_mx_scheme in {
|
||||
OCP_MX_Scheme.w_mxfp4_a_mxfp6_e2m3,
|
||||
OCP_MX_Scheme.w_mxfp6_e2m3_a_mxfp6_e2m3,
|
||||
}:
|
||||
return "mxfp6_e2m3"
|
||||
elif ocp_mx_scheme in {
|
||||
OCP_MX_Scheme.w_mxfp4_a_fp8,
|
||||
OCP_MX_Scheme.w_mxfp6_e3m2_a_fp8,
|
||||
OCP_MX_Scheme.w_mxfp6_e2m3_a_fp8,
|
||||
}:
|
||||
return current_platform.fp8_dtype()
|
||||
raise NotImplementedError(
|
||||
f"No emulated activation dtype for OCP MX scheme {ocp_mx_scheme}."
|
||||
" Please open an issue."
|
||||
)
|
||||
|
||||
|
||||
class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
|
||||
"""
|
||||
Extension of TritonExperts to support emulated OCP MX MoE experts.
|
||||
@@ -72,29 +118,7 @@ class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
|
||||
|
||||
self.quantization_emulation = True
|
||||
|
||||
if self.ocp_mx_scheme in {
|
||||
OCP_MX_Scheme.w_mxfp4,
|
||||
OCP_MX_Scheme.w_mxfp6_e3m2,
|
||||
OCP_MX_Scheme.w_mxfp6_e2m3,
|
||||
}:
|
||||
# Weight-only schemes leave activations unquantized.
|
||||
self._quant_dtype = None
|
||||
elif self.ocp_mx_scheme in {
|
||||
OCP_MX_Scheme.w_mxfp4_a_mxfp4,
|
||||
}:
|
||||
self._quant_dtype = "mxfp4"
|
||||
elif self.ocp_mx_scheme in [
|
||||
OCP_MX_Scheme.w_mxfp4_a_mxfp6_e3m2,
|
||||
OCP_MX_Scheme.w_mxfp4_a_mxfp6_e2m3,
|
||||
OCP_MX_Scheme.w_mxfp6_e3m2_a_mxfp6_e3m2,
|
||||
OCP_MX_Scheme.w_mxfp6_e2m3_a_mxfp6_e2m3,
|
||||
]:
|
||||
self._quant_dtype = "mxfp6"
|
||||
elif self.ocp_mx_scheme in [
|
||||
OCP_MX_Scheme.w_mxfp4_a_fp8,
|
||||
OCP_MX_Scheme.w_mxfp6_e3m2_a_fp8,
|
||||
]:
|
||||
self._quant_dtype = current_platform.fp8_dtype()
|
||||
self._quant_dtype = activation_quant_dtype(self.ocp_mx_scheme)
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
|
||||
Reference in New Issue
Block a user