[NVFP4] NVFP4 MOE emulation fallback for H100/MI300/MI350, standardize TritonExperts usage for OCP MX emulation (#35737)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: fxmarty-amd <felmarty@amd.com>
Co-authored-by: Kyle Sayers <kylesayrs@gmail.com>
This commit is contained in:
fxmarty-amd
2026-04-22 08:58:54 -07:00
committed by GitHub
co-authored by Kyle Sayers
parent 5f76b3fb30
commit d622e27d2b
12 changed files with 601 additions and 121 deletions
+42 -1
View File
@@ -22,6 +22,9 @@ from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
ref_nvfp4_quant_dequant,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
per_tensor_dequantize,
)
@@ -253,6 +256,7 @@ def moe_kernel_quantize_input(
block_shape: list[int] | None = None,
is_fp4_scale_swizzled: bool = True,
ocp_mx_scheme: str | None = None,
quantization_emulation: bool = False,
) -> tuple[torch.Tensor, torch.Tensor | None]:
# Handle OCP MX scheme that requires QDQ (quantize-dequantize) for emulation
if ocp_mx_scheme is not None:
@@ -274,16 +278,41 @@ def moe_kernel_quantize_input(
# activation quantization below.
if quant_dtype == current_platform.fp8_dtype():
if quantization_emulation:
raise NotImplementedError(
f"moe_kernel_quantize_input does not support quant_dtype={quant_dtype}"
" MOE quantization emulation. Please open an issue."
)
return _fp8_quantize(A, A_scale, per_act_token_quant, block_shape)
elif quant_dtype == torch.int8:
if quantization_emulation:
raise NotImplementedError(
"moe_kernel_quantize_input does not support quant_dtype=torch.int8"
" MOE quantization emulation. Please open an issue."
)
return _int8_quantize(A, A_scale, per_act_token_quant, block_shape)
elif quant_dtype == "nvfp4":
return _nvfp4_quantize(A, A_scale, is_sf_swizzled_layout=is_fp4_scale_swizzled)
if not quantization_emulation:
return _nvfp4_quantize(
A, A_scale, is_sf_swizzled_layout=is_fp4_scale_swizzled
)
else:
return ref_nvfp4_quant_dequant(A, A_scale, block_size=16)
elif quant_dtype == "mxfp4":
if not quantization_emulation:
raise NotImplementedError(
"moe_kernel_quantize_input should not be used for native"
" quant_dtype='mxfp4' MOE. Please open an issue."
)
return _mxfp4_quantize(A, A_scale, per_act_token_quant, block_shape)
elif quant_dtype == "mxfp8":
# TODO: `quant_dtype == "mxfp8"` is ambiguous,
# should be fp8_e4m3. OCP MX also defines `fp8_e5m2`.
if quantization_emulation:
raise NotImplementedError(
"moe_kernel_quantize_input does not support quant_dtype='mxfp8' MOE "
"quantization emulation. Please open an issue."
)
return _mxfp8_e4m3_quantize(
A,
A_scale,
@@ -292,8 +321,20 @@ def moe_kernel_quantize_input(
is_sf_swizzled_layout=is_fp4_scale_swizzled,
)
elif quant_dtype == "mxfp6_e3m2":
if not quantization_emulation:
raise NotImplementedError(
"moe_kernel_quantize_input should not be used for native "
" quant_dtype='mxfp6_e3m2'MOE. Please open an issue."
)
return _mxfp6_e3m2_quantize(A, A_scale, per_act_token_quant, block_shape)
elif quant_dtype == "mxfp6_e2m3":
if not quantization_emulation:
raise NotImplementedError(
"moe_kernel_quantize_input should not be used for native"
" quant_dtype='mxfp6_e2m3' MOE. Please open an issue."
)
return _mxfp6_e2m3_quantize(A, A_scale, per_act_token_quant, block_shape)
else:
return A, A_scale