[NVFP4] NVFP4 MOE emulation fallback for H100/MI300/MI350, standardize TritonExperts usage for OCP MX emulation (#35737)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: fxmarty-amd <felmarty@amd.com> Co-authored-by: Kyle Sayers <kylesayrs@gmail.com>
This commit is contained in:
co-authored by
Kyle Sayers
parent
5f76b3fb30
commit
d622e27d2b
@@ -22,6 +22,9 @@ from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
ref_nvfp4_quant_dequant,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
per_tensor_dequantize,
|
||||
)
|
||||
@@ -253,6 +256,7 @@ def moe_kernel_quantize_input(
|
||||
block_shape: list[int] | None = None,
|
||||
is_fp4_scale_swizzled: bool = True,
|
||||
ocp_mx_scheme: str | None = None,
|
||||
quantization_emulation: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
# Handle OCP MX scheme that requires QDQ (quantize-dequantize) for emulation
|
||||
if ocp_mx_scheme is not None:
|
||||
@@ -274,16 +278,41 @@ def moe_kernel_quantize_input(
|
||||
# activation quantization below.
|
||||
|
||||
if quant_dtype == current_platform.fp8_dtype():
|
||||
if quantization_emulation:
|
||||
raise NotImplementedError(
|
||||
f"moe_kernel_quantize_input does not support quant_dtype={quant_dtype}"
|
||||
" MOE quantization emulation. Please open an issue."
|
||||
)
|
||||
return _fp8_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
elif quant_dtype == torch.int8:
|
||||
if quantization_emulation:
|
||||
raise NotImplementedError(
|
||||
"moe_kernel_quantize_input does not support quant_dtype=torch.int8"
|
||||
" MOE quantization emulation. Please open an issue."
|
||||
)
|
||||
return _int8_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
elif quant_dtype == "nvfp4":
|
||||
return _nvfp4_quantize(A, A_scale, is_sf_swizzled_layout=is_fp4_scale_swizzled)
|
||||
if not quantization_emulation:
|
||||
return _nvfp4_quantize(
|
||||
A, A_scale, is_sf_swizzled_layout=is_fp4_scale_swizzled
|
||||
)
|
||||
else:
|
||||
return ref_nvfp4_quant_dequant(A, A_scale, block_size=16)
|
||||
elif quant_dtype == "mxfp4":
|
||||
if not quantization_emulation:
|
||||
raise NotImplementedError(
|
||||
"moe_kernel_quantize_input should not be used for native"
|
||||
" quant_dtype='mxfp4' MOE. Please open an issue."
|
||||
)
|
||||
return _mxfp4_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
elif quant_dtype == "mxfp8":
|
||||
# TODO: `quant_dtype == "mxfp8"` is ambiguous,
|
||||
# should be fp8_e4m3. OCP MX also defines `fp8_e5m2`.
|
||||
if quantization_emulation:
|
||||
raise NotImplementedError(
|
||||
"moe_kernel_quantize_input does not support quant_dtype='mxfp8' MOE "
|
||||
"quantization emulation. Please open an issue."
|
||||
)
|
||||
return _mxfp8_e4m3_quantize(
|
||||
A,
|
||||
A_scale,
|
||||
@@ -292,8 +321,20 @@ def moe_kernel_quantize_input(
|
||||
is_sf_swizzled_layout=is_fp4_scale_swizzled,
|
||||
)
|
||||
elif quant_dtype == "mxfp6_e3m2":
|
||||
if not quantization_emulation:
|
||||
raise NotImplementedError(
|
||||
"moe_kernel_quantize_input should not be used for native "
|
||||
" quant_dtype='mxfp6_e3m2'MOE. Please open an issue."
|
||||
)
|
||||
|
||||
return _mxfp6_e3m2_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
elif quant_dtype == "mxfp6_e2m3":
|
||||
if not quantization_emulation:
|
||||
raise NotImplementedError(
|
||||
"moe_kernel_quantize_input should not be used for native"
|
||||
" quant_dtype='mxfp6_e2m3' MOE. Please open an issue."
|
||||
)
|
||||
|
||||
return _mxfp6_e2m3_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
else:
|
||||
return A, A_scale
|
||||
|
||||
Reference in New Issue
Block a user