[Feature][OCP MX] Support mxfp6 and mixed mxfp6-mxfp4 (#21166)

This commit is contained in:
fxmarty-amd
2025-10-07 09:35:26 -04:00
committed by GitHub
parent 08d26a1b7e
commit 41f1cf38f2
18 changed files with 656 additions and 180 deletions
@@ -10,8 +10,6 @@ from vllm.utils import direct_register_custom_op, is_torch_equal_or_newer
logger = init_logger(__name__)
OCP_MX_BLOCK_SIZE = 32
def _swizzle_mxfp4(quant_tensor, scale, num_warps):
"""weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel"""
@@ -144,6 +142,14 @@ def _quant_dequant_mxfp4_fake(
return torch.empty_like(x)
# Protect these operations into a torch custom op to avoid errors as
# torch._dynamo.exc.Unsupported: Attempted to call function marked as skipped
# Explanation: Dynamo does not know how to trace the builtin
# `kernel_ext.PyCapsule.dq_uint8_mxfp4_to_half.` This function is either a
# Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python
# extension (perhaps created with pybind).
# TODO: Make sure there is no way to avoid having these functions
# marked as skipped by dynamo.
try:
direct_register_custom_op(
op_name="dequant_mxfp4",
@@ -0,0 +1,142 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import OCP_MX_BLOCK_SIZE
from vllm.utils import direct_register_custom_op
def _quant_dequant_mxfp6(
x: torch.Tensor,
quant_dtype: str,
scale_calculation_mode: str = "even",
) -> torch.Tensor:
try:
from quark.torch.kernel.hw_emulation.hw_emulation_interface import (
fake_quantize_fp4_fp6_per_group_with_scale,
)
from quark.torch.quantization.utils import even_round, reshape_to_blocks
except ImportError as err:
raise ImportError(
"The package `amd-quark` is required to use "
"MX-FP6 models. Please install it with `pip install "
"amd-quark`."
) from err
axis = -1
block_x = reshape_to_blocks(x, OCP_MX_BLOCK_SIZE, axis)
amax, _ = torch.max(torch.abs(block_x), dim=-1, keepdim=True)
amax = amax.squeeze(-1)
# TODO: there are other rounding strategies supported in quark and in the
# config.json that we do not check for here!
if scale_calculation_mode != "even":
raise NotImplementedError(
f"Scale calculation mode {scale_calculation_mode} is not yet "
"supported in MX-FP6 quantization"
)
scale = even_round(amax, quant_dtype)
# Apply dequantize(quantize(x)).
x = fake_quantize_fp4_fp6_per_group_with_scale(
x,
scale.to(x.device),
axis=axis,
group_size=OCP_MX_BLOCK_SIZE,
quant_dtype=quant_dtype,
)
return x
def _quant_dequant_mxfp6_fake(
x: torch.Tensor,
quant_dtype: str,
scale_calculation_mode: str = "even",
) -> torch.Tensor:
return torch.empty_like(x)
def _dequant_mxfp6(
x: torch.Tensor, scale: torch.Tensor, float_dtype: torch.dtype, quant_dtype: str
) -> torch.Tensor:
try:
from quark.torch.kernel.hw_emulation.hw_emulation_interface import (
dequantize_fp4_fp6_per_group,
)
from quark.torch.utils.pack import create_pack_method
except ImportError as e:
raise ImportError(
"The package `amd-quark` is required to use "
"MX-FP6 models. Please install it with `pip install "
"amd-quark`."
) from e
pack_method = create_pack_method(None, dtype=quant_dtype)
unpacked_x = pack_method.unpack(x, reorder=False)
scale = 2 ** (scale.view(torch.uint8).to(torch.int16) - 127).to(float_dtype)
# TODO: `dequantize_fp4_fp6_per_group` and `prepare_inputs_per_group`
# always return fp32.
return dequantize_fp4_fp6_per_group(
unpacked_x,
scale,
axis=-1,
group_size=OCP_MX_BLOCK_SIZE,
quant_dtype=quant_dtype,
).to(float_dtype)
def _dequant_mxfp6_fake(
x: torch.Tensor, scale: torch.Tensor, float_dtype: torch.dtype, quant_dtype: str
) -> torch.Tensor:
assert (x.shape[-1] * 4) % 3 == 0
return torch.empty(
(*x.shape[:-1], (x.shape[-1] * 4) // 3), dtype=float_dtype, device=x.device
)
# Protect these operations into a torch custom op to avoid errors as
# torch._dynamo.exc.Unsupported: Attempted to call function marked as skipped
# Explanation: Dynamo does not know how to trace the builtin
# `kernel_ext.PyCapsule.dq_uint8_mxfp4_to_half.` This function is either a
# Python builtin (e.g. _warnings.warn) or a third-party C/C++ Python
# extension (perhaps created with pybind).
# TODO: Make sure there is no way to avoid having these functions
# marked as skipped by dynamo.
try:
direct_register_custom_op(
op_name="quant_dequant_mxfp6",
op_func=_quant_dequant_mxfp6,
mutates_args=[],
fake_impl=_quant_dequant_mxfp6_fake,
)
except AttributeError as error:
raise error
# Expose keyword arguments.
def quant_dequant_mxfp6(
x: torch.Tensor,
quant_dtype: str,
scale_calculation_mode: str = "even",
) -> torch.Tensor:
return torch.ops.vllm.quant_dequant_mxfp6(x, quant_dtype, scale_calculation_mode)
try:
direct_register_custom_op(
op_name="dequant_mxfp6",
op_func=_dequant_mxfp6,
mutates_args=[],
fake_impl=_dequant_mxfp6_fake,
)
except AttributeError as error:
raise error
def dequant_mxfp6(
x: torch.Tensor, scale: torch.Tensor, float_dtype: torch.dtype, quant_dtype: str
) -> torch.Tensor:
return torch.ops.vllm.dequant_mxfp6(x, scale, float_dtype, quant_dtype)
@@ -8,9 +8,9 @@ from vllm.logger import init_logger
logger = init_logger(__name__)
def mxfp8_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
def mxfp8_e4m3_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
try:
from flashinfer import mxfp8_quantize
from flashinfer import mxfp8_quantize as mxfp8_e4m3_quantize
except ImportError as err:
raise ImportError(
"The package `flashinfer` is required to do "
@@ -18,4 +18,4 @@ def mxfp8_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"`pip install flashinfer`"
) from err
return mxfp8_quantize(x, is_sf_swizzled_layout=False)
return mxfp8_e4m3_quantize(x, is_sf_swizzled_layout=False)
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Union
from vllm.logger import init_logger
logger = init_logger(__name__)
OCP_MX_BLOCK_SIZE = 32
OCP_MX_DTYPES = {
"mxfp4",
"mxfp6_e3m2",
"mxfp6_e2m3",
"mxfp8_e4m3",
"mxfp8_e5m2",
"mxint8",
}
SUPPORTED_OCP_MX_DTYPES = {"mxfp4", "mxfp6_e3m2", "mxfp6_e2m3"}
class OCP_MX_Scheme(str, Enum):
w_mxfp4_a_mxfp4 = "w_mxfp4_a_mxfp4"
w_mxfp4_a_mxfp6_e3m2 = "w_mxfp4_a_mxfp6_e3m2"
w_mxfp4_a_mxfp6_e2m3 = "w_mxfp4_a_mxfp6_e2m3"
w_mxfp6_e3m2_a_mxfp6_e3m2 = "w_mxfp6_e3m2_a_mxfp6_e3m2"
w_mxfp6_e2m3_a_mxfp6_e2m3 = "w_mxfp6_e2m3_a_mxfp6_e2m3"
@classmethod
def from_quant_dtype(
cls, input_dtype: Union[str, None], weight_dtype: Union[str, None]
):
if input_dtype not in OCP_MX_DTYPES or weight_dtype not in OCP_MX_DTYPES:
return None
elif input_dtype == "mxfp4" and weight_dtype == "mxfp4":
return cls.w_mxfp4_a_mxfp4
elif input_dtype == "mxfp6_e3m2" and weight_dtype == "mxfp4":
return cls.w_mxfp4_a_mxfp6_e3m2
elif input_dtype == "mxfp6_e2m3" and weight_dtype == "mxfp4":
return cls.w_mxfp4_a_mxfp6_e2m3
elif input_dtype == "mxfp6_e3m2" and weight_dtype == "mxfp6_e3m2":
return cls.w_mxfp6_e3m2_a_mxfp6_e3m2
elif input_dtype == "mxfp6_e2m3" and weight_dtype == "mxfp6_e2m3":
return cls.w_mxfp6_e2m3_a_mxfp6_e2m3
else:
logger.warning(
"input_dtype='%s' and"
" weight_dtype='%s' is not supported "
"in OCP_MX_Scheme at the moment.",
input_dtype,
weight_dtype,
)
return None