[ROCm][Quark][7/N] Use MXFP4 linear kernel abstraction for emulation backend (#48949)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-07-31 10:07:15 -07:00
committed by GitHub
parent d87d2ca747
commit e67a2e0a56
17 changed files with 824 additions and 90 deletions
+98 -2
View File
@@ -77,6 +77,9 @@ from vllm.model_executor.kernels.linear.mxfp4 import (
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
AiterMxfp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.emulation import (
EmulationMxfp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.flashinfer import (
FlashInferMxFp4LinearKernel,
)
@@ -89,6 +92,13 @@ from vllm.model_executor.kernels.linear.mxfp4.marlin import (
from vllm.model_executor.kernels.linear.mxfp4.xpu import (
XPUMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp6 import (
MxFp6LinearKernel,
MxFp6LinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mxfp6.emulation import (
EmulationMxfp6LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8 import (
Mxfp8LinearKernel,
Mxfp8LinearLayerConfig,
@@ -294,6 +304,8 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
"emulation": {
EmulationMxfp8LinearKernel,
EmulationNvFp4LinearKernel,
EmulationMxfp6LinearKernel,
EmulationMxfp4LinearKernel,
},
"xpu": {
XPUW8A8FP8LinearKernel,
@@ -467,14 +479,25 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
],
}
_POSSIBLE_MXFP6_KERNELS: dict[PlatformEnum, list[type[MxFp6LinearKernel]]] = {
PlatformEnum.CUDA: [
EmulationMxfp6LinearKernel,
],
PlatformEnum.ROCM: [
EmulationMxfp6LinearKernel,
],
}
_POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
PlatformEnum.CUDA: [
FlashInferMxFp4LinearKernel,
MarlinMxFp4LinearKernel,
HummingMxFp4LinearKernel,
EmulationMxfp4LinearKernel,
],
PlatformEnum.ROCM: [
AiterMxfp4LinearKernel,
EmulationMxfp4LinearKernel,
],
PlatformEnum.XPU: [
XPUMxFp4LinearKernel,
@@ -812,9 +835,15 @@ def init_mxfp8_linear_kernel() -> Mxfp8LinearKernel:
)
def init_mxfp4_linear_kernel() -> MxFp4LinearKernel:
def init_mxfp4_linear_kernel(
activation_quant_key: QuantKey | None = None,
) -> MxFp4LinearKernel:
"""Select and instantiate the best MXFP4 linear kernel for the
current platform."""
config = MxFp4LinearLayerConfig(
activation_quant_key=activation_quant_key,
)
linear_backend = _get_linear_backend()
platform = current_platform._enum
@@ -843,8 +872,13 @@ def init_mxfp4_linear_kernel() -> MxFp4LinearKernel:
failure_reasons.append(f"{kernel_cls.__name__}: {reason}")
continue
can_implement, reason = kernel_cls.can_implement(config)
if not can_implement:
failure_reasons.append(f"{kernel_cls.__name__}: {reason}")
continue
logger.info_once("Using %s for MXFP4 GEMM", kernel_cls.__name__)
return kernel_cls(MxFp4LinearLayerConfig())
return kernel_cls(config)
raise ValueError(
"Failed to find a kernel that can implement the "
@@ -852,6 +886,59 @@ def init_mxfp4_linear_kernel() -> MxFp4LinearKernel:
)
def init_mxfp6_linear_kernel(
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None = None,
) -> MxFp6LinearKernel:
"""Select and instantiate the best MXFP6 linear kernel for the
current platform."""
config = MxFp6LinearLayerConfig(
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
)
linear_backend = _get_linear_backend()
platform = current_platform._enum
possible = list(_POSSIBLE_MXFP6_KERNELS.get(platform, []))
# Apply --linear-backend filtering when set.
if linear_backend != "auto":
filtered = _filter_kernels_by_backend(linear_backend, possible)
if not filtered:
raise ValueError(
f"--linear-backend={linear_backend} was requested but no "
f"'{linear_backend}' kernel exists for MXFP6 layers."
)
possible = filtered
failure_reasons = []
for kernel_cls in possible:
if kernel_cls.__name__ in envs.VLLM_DISABLED_KERNELS:
failure_reasons.append(
f" {kernel_cls.__name__} disabled by environment variable"
)
continue
is_supported, reason = kernel_cls.is_supported()
if not is_supported:
failure_reasons.append(f"{kernel_cls.__name__}: {reason}")
continue
can_implement, reason = kernel_cls.can_implement(config)
if not can_implement:
failure_reasons.append(f"{kernel_cls.__name__}: {reason}")
continue
logger.info_once("Using %s for MXFP6 GEMM", kernel_cls.__name__)
return kernel_cls(config)
raise ValueError(
"Failed to find a kernel that can implement the "
"MXFP6 linear layer. Reasons: \n" + "\n".join(failure_reasons)
)
def init_wfp8_a16_linear_kernel(
weight_quant_key: QuantKey,
activation_quant_key: QuantKey,
@@ -1037,6 +1124,10 @@ def register_linear_kernel(
if platform not in _POSSIBLE_MXFP4_KERNELS:
_POSSIBLE_MXFP4_KERNELS[platform] = []
_POSSIBLE_MXFP4_KERNELS[platform].append(kernel_class)
elif kernel_type == "mxfp6":
if platform not in _POSSIBLE_MXFP6_KERNELS:
_POSSIBLE_MXFP6_KERNELS[platform] = []
_POSSIBLE_MXFP6_KERNELS[platform].append(kernel_class)
else:
raise ValueError(f"Unrecognized kernel type: {kernel_type}")
@@ -1091,7 +1182,12 @@ __all__ = [
"init_mxfp4_linear_kernel",
"MxFp4LinearKernel",
"MxFp4LinearLayerConfig",
"MxFp6LinearKernel",
"MxFp6LinearLayerConfig",
"init_mxfp6_linear_kernel",
"EmulationMxfp6LinearKernel",
"AiterMxfp4LinearKernel",
"EmulationMxfp4LinearKernel",
"FlashInferMxFp4LinearKernel",
"MarlinMxFp4LinearKernel",
"FlashInferCutedslMxfp8LinearKernel",
@@ -4,11 +4,18 @@
import torch
from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
)
from vllm.platforms import current_platform
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
logger = init_logger(__name__)
# NOTE: Do not import aiter at module scope. Importing aiter eagerly initializes HIP
# which can force the engine core to spawn instead of fork.
# is_aiter_found_and_supported() checks platform + arch + library availability via
@@ -131,12 +138,34 @@ class AiterMxfp4LinearKernel(MxFp4LinearKernel):
) -> tuple[bool, str | None]:
if not current_platform.supports_mx():
return False, "current platform does not support native MXFP4 computation"
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.model_executor.kernels.linear import _get_linear_backend
linear_backend = _get_linear_backend()
if (
current_platform.is_rocm()
and current_platform.supports_mx()
and "AiterMxfp4LinearKernel" not in envs.VLLM_DISABLED_KERNELS
and linear_backend == "auto"
and not is_aiter_found_and_supported()
):
logger.warning_once(
"This platform supports native MXFP4 W4A4 MOE "
"computation via AITER MOE backend, but AITER is not "
"found or not supported. Consider installing AITER: "
"https://github.com/ROCm/aiter."
)
if is_aiter_found_and_supported():
return True, None
return False, "AITER not found or not supported on the current platform"
@classmethod
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key != kMxfp4Dynamic:
return False, "only supports MXFP4 dynamic activation"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -6,6 +6,8 @@ from dataclasses import dataclass
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
@dataclass
class MxFp4LinearLayerConfig:
@@ -13,9 +15,13 @@ class MxFp4LinearLayerConfig:
All MXFP4 layers share the same structure: packed uint8 weights (2 FP4 values per
byte) and per-block weight scales (group size 32).
Attributes:
activation_quant_key: Identifies the activation quantization format,
or `None` when activations must not be quantized.
"""
pass
activation_quant_key: QuantKey | None = None
class MxFp4LinearKernel(ABC):
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from functools import partial
import torch
import torch.nn.functional as F
from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
dequant_mxfp4,
quant_dequant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
quant_dequant_mxfp6,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
kMxfp6E2M3Dynamic,
kMxfp6E3M2Dynamic,
)
from vllm.platforms import current_platform
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
logger = init_logger(__name__)
_ACTIVATION_QUANT_DEQUANT_FUNCS: dict[
QuantKey, Callable[[torch.Tensor], torch.Tensor]
] = {
kMxfp4Dynamic: quant_dequant_mxfp4,
kMxfp6E3M2Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e3m2"),
kMxfp6E2M3Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e2m3"),
}
class EmulationMxfp4LinearKernel(MxFp4LinearKernel):
"""Software emulation fallback for OCP MXFP4/MXFP6 (dequant + F.linear)."""
def __init__(self, config: MxFp4LinearLayerConfig) -> None:
super().__init__(config)
if config.activation_quant_key is None:
# no input Q/DQ for weight-only
self.quant_dequant_func: Callable[[torch.Tensor], torch.Tensor] = (
lambda x: x
)
else:
self.quant_dequant_func = _ACTIVATION_QUANT_DEQUANT_FUNCS[
config.activation_quant_key
]
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
return True, None
@classmethod
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key not in (
None,
kMxfp4Dynamic,
kMxfp6E3M2Dynamic,
kMxfp6E2M3Dynamic,
):
return False, "only supports MXFP4 or MXFP6 or unquantized activations"
if (
current_platform.is_rocm()
and current_platform.supports_mx()
and config.activation_quant_key != kMxfp4Dynamic
):
logger.warning_once(
"The current platform supports native MXFP4/MXFP6 computation, "
f"but kernels for activation_quant_key={config.activation_quant_key} "
f"are not yet integrated in vLLM. Using EmulationMxfp4LinearKernel, "
"with simulated weight dequantization and activation "
"QDQ (quantize and dequantize), with the linear "
"layers computed in high precision."
)
if not current_platform.supports_mx():
logger.warning_once(
"The current platform does not support native MXFP4 "
"computation. Using EmulationMxfp4LinearKernel, with simulated weight "
"dequantization and activation QDQ (quantize and dequantize), with "
"the linear layers computed in high precision."
)
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
dq_w = dequant_mxfp4(layer.weight, layer.weight_scale, x.dtype)
qdq_x = self.quant_dequant_func(x)
return F.linear(qdq_x, dq_w, bias)
@@ -7,6 +7,9 @@ from torch.nn.parameter import Parameter
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
swizzle_mxfp4_scales,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_cutedsl
@@ -28,6 +31,8 @@ class FlashInferMxFp4LinearKernel(MxFp4LinearKernel):
@classmethod
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key != kMxfp4Dynamic:
return False, "only supports MXFP4 dynamic activation"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -3,14 +3,18 @@
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from vllm.platforms import current_platform
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
logger = init_logger(__name__)
class HummingMxFp4LinearKernel(MxFp4LinearKernel):
"""Humming GEMM Kernel for MXFP4."""
@@ -28,7 +32,15 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
return True, None
@classmethod
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key not in (None, kMxfp4Dynamic):
return False, "only supports MXFP4 dynamic or unquantized activations"
if config.activation_quant_key is not None:
logger.warning_once(
"HummingMxFp4LinearKernel is a weight-only (A16) kernel; "
"the requested activation quantization (%s) is ignored.",
config.activation_quant_key,
)
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -3,8 +3,13 @@
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
logger = init_logger(__name__)
class MarlinMxFp4LinearKernel(MxFp4LinearKernel):
@classmethod
@@ -20,7 +25,15 @@ class MarlinMxFp4LinearKernel(MxFp4LinearKernel):
return False, "Marlin FP4 not available"
@classmethod
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key not in (None, kMxfp4Dynamic):
return False, "only supports MXFP4 dynamic or unquantized activations"
if config.activation_quant_key is not None:
logger.warning_once(
"MarlinMxFp4LinearKernel is a weight-only (A16) kernel; "
"the requested activation quantization (%s) is ignored.",
config.activation_quant_key,
)
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -6,6 +6,9 @@ import torch
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
xpu_mxfp4_quantize as quant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
@@ -24,7 +27,9 @@ class XPUMxFp4LinearKernel(MxFp4LinearKernel):
return True, None
@classmethod
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key != kMxfp4Dynamic:
return False, "only supports MXFP4 dynamic activation"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -0,0 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.kernels.linear.mxfp6.base import (
MxFp6LinearKernel,
MxFp6LinearLayerConfig,
)
__all__ = [
"MxFp6LinearKernel",
"MxFp6LinearLayerConfig",
]
@@ -0,0 +1,76 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
@dataclass
class MxFp6LinearLayerConfig:
"""Configuration for an MXFP6 linear layer.
All MXFP6 layers share the same structure: packed uint8 weights (2 FP4 values per
byte) and per-block weight scales (group size 32).
Attributes:
weight_quant_key: Identifies the weight quantization format. Can be
kMxfp6E2M3Static or kMxfp6E3M2Static.
activation_quant_key: Identifies the activation quantization format,
or `None` when activations must not be quantized.
"""
weight_quant_key: QuantKey
activation_quant_key: QuantKey | None = None
class MxFp6LinearKernel(ABC):
"""Base class for MXFP6 quantized linear kernels.
Each subclass implements a specific GEMM backend (CUTLASS, Marlin, etc).
The kernel selection mechanism iterates over registered subclasses in
priority order,calling ``is_supported`` and ``can_implement`` to find the best
match for the current hardware.
"""
def __init__(self, config: MxFp6LinearLayerConfig) -> None:
assert self.can_implement(config)[0]
assert self.is_supported()[0]
self.config = config
@classmethod
@abstractmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
"""Return whether this kernel can run on the current platform."""
raise NotImplementedError
@classmethod
@abstractmethod
def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]:
"""Return whether this kernel can handle *config*."""
raise NotImplementedError
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Transform weights into the format required by this kernel.
Called once after checkpoint weights have been loaded onto the
device. Implementations should repack / swizzle / pad weights
and scales in-place on *layer*.
"""
raise NotImplementedError
@abstractmethod
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run the quantized GEMM."""
raise NotImplementedError
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from functools import partial
import torch
import torch.nn.functional as F
from torch.nn.parameter import Parameter
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
dequant_mxfp6,
quant_dequant_mxfp6,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
kMxfp6E2M3Dynamic,
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
)
from .base import MxFp6LinearKernel, MxFp6LinearLayerConfig
_WEIGHT_DEQUANT_FUNCS: dict[QuantKey, Callable[..., torch.Tensor]] = {
kMxfp6E3M2Static: partial(dequant_mxfp6, quant_dtype="fp6_e3m2"),
kMxfp6E2M3Static: partial(dequant_mxfp6, quant_dtype="fp6_e2m3"),
}
_ACTIVATION_QUANT_DEQUANT_FUNCS: dict[
QuantKey, Callable[[torch.Tensor], torch.Tensor]
] = {
kMxfp4Dynamic: quant_dequant_mxfp4,
kMxfp6E3M2Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e3m2"),
kMxfp6E2M3Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e2m3"),
}
class EmulationMxfp6LinearKernel(MxFp6LinearKernel):
"""Software emulation fallback for OCP MXFP4/MXFP6 (dequant + F.linear)."""
def __init__(self, config: MxFp6LinearLayerConfig) -> None:
super().__init__(config)
self.dequant_func = _WEIGHT_DEQUANT_FUNCS[config.weight_quant_key]
if config.activation_quant_key is None:
# no input Q/DQ for weight-only
self.quant_dequant_func: Callable[[torch.Tensor], torch.Tensor] = (
lambda x: x
)
else:
self.quant_dequant_func = _ACTIVATION_QUANT_DEQUANT_FUNCS[
config.activation_quant_key
]
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
return True, None
@classmethod
def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]:
if config.weight_quant_key not in (
kMxfp6E2M3Static,
kMxfp6E3M2Static,
):
return False, "only supports MXFP6 weights"
if config.activation_quant_key not in (
None,
kMxfp4Dynamic,
kMxfp6E3M2Dynamic,
kMxfp6E2M3Dynamic,
):
return False, "only supports MXFP4 or MXFP6 or unquantized activations"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
dq_w = self.dequant_func(layer.weight, layer.weight_scale, x.dtype)
qdq_x = self.quant_dequant_func(x)
return F.linear(qdq_x, dq_w, bias)