[ROCm][Quark][7/N] Use MXFP4 linear kernel abstraction for emulation backend (#48949)
Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
@@ -4,11 +4,18 @@
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# NOTE: Do not import aiter at module scope. Importing aiter eagerly initializes HIP
|
||||
# which can force the engine core to spawn instead of fork.
|
||||
# is_aiter_found_and_supported() checks platform + arch + library availability via
|
||||
@@ -131,12 +138,34 @@ class AiterMxfp4LinearKernel(MxFp4LinearKernel):
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.supports_mx():
|
||||
return False, "current platform does not support native MXFP4 computation"
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.model_executor.kernels.linear import _get_linear_backend
|
||||
|
||||
linear_backend = _get_linear_backend()
|
||||
|
||||
if (
|
||||
current_platform.is_rocm()
|
||||
and current_platform.supports_mx()
|
||||
and "AiterMxfp4LinearKernel" not in envs.VLLM_DISABLED_KERNELS
|
||||
and linear_backend == "auto"
|
||||
and not is_aiter_found_and_supported()
|
||||
):
|
||||
logger.warning_once(
|
||||
"This platform supports native MXFP4 W4A4 MOE "
|
||||
"computation via AITER MOE backend, but AITER is not "
|
||||
"found or not supported. Consider installing AITER: "
|
||||
"https://github.com/ROCm/aiter."
|
||||
)
|
||||
|
||||
if is_aiter_found_and_supported():
|
||||
return True, None
|
||||
return False, "AITER not found or not supported on the current platform"
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.activation_quant_key != kMxfp4Dynamic:
|
||||
return False, "only supports MXFP4 dynamic activation"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
|
||||
@@ -6,6 +6,8 @@ from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
|
||||
@dataclass
|
||||
class MxFp4LinearLayerConfig:
|
||||
@@ -13,9 +15,13 @@ class MxFp4LinearLayerConfig:
|
||||
|
||||
All MXFP4 layers share the same structure: packed uint8 weights (2 FP4 values per
|
||||
byte) and per-block weight scales (group size 32).
|
||||
|
||||
Attributes:
|
||||
activation_quant_key: Identifies the activation quantization format,
|
||||
or `None` when activations must not be quantized.
|
||||
"""
|
||||
|
||||
pass
|
||||
activation_quant_key: QuantKey | None = None
|
||||
|
||||
|
||||
class MxFp4LinearKernel(ABC):
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
dequant_mxfp4,
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
|
||||
quant_dequant_mxfp6,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E3M2Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_ACTIVATION_QUANT_DEQUANT_FUNCS: dict[
|
||||
QuantKey, Callable[[torch.Tensor], torch.Tensor]
|
||||
] = {
|
||||
kMxfp4Dynamic: quant_dequant_mxfp4,
|
||||
kMxfp6E3M2Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e3m2"),
|
||||
kMxfp6E2M3Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e2m3"),
|
||||
}
|
||||
|
||||
|
||||
class EmulationMxfp4LinearKernel(MxFp4LinearKernel):
|
||||
"""Software emulation fallback for OCP MXFP4/MXFP6 (dequant + F.linear)."""
|
||||
|
||||
def __init__(self, config: MxFp4LinearLayerConfig) -> None:
|
||||
super().__init__(config)
|
||||
if config.activation_quant_key is None:
|
||||
# no input Q/DQ for weight-only
|
||||
self.quant_dequant_func: Callable[[torch.Tensor], torch.Tensor] = (
|
||||
lambda x: x
|
||||
)
|
||||
else:
|
||||
self.quant_dequant_func = _ACTIVATION_QUANT_DEQUANT_FUNCS[
|
||||
config.activation_quant_key
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.activation_quant_key not in (
|
||||
None,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E2M3Dynamic,
|
||||
):
|
||||
return False, "only supports MXFP4 or MXFP6 or unquantized activations"
|
||||
|
||||
if (
|
||||
current_platform.is_rocm()
|
||||
and current_platform.supports_mx()
|
||||
and config.activation_quant_key != kMxfp4Dynamic
|
||||
):
|
||||
logger.warning_once(
|
||||
"The current platform supports native MXFP4/MXFP6 computation, "
|
||||
f"but kernels for activation_quant_key={config.activation_quant_key} "
|
||||
f"are not yet integrated in vLLM. Using EmulationMxfp4LinearKernel, "
|
||||
"with simulated weight dequantization and activation "
|
||||
"QDQ (quantize and dequantize), with the linear "
|
||||
"layers computed in high precision."
|
||||
)
|
||||
|
||||
if not current_platform.supports_mx():
|
||||
logger.warning_once(
|
||||
"The current platform does not support native MXFP4 "
|
||||
"computation. Using EmulationMxfp4LinearKernel, with simulated weight "
|
||||
"dequantization and activation QDQ (quantize and dequantize), with "
|
||||
"the linear layers computed in high precision."
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
dq_w = dequant_mxfp4(layer.weight, layer.weight_scale, x.dtype)
|
||||
qdq_x = self.quant_dequant_func(x)
|
||||
return F.linear(qdq_x, dq_w, bias)
|
||||
@@ -7,6 +7,9 @@ from torch.nn.parameter import Parameter
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
swizzle_mxfp4_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutedsl
|
||||
|
||||
@@ -28,6 +31,8 @@ class FlashInferMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.activation_quant_key != kMxfp4Dynamic:
|
||||
return False, "only supports MXFP4 dynamic activation"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
|
||||
@@ -3,14 +3,18 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
"""Humming GEMM Kernel for MXFP4."""
|
||||
@@ -28,7 +32,15 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.activation_quant_key not in (None, kMxfp4Dynamic):
|
||||
return False, "only supports MXFP4 dynamic or unquantized activations"
|
||||
if config.activation_quant_key is not None:
|
||||
logger.warning_once(
|
||||
"HummingMxFp4LinearKernel is a weight-only (A16) kernel; "
|
||||
"the requested activation quantization (%s) is ignored.",
|
||||
config.activation_quant_key,
|
||||
)
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
|
||||
@@ -3,8 +3,13 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MarlinMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
@classmethod
|
||||
@@ -20,7 +25,15 @@ class MarlinMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
return False, "Marlin FP4 not available"
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.activation_quant_key not in (None, kMxfp4Dynamic):
|
||||
return False, "only supports MXFP4 dynamic or unquantized activations"
|
||||
if config.activation_quant_key is not None:
|
||||
logger.warning_once(
|
||||
"MarlinMxFp4LinearKernel is a weight-only (A16) kernel; "
|
||||
"the requested activation quantization (%s) is ignored.",
|
||||
config.activation_quant_key,
|
||||
)
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
|
||||
@@ -6,6 +6,9 @@ import torch
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
xpu_mxfp4_quantize as quant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -24,7 +27,9 @@ class XPUMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.activation_quant_key != kMxfp4Dynamic:
|
||||
return False, "only supports MXFP4 dynamic activation"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
|
||||
Reference in New Issue
Block a user