[MXFP4] Support for linear layers + compressed-tensors integration (#41664)

This commit is contained in:
Dipika Sikka
2026-05-12 07:49:33 -04:00
committed by GitHub
parent 4df1be9547
commit a7b801e26d
11 changed files with 358 additions and 26 deletions
@@ -42,10 +42,10 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
CompressedTensors24,
CompressedTensorsScheme,
CompressedTensorsW4A4Fp4,
CompressedTensorsW4A4Mxfp4,
CompressedTensorsW4A8Fp8,
CompressedTensorsW4A8Int,
CompressedTensorsW4A16Fp4,
CompressedTensorsW4A16Mxfp4,
CompressedTensorsW8A8Fp8,
CompressedTensorsW8A8Int8,
CompressedTensorsW8A8Mxfp8,
@@ -625,7 +625,7 @@ class CompressedTensorsConfig(QuantizationConfig):
return CompressedTensorsW4A16Fp4()
if self._is_mxfp4(weight_quant):
return CompressedTensorsW4A16Mxfp4()
return CompressedTensorsW4A4Mxfp4()
if self._is_mxfp8(weight_quant):
return CompressedTensorsW8A8Mxfp8()
@@ -41,6 +41,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
super().__init__(moe)
self.group_size = 32
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
# use cutlass if supported, otherwise fallback to marlin for weight-only FP4
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
self.experts_cls: type[mk.FusedMoEExperts]
if self.use_cutlass_mxfp4:
@@ -2,10 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .compressed_tensors_scheme import CompressedTensorsScheme
from .compressed_tensors_w4a4_mxfp4 import CompressedTensorsW4A4Mxfp4
from .compressed_tensors_w4a4_nvfp4 import CompressedTensorsW4A4Fp4
from .compressed_tensors_w4a8_fp8 import CompressedTensorsW4A8Fp8
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
from .compressed_tensors_w4a16_mxfp4 import CompressedTensorsW4A16Mxfp4
from .compressed_tensors_w4a16_nvfp4 import CompressedTensorsW4A16Fp4
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
@@ -25,7 +25,7 @@ __all__ = [
"WNA16_SUPPORTED_BITS",
"CompressedTensors24",
"CompressedTensorsW4A16Fp4",
"CompressedTensorsW4A16Mxfp4",
"CompressedTensorsW4A4Mxfp4",
"CompressedTensorsW4A4Fp4",
"CompressedTensorsW4A8Int",
"CompressedTensorsW4A8Fp8",
@@ -5,24 +5,21 @@ from collections.abc import Callable
import torch
from torch.nn.parameter import Parameter
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsScheme,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
apply_fp4_marlin_linear,
prepare_fp4_layer_for_marlin,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
)
__all__ = ["CompressedTensorsW4A16Mxfp4"]
__all__ = ["CompressedTensorsW4A4Mxfp4"]
class CompressedTensorsW4A16Mxfp4(CompressedTensorsScheme):
class CompressedTensorsW4A4Mxfp4(CompressedTensorsScheme):
"""
Compressed tensors scheme for MXFP4 weight-only quantization.
Compressed tensors scheme for MXFP4.
Supports models quantized with the compressed-tensors mxfp4-pack-quantized
format.
@@ -31,10 +28,14 @@ class CompressedTensorsW4A16Mxfp4(CompressedTensorsScheme):
- 4-bit float weights (E2M1) packed into uint8
- Per-group E8M0 scales with group_size=32
- No global scale (unlike NVFP4)
On SM100+ with FlashInfer: true W4A4 (activations dynamically quantized).
Otherwise: W4A16 weight-only via Marlin.
"""
def __init__(self):
self.group_size = 32
self.kernel = init_mxfp4_linear_kernel()
@classmethod
def get_min_capability(cls) -> int:
@@ -82,11 +83,9 @@ class CompressedTensorsW4A16Mxfp4(CompressedTensorsScheme):
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Rename weight_packed to weight that marlin expects
layer.weight = Parameter(layer.weight_packed.data, requires_grad=False)
del layer.weight_packed
prepare_fp4_layer_for_marlin(layer)
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
@@ -94,13 +93,4 @@ class CompressedTensorsW4A16Mxfp4(CompressedTensorsScheme):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return apply_fp4_marlin_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale,
weight_global_scale=None,
workspace=layer.workspace,
size_n=layer.output_size_per_partition,
size_k=layer.input_size_per_partition,
bias=bias,
)
return self.kernel.apply_weights(layer, x, bias)