[CompressedTensors] FP4 Qutlass Integration (#43229)

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
Signed-off-by: Brian Dellabetta <bdellabe@redhat.com>
Signed-off-by: Brian Dellabetta <brian-dellabetta@users.noreply.github.com>
Co-authored-by: Brian Dellabetta <bdellabe@redhat.com>
Co-authored-by: Brian Dellabetta <brian-dellabetta@users.noreply.github.com>
Co-authored-by: Dipika Sikka <dipikasikka1@gmail.com>
This commit is contained in:
Kyle Sayers
2026-07-28 20:34:20 -06:00
committed by GitHub
co-authored by Brian Dellabetta Brian Dellabetta Dipika Sikka
parent 7398a30d79
commit 30c2718eaa
7 changed files with 169 additions and 34 deletions
@@ -473,6 +473,11 @@ def test_compressed_tensors_w4a8_fp8(vllm_runner, args):
"Flat is better than nested.\nSparse is better than dense.",
150.0,
),
(
"nm-testing/Llama-3.2-1B-Instruct-quipv16-nvfp4",
"Flat is better than nested.\nSparse is better than dense.",
150.0,
),
],
)
def test_compressed_tensors_transforms_perplexity(
+34 -3
View File
@@ -3994,9 +3994,40 @@ def fusedQuantizeNv(
padded_rows, padded_cols, dtype=torch.float8_e4m3fn, device=a.device
)
return torch.ops._qutlass_C.fusedQuantizeNvAbsMax(
a, b, xh_e2m1, xh_e4m3, global_scale
)
safeFusedQuantizeNv(a, b, xh_e2m1, xh_e4m3, global_scale)
return xh_e2m1, xh_e4m3
@torch.library.custom_op(
"vllm::safeFusedQuantizeNv", mutates_args=("xh_e2m1", "xh_e4m3")
)
def safeFusedQuantizeNv(
a: torch.Tensor,
b: torch.Tensor,
xh_e2m1: torch.Tensor,
xh_e4m3: torch.Tensor,
global_scale: torch.Tensor,
) -> None:
"""
Wrapper for QUTLASS fusedQuantizeNv method that operates on tensors in-place
rather than returning them, to prevent torch 2.12+ errors that outputs of custom
operators may not alias any inputs to the custom operator.
"""
torch.ops._qutlass_C.fusedQuantizeNvAbsMax(a, b, xh_e2m1, xh_e4m3, global_scale)
return
if hasattr(torch.ops._qutlass_C, "fusedQuantizeNv"):
@register_fake("vllm::safeFusedQuantizeNv")
def _fake_fused_quantize_nv(
a: torch.Tensor,
b: torch.Tensor,
xh_e2m1: torch.Tensor,
xh_e4m3: torch.Tensor,
global_scale: torch.Tensor,
) -> None:
return
def hadacore_transform(x: torch.Tensor, inplace: bool = True) -> torch.Tensor:
+1
View File
@@ -50,6 +50,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
"UnquantizedLinearMethod",
"CompressedTensorsLinearMethod",
"CompressedTensorsLinearTransformMethod",
"QutlassNvFP4LinearMethod",
"AutoAWQMarlinLinearMethod",
"AutoAWQLinearMethod",
"AutoGPTQLinearMethod",
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.transform.module
from vllm.model_executor.layers.quantization.compressed_tensors.transform.utils import ( # noqa: E501
TransformTuple,
)
from vllm.platforms import current_platform
class CompressedTensorsLinearTransformMethod(LinearMethodBase):
@@ -48,11 +49,12 @@ class CompressedTensorsLinearTransformMethod(LinearMethodBase):
assert input_tfms or output_tfms
if is_qutlass_fp4_scheme(quant_scheme, input_tfms):
if is_qutlass_fp4_scheme(
quant_scheme, input_tfms
) and current_platform.has_device_capability(100):
return QutlassNvFP4LinearMethod(quant_method, input_tfms, output_tfms)
# hadacore or dense gemm is selected by Transform module
return cls(quant_method, input_tfms, output_tfms)
def __init__(
@@ -32,7 +32,7 @@ class HadamardTransform(torch.nn.Module):
transforms: dict[int, TransformTuple] # info parsed from transforms config
weight: SharedWeightParameter # container for shared tensors
scales: dict[int, float] # hadamard scale, usually sqrt(matrix.size(0))
scaled_data_ptrs: set[int] = set()
def __init__(
self,
@@ -44,7 +44,6 @@ class HadamardTransform(torch.nn.Module):
):
super().__init__()
self.transforms = transforms
self.scales = {}
if get_tensor_model_parallel_world_size() > 1:
raise NotImplementedError(
@@ -64,26 +63,26 @@ class HadamardTransform(torch.nn.Module):
)
data_key = self._get_data_key(scheme, weight_size)
# load up in model's default precision, rather than using scheme.precision
self.weight.add_partition(
part_index,
data_key,
size=(weight_size, weight_size),
dtype=scheme.precision,
)
# validate that shared tensors and schemes are correct
self._validate_input_transforms()
def process_weights_after_loading(self):
for part_id in self.weight.partitions:
data = self.weight.partitions[part_id].data
for part_id, partition in self.weight.partitions.items():
# required by torch.compile
self.weight.process_weights_after_loading()
# precompute scale as a runtime multiply, not division
# do not fold into weight in order to utilize FWHT
self.scales[part_id] = 1 / math.sqrt(data.size(0))
# Merge normalization scale directly into weight, must be done only once
data_ptr = partition.data.data_ptr()
if data_ptr not in HadamardTransform.scaled_data_ptrs:
partition.data.div_(math.sqrt(partition.data.size(0)))
HadamardTransform.scaled_data_ptrs.add(data_ptr)
# FUTURE: avoid runtime transpose by processing weights
# prior to apply
@@ -111,26 +110,19 @@ class HadamardTransform(torch.nn.Module):
weight = (
weight if self.transforms[part_id].args.inverse else weight.T
) # linear := x(W.T)
scale = self.scales[part_id]
if self.transforms[part_id].scheme.head_dim is not None:
value = value.unflatten(-1, (-1, weight.size(0)))
value = (
dispatch_unquantized_gemm()(
self, value.to(weight.dtype), weight, None
).to(value.dtype)
* scale
)
value = dispatch_unquantized_gemm()(
self, value.to(weight.dtype), weight, None
).to(value.dtype)
value = value.flatten(-2, -1)
return value
return (
dispatch_unquantized_gemm()(
self, value.to(weight.dtype), weight, None
).to(value.dtype)
* scale
)
return dispatch_unquantized_gemm()(
self, value.to(weight.dtype), weight, None
).to(value.dtype)
def _get_data_key(self, scheme: TransformScheme, weight_size: int) -> Hashable:
return (id(scheme), weight_size)
@@ -2,7 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch.nn.parameter import Parameter
from vllm._custom_ops import fusedQuantizeNv
from vllm.model_executor.kernels.linear import (
_LINEAR_BACKEND_KERNEL_MAP,
NvFp4LinearKernel,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
CompressedTensorsScheme,
CompressedTensorsW4A4Fp4,
@@ -11,18 +17,32 @@ from vllm.model_executor.layers.quantization.compressed_tensors.transform.linear
CompressedTensorsLinearTransformMethod,
TransformTuple,
)
from vllm.model_executor.layers.quantization.qutlass_utils import to_blocked
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
slice_nvfp4_output,
)
from vllm.utils.flashinfer import (
flashinfer_scaled_fp4_mm,
)
__all__ = ["is_qutlass_fp4_scheme", "QutlassNvFP4LinearMethod"]
NVFP4_MAX = 6.0
# QUTLASS supports transform block sizes (16, 32, 64, 128) for NVFP4
# https://github.com/IST-DASLab/qutlass/blob/v0.2.0/qutlass/csrc/bindings.cpp#L413-L414
def is_qutlass_fp4_scheme(
quant_scheme: CompressedTensorsScheme | None,
input_tfms: dict[int, TransformTuple],
) -> bool:
return (
isinstance(quant_scheme, (CompressedTensorsW4A4Fp4,))
and len(input_tfms) == 1
and input_tfms[0].scheme.head_dim == quant_scheme.group_size
isinstance(quant_scheme, CompressedTensorsW4A4Fp4)
and len(input_tfms) >= 1
and all(
input_tfm.scheme.head_dim in (16, 32, 64, 128)
for input_tfm in input_tfms.values()
)
)
@@ -50,15 +70,90 @@ class QutlassNvFP4LinearMethod(CompressedTensorsLinearTransformMethod):
)
assert self.input_transform is not None
assert len(self.input_transform.weight) == 1
assert self.input_transform.weight[0].size(0) == layer.scheme.group_size
assert len(self.input_transform.weight.partitions) >= 1
return ret
@staticmethod
def _get_flashinfer_gemm_backend(kernel: NvFp4LinearKernel) -> str:
"""
Given a kernel, find the string that is needed to be passed into
`flashinfer_scaled_fp4_mm`, using
vllm.model_executor.kernels.linear._LINEAR_BACKEND_KERNEL_MAP as source of truth
"""
kernel_type = type(kernel)
for key, kernels in _LINEAR_BACKEND_KERNEL_MAP.items():
if not key.startswith("flashinfer_") or kernel_type not in kernels:
continue
backend = key.removeprefix("flashinfer_")
# flashinfer GEMM backend uses "cute-dsl", not "cutedsl"
return backend.replace("cutedsl", "cute-dsl")
raise ValueError(
f"QutlassNvFP4 transform requires a FlashInfer kernel, "
f"got {kernel_type.__name__}"
)
def process_weights_after_loading(self, layer):
super().process_weights_after_loading(layer)
assert self.input_transform is not None
layer.hadamard_matrix = self.input_transform.weight.partitions[0].data
# fusedQuantizeNv stores raw absmax as block scales (sf = absmax),
# while CT weights use sf = absmax * SFScaleVal / 6.0. The GEMM
# computes alpha * sum(fp4_a * sf_a * fp4_w * sf_w), so alpha must
# compensate: alpha = weight_global_scale / 6.0
layer.fused_alpha = Parameter(
layer.weight_global_scale / NVFP4_MAX, requires_grad=False
)
layer.fused_global_scale = Parameter(
torch.tensor(
[NVFP4_MAX],
dtype=torch.float32,
device=layer.weight_global_scale.device,
),
requires_grad=False,
)
layer.flashinfer_gemm_backend = self._get_flashinfer_gemm_backend(
layer.scheme.kernel
)
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
raise NotImplementedError()
assert bias is None
output_size = layer.output_size_per_partition
output_shape = [*x.shape[:-1], output_size]
x_flat = x.contiguous().flatten(end_dim=-2)
x_fp4, x_scales = fusedQuantizeNv(
x_flat, layer.hadamard_matrix, layer.fused_global_scale
)
x_scales_blocked = to_blocked(x_scales, backend="triton").view(x_scales.shape)
out = flashinfer_scaled_fp4_mm(
x_fp4,
layer.weight,
x_scales_blocked,
layer.weight_scale,
layer.fused_alpha,
x.dtype,
backend=layer.flashinfer_gemm_backend,
)
out = slice_nvfp4_output(out, output_size)
if self.output_transform is not None:
for part_id, (start, length) in enumerate(self.partition_ranges):
out[:, start : start + length] = self.output_transform(
out[:, start : start + length].clone(), part_id=part_id
)
return out.view(*output_shape)
@@ -84,6 +84,7 @@ def triton_scale_swizzle(
)
@torch.library.custom_op("vllm::triton_mx_block_rearrange", mutates_args=())
def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor:
"""
Rearranges an E8M0 tensor scale from row-major format to
@@ -142,6 +143,14 @@ def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor:
return out
@triton_mx_block_rearrange.register_fake
def _triton_mx_block_rearrange_fake(scale_tensor: torch.Tensor) -> torch.Tensor:
rows, cols = scale_tensor.shape
padded_rows = cdiv(rows, 128) * 128
padded_cols = cdiv(cols, 4) * 4
return scale_tensor.new_empty((padded_rows, padded_cols))
def to_blocked(
input_matrix: torch.Tensor, backend: Literal["torch", "triton"] = "triton"
) -> torch.Tensor:
@@ -157,7 +166,7 @@ def to_blocked(
backend: "torch" (PyTorch path) or "triton" (Triton kernel)
Returns:
Rearranged tensor of shape (32*cdiv(H,128), 16*cdiv(W,4))
Rearranged flattened tensor of size (32*cdiv(H,128) * 16*cdiv(W,4))
"""
if backend == "triton":
return triton_mx_block_rearrange(input_matrix).flatten()