[CompressedTensors] FP4 Qutlass Integration (#43229)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com> Signed-off-by: Brian Dellabetta <bdellabe@redhat.com> Signed-off-by: Brian Dellabetta <brian-dellabetta@users.noreply.github.com> Co-authored-by: Brian Dellabetta <bdellabe@redhat.com> Co-authored-by: Brian Dellabetta <brian-dellabetta@users.noreply.github.com> Co-authored-by: Dipika Sikka <dipikasikka1@gmail.com>
This commit is contained in:
co-authored by
Brian Dellabetta
Brian Dellabetta
Dipika Sikka
parent
7398a30d79
commit
30c2718eaa
@@ -473,6 +473,11 @@ def test_compressed_tensors_w4a8_fp8(vllm_runner, args):
|
||||
"Flat is better than nested.\nSparse is better than dense.",
|
||||
150.0,
|
||||
),
|
||||
(
|
||||
"nm-testing/Llama-3.2-1B-Instruct-quipv16-nvfp4",
|
||||
"Flat is better than nested.\nSparse is better than dense.",
|
||||
150.0,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_compressed_tensors_transforms_perplexity(
|
||||
|
||||
+34
-3
@@ -3994,9 +3994,40 @@ def fusedQuantizeNv(
|
||||
padded_rows, padded_cols, dtype=torch.float8_e4m3fn, device=a.device
|
||||
)
|
||||
|
||||
return torch.ops._qutlass_C.fusedQuantizeNvAbsMax(
|
||||
a, b, xh_e2m1, xh_e4m3, global_scale
|
||||
)
|
||||
safeFusedQuantizeNv(a, b, xh_e2m1, xh_e4m3, global_scale)
|
||||
return xh_e2m1, xh_e4m3
|
||||
|
||||
|
||||
@torch.library.custom_op(
|
||||
"vllm::safeFusedQuantizeNv", mutates_args=("xh_e2m1", "xh_e4m3")
|
||||
)
|
||||
def safeFusedQuantizeNv(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
xh_e2m1: torch.Tensor,
|
||||
xh_e4m3: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
Wrapper for QUTLASS fusedQuantizeNv method that operates on tensors in-place
|
||||
rather than returning them, to prevent torch 2.12+ errors that outputs of custom
|
||||
operators may not alias any inputs to the custom operator.
|
||||
"""
|
||||
torch.ops._qutlass_C.fusedQuantizeNvAbsMax(a, b, xh_e2m1, xh_e4m3, global_scale)
|
||||
return
|
||||
|
||||
|
||||
if hasattr(torch.ops._qutlass_C, "fusedQuantizeNv"):
|
||||
|
||||
@register_fake("vllm::safeFusedQuantizeNv")
|
||||
def _fake_fused_quantize_nv(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
xh_e2m1: torch.Tensor,
|
||||
xh_e4m3: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
def hadacore_transform(x: torch.Tensor, inplace: bool = True) -> torch.Tensor:
|
||||
|
||||
@@ -50,6 +50,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
|
||||
"UnquantizedLinearMethod",
|
||||
"CompressedTensorsLinearMethod",
|
||||
"CompressedTensorsLinearTransformMethod",
|
||||
"QutlassNvFP4LinearMethod",
|
||||
"AutoAWQMarlinLinearMethod",
|
||||
"AutoAWQLinearMethod",
|
||||
"AutoGPTQLinearMethod",
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.transform.module
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.transform.utils import ( # noqa: E501
|
||||
TransformTuple,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class CompressedTensorsLinearTransformMethod(LinearMethodBase):
|
||||
@@ -48,11 +49,12 @@ class CompressedTensorsLinearTransformMethod(LinearMethodBase):
|
||||
|
||||
assert input_tfms or output_tfms
|
||||
|
||||
if is_qutlass_fp4_scheme(quant_scheme, input_tfms):
|
||||
if is_qutlass_fp4_scheme(
|
||||
quant_scheme, input_tfms
|
||||
) and current_platform.has_device_capability(100):
|
||||
return QutlassNvFP4LinearMethod(quant_method, input_tfms, output_tfms)
|
||||
|
||||
# hadacore or dense gemm is selected by Transform module
|
||||
|
||||
return cls(quant_method, input_tfms, output_tfms)
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -32,7 +32,7 @@ class HadamardTransform(torch.nn.Module):
|
||||
transforms: dict[int, TransformTuple] # info parsed from transforms config
|
||||
weight: SharedWeightParameter # container for shared tensors
|
||||
|
||||
scales: dict[int, float] # hadamard scale, usually sqrt(matrix.size(0))
|
||||
scaled_data_ptrs: set[int] = set()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -44,7 +44,6 @@ class HadamardTransform(torch.nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.transforms = transforms
|
||||
self.scales = {}
|
||||
|
||||
if get_tensor_model_parallel_world_size() > 1:
|
||||
raise NotImplementedError(
|
||||
@@ -64,26 +63,26 @@ class HadamardTransform(torch.nn.Module):
|
||||
)
|
||||
|
||||
data_key = self._get_data_key(scheme, weight_size)
|
||||
# load up in model's default precision, rather than using scheme.precision
|
||||
self.weight.add_partition(
|
||||
part_index,
|
||||
data_key,
|
||||
size=(weight_size, weight_size),
|
||||
dtype=scheme.precision,
|
||||
)
|
||||
|
||||
# validate that shared tensors and schemes are correct
|
||||
self._validate_input_transforms()
|
||||
|
||||
def process_weights_after_loading(self):
|
||||
for part_id in self.weight.partitions:
|
||||
data = self.weight.partitions[part_id].data
|
||||
|
||||
for part_id, partition in self.weight.partitions.items():
|
||||
# required by torch.compile
|
||||
self.weight.process_weights_after_loading()
|
||||
|
||||
# precompute scale as a runtime multiply, not division
|
||||
# do not fold into weight in order to utilize FWHT
|
||||
self.scales[part_id] = 1 / math.sqrt(data.size(0))
|
||||
# Merge normalization scale directly into weight, must be done only once
|
||||
data_ptr = partition.data.data_ptr()
|
||||
if data_ptr not in HadamardTransform.scaled_data_ptrs:
|
||||
partition.data.div_(math.sqrt(partition.data.size(0)))
|
||||
HadamardTransform.scaled_data_ptrs.add(data_ptr)
|
||||
|
||||
# FUTURE: avoid runtime transpose by processing weights
|
||||
# prior to apply
|
||||
@@ -111,26 +110,19 @@ class HadamardTransform(torch.nn.Module):
|
||||
weight = (
|
||||
weight if self.transforms[part_id].args.inverse else weight.T
|
||||
) # linear := x(W.T)
|
||||
scale = self.scales[part_id]
|
||||
|
||||
if self.transforms[part_id].scheme.head_dim is not None:
|
||||
value = value.unflatten(-1, (-1, weight.size(0)))
|
||||
value = (
|
||||
dispatch_unquantized_gemm()(
|
||||
self, value.to(weight.dtype), weight, None
|
||||
).to(value.dtype)
|
||||
* scale
|
||||
)
|
||||
value = dispatch_unquantized_gemm()(
|
||||
self, value.to(weight.dtype), weight, None
|
||||
).to(value.dtype)
|
||||
value = value.flatten(-2, -1)
|
||||
|
||||
return value
|
||||
|
||||
return (
|
||||
dispatch_unquantized_gemm()(
|
||||
self, value.to(weight.dtype), weight, None
|
||||
).to(value.dtype)
|
||||
* scale
|
||||
)
|
||||
return dispatch_unquantized_gemm()(
|
||||
self, value.to(weight.dtype), weight, None
|
||||
).to(value.dtype)
|
||||
|
||||
def _get_data_key(self, scheme: TransformScheme, weight_size: int) -> Hashable:
|
||||
return (id(scheme), weight_size)
|
||||
|
||||
+101
-6
@@ -2,7 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm._custom_ops import fusedQuantizeNv
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
_LINEAR_BACKEND_KERNEL_MAP,
|
||||
NvFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsScheme,
|
||||
CompressedTensorsW4A4Fp4,
|
||||
@@ -11,18 +17,32 @@ from vllm.model_executor.layers.quantization.compressed_tensors.transform.linear
|
||||
CompressedTensorsLinearTransformMethod,
|
||||
TransformTuple,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.qutlass_utils import to_blocked
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
slice_nvfp4_output,
|
||||
)
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_scaled_fp4_mm,
|
||||
)
|
||||
|
||||
__all__ = ["is_qutlass_fp4_scheme", "QutlassNvFP4LinearMethod"]
|
||||
|
||||
NVFP4_MAX = 6.0
|
||||
|
||||
|
||||
# QUTLASS supports transform block sizes (16, 32, 64, 128) for NVFP4
|
||||
# https://github.com/IST-DASLab/qutlass/blob/v0.2.0/qutlass/csrc/bindings.cpp#L413-L414
|
||||
def is_qutlass_fp4_scheme(
|
||||
quant_scheme: CompressedTensorsScheme | None,
|
||||
input_tfms: dict[int, TransformTuple],
|
||||
) -> bool:
|
||||
return (
|
||||
isinstance(quant_scheme, (CompressedTensorsW4A4Fp4,))
|
||||
and len(input_tfms) == 1
|
||||
and input_tfms[0].scheme.head_dim == quant_scheme.group_size
|
||||
isinstance(quant_scheme, CompressedTensorsW4A4Fp4)
|
||||
and len(input_tfms) >= 1
|
||||
and all(
|
||||
input_tfm.scheme.head_dim in (16, 32, 64, 128)
|
||||
for input_tfm in input_tfms.values()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -50,15 +70,90 @@ class QutlassNvFP4LinearMethod(CompressedTensorsLinearTransformMethod):
|
||||
)
|
||||
|
||||
assert self.input_transform is not None
|
||||
assert len(self.input_transform.weight) == 1
|
||||
assert self.input_transform.weight[0].size(0) == layer.scheme.group_size
|
||||
assert len(self.input_transform.weight.partitions) >= 1
|
||||
|
||||
return ret
|
||||
|
||||
@staticmethod
|
||||
def _get_flashinfer_gemm_backend(kernel: NvFp4LinearKernel) -> str:
|
||||
"""
|
||||
Given a kernel, find the string that is needed to be passed into
|
||||
`flashinfer_scaled_fp4_mm`, using
|
||||
vllm.model_executor.kernels.linear._LINEAR_BACKEND_KERNEL_MAP as source of truth
|
||||
"""
|
||||
kernel_type = type(kernel)
|
||||
for key, kernels in _LINEAR_BACKEND_KERNEL_MAP.items():
|
||||
if not key.startswith("flashinfer_") or kernel_type not in kernels:
|
||||
continue
|
||||
backend = key.removeprefix("flashinfer_")
|
||||
# flashinfer GEMM backend uses "cute-dsl", not "cutedsl"
|
||||
return backend.replace("cutedsl", "cute-dsl")
|
||||
raise ValueError(
|
||||
f"QutlassNvFP4 transform requires a FlashInfer kernel, "
|
||||
f"got {kernel_type.__name__}"
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
super().process_weights_after_loading(layer)
|
||||
|
||||
assert self.input_transform is not None
|
||||
layer.hadamard_matrix = self.input_transform.weight.partitions[0].data
|
||||
|
||||
# fusedQuantizeNv stores raw absmax as block scales (sf = absmax),
|
||||
# while CT weights use sf = absmax * SFScaleVal / 6.0. The GEMM
|
||||
# computes alpha * sum(fp4_a * sf_a * fp4_w * sf_w), so alpha must
|
||||
# compensate: alpha = weight_global_scale / 6.0
|
||||
layer.fused_alpha = Parameter(
|
||||
layer.weight_global_scale / NVFP4_MAX, requires_grad=False
|
||||
)
|
||||
|
||||
layer.fused_global_scale = Parameter(
|
||||
torch.tensor(
|
||||
[NVFP4_MAX],
|
||||
dtype=torch.float32,
|
||||
device=layer.weight_global_scale.device,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
layer.flashinfer_gemm_backend = self._get_flashinfer_gemm_backend(
|
||||
layer.scheme.kernel
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError()
|
||||
assert bias is None
|
||||
output_size = layer.output_size_per_partition
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
|
||||
x_flat = x.contiguous().flatten(end_dim=-2)
|
||||
|
||||
x_fp4, x_scales = fusedQuantizeNv(
|
||||
x_flat, layer.hadamard_matrix, layer.fused_global_scale
|
||||
)
|
||||
|
||||
x_scales_blocked = to_blocked(x_scales, backend="triton").view(x_scales.shape)
|
||||
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
x_fp4,
|
||||
layer.weight,
|
||||
x_scales_blocked,
|
||||
layer.weight_scale,
|
||||
layer.fused_alpha,
|
||||
x.dtype,
|
||||
backend=layer.flashinfer_gemm_backend,
|
||||
)
|
||||
|
||||
out = slice_nvfp4_output(out, output_size)
|
||||
|
||||
if self.output_transform is not None:
|
||||
for part_id, (start, length) in enumerate(self.partition_ranges):
|
||||
out[:, start : start + length] = self.output_transform(
|
||||
out[:, start : start + length].clone(), part_id=part_id
|
||||
)
|
||||
|
||||
return out.view(*output_shape)
|
||||
|
||||
@@ -84,6 +84,7 @@ def triton_scale_swizzle(
|
||||
)
|
||||
|
||||
|
||||
@torch.library.custom_op("vllm::triton_mx_block_rearrange", mutates_args=())
|
||||
def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Rearranges an E8M0 tensor scale from row-major format to
|
||||
@@ -142,6 +143,14 @@ def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor:
|
||||
return out
|
||||
|
||||
|
||||
@triton_mx_block_rearrange.register_fake
|
||||
def _triton_mx_block_rearrange_fake(scale_tensor: torch.Tensor) -> torch.Tensor:
|
||||
rows, cols = scale_tensor.shape
|
||||
padded_rows = cdiv(rows, 128) * 128
|
||||
padded_cols = cdiv(cols, 4) * 4
|
||||
return scale_tensor.new_empty((padded_rows, padded_cols))
|
||||
|
||||
|
||||
def to_blocked(
|
||||
input_matrix: torch.Tensor, backend: Literal["torch", "triton"] = "triton"
|
||||
) -> torch.Tensor:
|
||||
@@ -157,7 +166,7 @@ def to_blocked(
|
||||
backend: "torch" (PyTorch path) or "triton" (Triton kernel)
|
||||
|
||||
Returns:
|
||||
Rearranged tensor of shape (32*cdiv(H,128), 16*cdiv(W,4))
|
||||
Rearranged flattened tensor of size (32*cdiv(H,128) * 16*cdiv(W,4))
|
||||
"""
|
||||
if backend == "triton":
|
||||
return triton_mx_block_rearrange(input_matrix).flatten()
|
||||
|
||||
Reference in New Issue
Block a user