diff --git a/tests/quantization/test_compressed_tensors.py b/tests/quantization/test_compressed_tensors.py index 626717cd4a..8c50988870 100644 --- a/tests/quantization/test_compressed_tensors.py +++ b/tests/quantization/test_compressed_tensors.py @@ -473,6 +473,11 @@ def test_compressed_tensors_w4a8_fp8(vllm_runner, args): "Flat is better than nested.\nSparse is better than dense.", 150.0, ), + ( + "nm-testing/Llama-3.2-1B-Instruct-quipv16-nvfp4", + "Flat is better than nested.\nSparse is better than dense.", + 150.0, + ), ], ) def test_compressed_tensors_transforms_perplexity( diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index b1d2f1344d..9f592386c1 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -3994,9 +3994,40 @@ def fusedQuantizeNv( padded_rows, padded_cols, dtype=torch.float8_e4m3fn, device=a.device ) - return torch.ops._qutlass_C.fusedQuantizeNvAbsMax( - a, b, xh_e2m1, xh_e4m3, global_scale - ) + safeFusedQuantizeNv(a, b, xh_e2m1, xh_e4m3, global_scale) + return xh_e2m1, xh_e4m3 + + +@torch.library.custom_op( + "vllm::safeFusedQuantizeNv", mutates_args=("xh_e2m1", "xh_e4m3") +) +def safeFusedQuantizeNv( + a: torch.Tensor, + b: torch.Tensor, + xh_e2m1: torch.Tensor, + xh_e4m3: torch.Tensor, + global_scale: torch.Tensor, +) -> None: + """ + Wrapper for QUTLASS fusedQuantizeNv method that operates on tensors in-place + rather than returning them, to prevent torch 2.12+ errors that outputs of custom + operators may not alias any inputs to the custom operator. + """ + torch.ops._qutlass_C.fusedQuantizeNvAbsMax(a, b, xh_e2m1, xh_e4m3, global_scale) + return + + +if hasattr(torch.ops._qutlass_C, "fusedQuantizeNv"): + + @register_fake("vllm::safeFusedQuantizeNv") + def _fake_fused_quantize_nv( + a: torch.Tensor, + b: torch.Tensor, + xh_e2m1: torch.Tensor, + xh_e4m3: torch.Tensor, + global_scale: torch.Tensor, + ) -> None: + return def hadacore_transform(x: torch.Tensor, inplace: bool = True) -> torch.Tensor: diff --git a/vllm/model_executor/layers/linear.py b/vllm/model_executor/layers/linear.py index df105f0663..e4662148ff 100644 --- a/vllm/model_executor/layers/linear.py +++ b/vllm/model_executor/layers/linear.py @@ -50,6 +50,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [ "UnquantizedLinearMethod", "CompressedTensorsLinearMethod", "CompressedTensorsLinearTransformMethod", + "QutlassNvFP4LinearMethod", "AutoAWQMarlinLinearMethod", "AutoAWQLinearMethod", "AutoGPTQLinearMethod", diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/transform/linear.py b/vllm/model_executor/layers/quantization/compressed_tensors/transform/linear.py index bd1964e667..0cde020627 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/transform/linear.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/transform/linear.py @@ -25,6 +25,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.transform.module from vllm.model_executor.layers.quantization.compressed_tensors.transform.utils import ( # noqa: E501 TransformTuple, ) +from vllm.platforms import current_platform class CompressedTensorsLinearTransformMethod(LinearMethodBase): @@ -48,11 +49,12 @@ class CompressedTensorsLinearTransformMethod(LinearMethodBase): assert input_tfms or output_tfms - if is_qutlass_fp4_scheme(quant_scheme, input_tfms): + if is_qutlass_fp4_scheme( + quant_scheme, input_tfms + ) and current_platform.has_device_capability(100): return QutlassNvFP4LinearMethod(quant_method, input_tfms, output_tfms) # hadacore or dense gemm is selected by Transform module - return cls(quant_method, input_tfms, output_tfms) def __init__( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/transform/module.py b/vllm/model_executor/layers/quantization/compressed_tensors/transform/module.py index f5589c8c07..7550593493 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/transform/module.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/transform/module.py @@ -32,7 +32,7 @@ class HadamardTransform(torch.nn.Module): transforms: dict[int, TransformTuple] # info parsed from transforms config weight: SharedWeightParameter # container for shared tensors - scales: dict[int, float] # hadamard scale, usually sqrt(matrix.size(0)) + scaled_data_ptrs: set[int] = set() def __init__( self, @@ -44,7 +44,6 @@ class HadamardTransform(torch.nn.Module): ): super().__init__() self.transforms = transforms - self.scales = {} if get_tensor_model_parallel_world_size() > 1: raise NotImplementedError( @@ -64,26 +63,26 @@ class HadamardTransform(torch.nn.Module): ) data_key = self._get_data_key(scheme, weight_size) + # load up in model's default precision, rather than using scheme.precision self.weight.add_partition( part_index, data_key, size=(weight_size, weight_size), - dtype=scheme.precision, ) # validate that shared tensors and schemes are correct self._validate_input_transforms() def process_weights_after_loading(self): - for part_id in self.weight.partitions: - data = self.weight.partitions[part_id].data - + for part_id, partition in self.weight.partitions.items(): # required by torch.compile self.weight.process_weights_after_loading() - # precompute scale as a runtime multiply, not division - # do not fold into weight in order to utilize FWHT - self.scales[part_id] = 1 / math.sqrt(data.size(0)) + # Merge normalization scale directly into weight, must be done only once + data_ptr = partition.data.data_ptr() + if data_ptr not in HadamardTransform.scaled_data_ptrs: + partition.data.div_(math.sqrt(partition.data.size(0))) + HadamardTransform.scaled_data_ptrs.add(data_ptr) # FUTURE: avoid runtime transpose by processing weights # prior to apply @@ -111,26 +110,19 @@ class HadamardTransform(torch.nn.Module): weight = ( weight if self.transforms[part_id].args.inverse else weight.T ) # linear := x(W.T) - scale = self.scales[part_id] if self.transforms[part_id].scheme.head_dim is not None: value = value.unflatten(-1, (-1, weight.size(0))) - value = ( - dispatch_unquantized_gemm()( - self, value.to(weight.dtype), weight, None - ).to(value.dtype) - * scale - ) + value = dispatch_unquantized_gemm()( + self, value.to(weight.dtype), weight, None + ).to(value.dtype) value = value.flatten(-2, -1) return value - return ( - dispatch_unquantized_gemm()( - self, value.to(weight.dtype), weight, None - ).to(value.dtype) - * scale - ) + return dispatch_unquantized_gemm()( + self, value.to(weight.dtype), weight, None + ).to(value.dtype) def _get_data_key(self, scheme: TransformScheme, weight_size: int) -> Hashable: return (id(scheme), weight_size) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/transform/schemes/linear_qutlass_nvfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/transform/schemes/linear_qutlass_nvfp4.py index f0bb47a728..b8b771e3c7 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/transform/schemes/linear_qutlass_nvfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/transform/schemes/linear_qutlass_nvfp4.py @@ -2,7 +2,13 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch +from torch.nn.parameter import Parameter +from vllm._custom_ops import fusedQuantizeNv +from vllm.model_executor.kernels.linear import ( + _LINEAR_BACKEND_KERNEL_MAP, + NvFp4LinearKernel, +) from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501 CompressedTensorsScheme, CompressedTensorsW4A4Fp4, @@ -11,18 +17,32 @@ from vllm.model_executor.layers.quantization.compressed_tensors.transform.linear CompressedTensorsLinearTransformMethod, TransformTuple, ) +from vllm.model_executor.layers.quantization.qutlass_utils import to_blocked +from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( + slice_nvfp4_output, +) +from vllm.utils.flashinfer import ( + flashinfer_scaled_fp4_mm, +) __all__ = ["is_qutlass_fp4_scheme", "QutlassNvFP4LinearMethod"] +NVFP4_MAX = 6.0 + +# QUTLASS supports transform block sizes (16, 32, 64, 128) for NVFP4 +# https://github.com/IST-DASLab/qutlass/blob/v0.2.0/qutlass/csrc/bindings.cpp#L413-L414 def is_qutlass_fp4_scheme( quant_scheme: CompressedTensorsScheme | None, input_tfms: dict[int, TransformTuple], ) -> bool: return ( - isinstance(quant_scheme, (CompressedTensorsW4A4Fp4,)) - and len(input_tfms) == 1 - and input_tfms[0].scheme.head_dim == quant_scheme.group_size + isinstance(quant_scheme, CompressedTensorsW4A4Fp4) + and len(input_tfms) >= 1 + and all( + input_tfm.scheme.head_dim in (16, 32, 64, 128) + for input_tfm in input_tfms.values() + ) ) @@ -50,15 +70,90 @@ class QutlassNvFP4LinearMethod(CompressedTensorsLinearTransformMethod): ) assert self.input_transform is not None - assert len(self.input_transform.weight) == 1 - assert self.input_transform.weight[0].size(0) == layer.scheme.group_size + assert len(self.input_transform.weight.partitions) >= 1 return ret + @staticmethod + def _get_flashinfer_gemm_backend(kernel: NvFp4LinearKernel) -> str: + """ + Given a kernel, find the string that is needed to be passed into + `flashinfer_scaled_fp4_mm`, using + vllm.model_executor.kernels.linear._LINEAR_BACKEND_KERNEL_MAP as source of truth + """ + kernel_type = type(kernel) + for key, kernels in _LINEAR_BACKEND_KERNEL_MAP.items(): + if not key.startswith("flashinfer_") or kernel_type not in kernels: + continue + backend = key.removeprefix("flashinfer_") + # flashinfer GEMM backend uses "cute-dsl", not "cutedsl" + return backend.replace("cutedsl", "cute-dsl") + raise ValueError( + f"QutlassNvFP4 transform requires a FlashInfer kernel, " + f"got {kernel_type.__name__}" + ) + + def process_weights_after_loading(self, layer): + super().process_weights_after_loading(layer) + + assert self.input_transform is not None + layer.hadamard_matrix = self.input_transform.weight.partitions[0].data + + # fusedQuantizeNv stores raw absmax as block scales (sf = absmax), + # while CT weights use sf = absmax * SFScaleVal / 6.0. The GEMM + # computes alpha * sum(fp4_a * sf_a * fp4_w * sf_w), so alpha must + # compensate: alpha = weight_global_scale / 6.0 + layer.fused_alpha = Parameter( + layer.weight_global_scale / NVFP4_MAX, requires_grad=False + ) + + layer.fused_global_scale = Parameter( + torch.tensor( + [NVFP4_MAX], + dtype=torch.float32, + device=layer.weight_global_scale.device, + ), + requires_grad=False, + ) + + layer.flashinfer_gemm_backend = self._get_flashinfer_gemm_backend( + layer.scheme.kernel + ) + def apply( self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: - raise NotImplementedError() + assert bias is None + output_size = layer.output_size_per_partition + output_shape = [*x.shape[:-1], output_size] + + x_flat = x.contiguous().flatten(end_dim=-2) + + x_fp4, x_scales = fusedQuantizeNv( + x_flat, layer.hadamard_matrix, layer.fused_global_scale + ) + + x_scales_blocked = to_blocked(x_scales, backend="triton").view(x_scales.shape) + + out = flashinfer_scaled_fp4_mm( + x_fp4, + layer.weight, + x_scales_blocked, + layer.weight_scale, + layer.fused_alpha, + x.dtype, + backend=layer.flashinfer_gemm_backend, + ) + + out = slice_nvfp4_output(out, output_size) + + if self.output_transform is not None: + for part_id, (start, length) in enumerate(self.partition_ranges): + out[:, start : start + length] = self.output_transform( + out[:, start : start + length].clone(), part_id=part_id + ) + + return out.view(*output_shape) diff --git a/vllm/model_executor/layers/quantization/qutlass_utils.py b/vllm/model_executor/layers/quantization/qutlass_utils.py index 315ecd0c00..86b0548307 100644 --- a/vllm/model_executor/layers/quantization/qutlass_utils.py +++ b/vllm/model_executor/layers/quantization/qutlass_utils.py @@ -84,6 +84,7 @@ def triton_scale_swizzle( ) +@torch.library.custom_op("vllm::triton_mx_block_rearrange", mutates_args=()) def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor: """ Rearranges an E8M0 tensor scale from row-major format to @@ -142,6 +143,14 @@ def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor: return out +@triton_mx_block_rearrange.register_fake +def _triton_mx_block_rearrange_fake(scale_tensor: torch.Tensor) -> torch.Tensor: + rows, cols = scale_tensor.shape + padded_rows = cdiv(rows, 128) * 128 + padded_cols = cdiv(cols, 4) * 4 + return scale_tensor.new_empty((padded_rows, padded_cols)) + + def to_blocked( input_matrix: torch.Tensor, backend: Literal["torch", "triton"] = "triton" ) -> torch.Tensor: @@ -157,7 +166,7 @@ def to_blocked( backend: "torch" (PyTorch path) or "triton" (Triton kernel) Returns: - Rearranged tensor of shape (32*cdiv(H,128), 16*cdiv(W,4)) + Rearranged flattened tensor of size (32*cdiv(H,128) * 16*cdiv(W,4)) """ if backend == "triton": return triton_mx_block_rearrange(input_matrix).flatten()