Add the QuantizedActivation linear-kernel contract (#44260)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-12 13:48:15 -07:00
committed by GitHub
co-authored by Claude
parent 9eaacb23ec
commit c90650088d
13 changed files with 327 additions and 27 deletions
@@ -6,6 +6,8 @@ from dataclasses import dataclass
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
@dataclass
class NvFp4LinearLayerConfig:
@@ -33,6 +35,12 @@ class NvFp4LinearKernel(ABC):
assert self.is_supported()[0]
self.config = config
def input_quant_key(self) -> QuantKey | None:
"""Return the input quantization key supported by this kernel. If the kernel
does not support input quantization outside of the kernel, return None.
"""
return None
@classmethod
@abstractmethod
def is_supported(
@@ -4,12 +4,20 @@
import torch
from vllm._custom_ops import scaled_fp4_quant
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
as_quantized_activation,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
pad_nvfp4_activation_for_cutlass,
pad_nvfp4_weight_for_cutlass,
slice_nvfp4_output,
swizzle_blockscale,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
flashinfer_scaled_fp4_mm,
@@ -23,6 +31,11 @@ from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
"""NVFP4 GEMM via FlashInfer's CUTLASS wrapper."""
def input_quant_key(self) -> QuantKey | None:
"""This kernel supports dynamic quantization of the input. By
convention, pre-quantized blockscales must use the swizzled layout."""
return kNvfp4Dynamic
@classmethod
def is_supported(
cls, compute_capability: int | None = None
@@ -56,21 +69,29 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
x: torch.Tensor | QuantizedActivation,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
output_size = layer.output_size_per_partition
output_dtype = x.dtype
output_shape = [*x.shape[:-1], output_size]
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
x_fp4, x_blockscale = scaled_fp4_quant(
x,
layer.input_global_scale_inv,
is_sf_swizzled_layout=True,
backend="flashinfer-cutlass",
padded_n=x.shape[-1] + weights_padding_bytes * 2,
)
qa = as_quantized_activation(x, self.input_quant_key())
if qa is not None:
x_fp4, x_blockscale = qa.data, qa.scale
x_fp4 = pad_nvfp4_activation_for_cutlass(x_fp4, weights_padding_bytes)
output_dtype = qa.orig_dtype
output_shape = [*qa.orig_shape[:-1], output_size]
else:
assert isinstance(x, torch.Tensor)
output_dtype = x.dtype
output_shape = [*x.shape[:-1], output_size]
x_fp4, x_blockscale = scaled_fp4_quant(
x,
layer.input_global_scale_inv,
is_sf_swizzled_layout=True,
backend="flashinfer-cutlass",
padded_n=x.shape[-1] + weights_padding_bytes * 2,
)
out = flashinfer_scaled_fp4_mm(
x_fp4,