Add the QuantizedActivation linear-kernel contract (#44260)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-12 13:48:15 -07:00
committed by GitHub
co-authored by Claude
parent 9eaacb23ec
commit c90650088d
13 changed files with 327 additions and 27 deletions
@@ -8,6 +8,8 @@ from typing import Any, ClassVar, Generic, TypeVar
import torch
from typing_extensions import Self
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
@dataclass
class MMLinearLayerConfig: ...
@@ -237,6 +239,12 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
"""
self.config = config
def input_quant_key(self) -> QuantKey | None:
"""Return the input quantization key supported by this kernel. If the kernel
does not support input quantization outside of the kernel, return None.
"""
return None
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Process and transform weights after loading from checkpoint.
@@ -6,6 +6,8 @@ from dataclasses import dataclass
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
@dataclass
class NvFp4LinearLayerConfig:
@@ -33,6 +35,12 @@ class NvFp4LinearKernel(ABC):
assert self.is_supported()[0]
self.config = config
def input_quant_key(self) -> QuantKey | None:
"""Return the input quantization key supported by this kernel. If the kernel
does not support input quantization outside of the kernel, return None.
"""
return None
@classmethod
@abstractmethod
def is_supported(
@@ -4,12 +4,20 @@
import torch
from vllm._custom_ops import scaled_fp4_quant
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
as_quantized_activation,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
pad_nvfp4_activation_for_cutlass,
pad_nvfp4_weight_for_cutlass,
slice_nvfp4_output,
swizzle_blockscale,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
flashinfer_scaled_fp4_mm,
@@ -23,6 +31,11 @@ from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
"""NVFP4 GEMM via FlashInfer's CUTLASS wrapper."""
def input_quant_key(self) -> QuantKey | None:
"""This kernel supports dynamic quantization of the input. By
convention, pre-quantized blockscales must use the swizzled layout."""
return kNvfp4Dynamic
@classmethod
def is_supported(
cls, compute_capability: int | None = None
@@ -56,21 +69,29 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
x: torch.Tensor | QuantizedActivation,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
output_size = layer.output_size_per_partition
output_dtype = x.dtype
output_shape = [*x.shape[:-1], output_size]
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
x_fp4, x_blockscale = scaled_fp4_quant(
x,
layer.input_global_scale_inv,
is_sf_swizzled_layout=True,
backend="flashinfer-cutlass",
padded_n=x.shape[-1] + weights_padding_bytes * 2,
)
qa = as_quantized_activation(x, self.input_quant_key())
if qa is not None:
x_fp4, x_blockscale = qa.data, qa.scale
x_fp4 = pad_nvfp4_activation_for_cutlass(x_fp4, weights_padding_bytes)
output_dtype = qa.orig_dtype
output_shape = [*qa.orig_shape[:-1], output_size]
else:
assert isinstance(x, torch.Tensor)
output_dtype = x.dtype
output_shape = [*x.shape[:-1], output_size]
x_fp4, x_blockscale = scaled_fp4_quant(
x,
layer.input_global_scale_inv,
is_sf_swizzled_layout=True,
backend="flashinfer-cutlass",
padded_n=x.shape[-1] + weights_padding_bytes * 2,
)
out = flashinfer_scaled_fp4_mm(
x_fp4,
@@ -8,6 +8,10 @@ from typing import Generic, TypeVar
import torch
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
as_quantized_activation,
)
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -71,6 +75,17 @@ class ScaledMMLinearKernel(Generic[_ConfigT, _ParamsT], ABC):
self.config = c
self.layer_param_names = layer_param_names
def input_quant_key(self) -> QuantKey | None:
"""The activation quant key this kernel can consume pre-quantized.
Manual fusion uses this to decide whether to hoist activation
quantization out of apply_weights into an upstream fused kernel.
Return None when the kernel needs in-kernel quantization (custom
padding or swizzling, dynamic scales, etc.). Kernels that return a
key must consume the activation via as_quantized_activation.
"""
return None
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
raise NotImplementedError
@@ -120,30 +135,30 @@ class FP8ScaledMMLinearKernel(
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
x: torch.Tensor | QuantizedActivation,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
fp8_dtype = self.fp8_dtype
maybe_out_dtype = self.config.out_dtype
w, w_s, x_s, x_s_ub = self._get_layer_params(layer)
# ops.scaled_fp8_quant supports both dynamic and static quant.
# If dynamic, layer.input_scale is None and x_s computed from x.
# If static, layer.input_scale is scalar and x_s is input_scale.
# View input as 2D matrix for fp8 methods
x_2d = x.view(-1, x.shape[-1])
output_shape = [*x.shape[:-1], w.shape[1]]
out_dtype = x.dtype if maybe_out_dtype is None else maybe_out_dtype
qa = as_quantized_activation(x, self.input_quant_key())
if qa is not None:
x_data, x_s = qa.data, qa.scale
orig_shape, orig_dtype = qa.orig_shape, qa.orig_dtype
assert x_data.dtype == fp8_dtype
else:
assert isinstance(x, torch.Tensor)
x_data = x
orig_shape, orig_dtype = x.shape, x.dtype
x_2d = x_data.view(-1, x_data.shape[-1])
output_shape = [*orig_shape[:-1], w.shape[1]]
out_dtype = orig_dtype if maybe_out_dtype is None else maybe_out_dtype
# If input not quantized
# TODO(luka) remove this path if not used anymore
x_2d_q = x_2d
if x.dtype != fp8_dtype:
x_2d_q, x_s = self.quant_fp8(
x_2d,
x_s,
x_s_ub,
)
if qa is None:
x_2d_q, x_s = self.quant_fp8(x_2d, x_s, x_s_ub)
return self.apply_scaled_mm(
A=x_2d_q,
B=w,
@@ -11,6 +11,8 @@ from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils import replace_parameter
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
CUTLASS_BLOCK_FP8_SUPPORTED,
@@ -171,6 +173,13 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def input_quant_key(self) -> QuantKey | None:
"""Only static per-tensor activation quantization is supported for external
quantization."""
if self.config.activation_quant_key == kFp8StaticTensorSym:
return kFp8StaticTensorSym
return None
@staticmethod
def _pad_to_alignment(
x: torch.Tensor, dim: int, alignment: int, value: float = 0.0
@@ -12,6 +12,8 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
@@ -62,6 +64,11 @@ class FlashInferFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
return True, None
def input_quant_key(self) -> QuantKey | None:
if self.config.activation_quant_key == kFp8StaticTensorSym:
return kFp8StaticTensorSym
return None
def apply_scaled_mm(
self,
*,