Add the QuantizedActivation linear-kernel contract (#44260)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,8 @@ from typing import Any, ClassVar, Generic, TypeVar
|
||||
import torch
|
||||
from typing_extensions import Self
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
|
||||
@dataclass
|
||||
class MMLinearLayerConfig: ...
|
||||
@@ -237,6 +239,12 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
|
||||
"""
|
||||
self.config = config
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""Return the input quantization key supported by this kernel. If the kernel
|
||||
does not support input quantization outside of the kernel, return None.
|
||||
"""
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Process and transform weights after loading from checkpoint.
|
||||
|
||||
@@ -6,6 +6,8 @@ from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
|
||||
@dataclass
|
||||
class NvFp4LinearLayerConfig:
|
||||
@@ -33,6 +35,12 @@ class NvFp4LinearKernel(ABC):
|
||||
assert self.is_supported()[0]
|
||||
self.config = config
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""Return the input quantization key supported by this kernel. If the kernel
|
||||
does not support input quantization outside of the kernel, return None.
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def is_supported(
|
||||
|
||||
@@ -4,12 +4,20 @@
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
as_quantized_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
pad_nvfp4_activation_for_cutlass,
|
||||
pad_nvfp4_weight_for_cutlass,
|
||||
slice_nvfp4_output,
|
||||
swizzle_blockscale,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_scaled_fp4_mm,
|
||||
@@ -23,6 +31,11 @@ from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
|
||||
class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
"""NVFP4 GEMM via FlashInfer's CUTLASS wrapper."""
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""This kernel supports dynamic quantization of the input. By
|
||||
convention, pre-quantized blockscales must use the swizzled layout."""
|
||||
return kNvfp4Dynamic
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
@@ -56,21 +69,29 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
x: torch.Tensor | QuantizedActivation,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="flashinfer-cutlass",
|
||||
padded_n=x.shape[-1] + weights_padding_bytes * 2,
|
||||
)
|
||||
qa = as_quantized_activation(x, self.input_quant_key())
|
||||
if qa is not None:
|
||||
x_fp4, x_blockscale = qa.data, qa.scale
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(x_fp4, weights_padding_bytes)
|
||||
output_dtype = qa.orig_dtype
|
||||
output_shape = [*qa.orig_shape[:-1], output_size]
|
||||
else:
|
||||
assert isinstance(x, torch.Tensor)
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="flashinfer-cutlass",
|
||||
padded_n=x.shape[-1] + weights_padding_bytes * 2,
|
||||
)
|
||||
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
x_fp4,
|
||||
|
||||
@@ -8,6 +8,10 @@ from typing import Generic, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
as_quantized_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -71,6 +75,17 @@ class ScaledMMLinearKernel(Generic[_ConfigT, _ParamsT], ABC):
|
||||
self.config = c
|
||||
self.layer_param_names = layer_param_names
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""The activation quant key this kernel can consume pre-quantized.
|
||||
|
||||
Manual fusion uses this to decide whether to hoist activation
|
||||
quantization out of apply_weights into an upstream fused kernel.
|
||||
Return None when the kernel needs in-kernel quantization (custom
|
||||
padding or swizzling, dynamic scales, etc.). Kernels that return a
|
||||
key must consume the activation via as_quantized_activation.
|
||||
"""
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
raise NotImplementedError
|
||||
@@ -120,30 +135,30 @@ class FP8ScaledMMLinearKernel(
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
x: torch.Tensor | QuantizedActivation,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
fp8_dtype = self.fp8_dtype
|
||||
maybe_out_dtype = self.config.out_dtype
|
||||
w, w_s, x_s, x_s_ub = self._get_layer_params(layer)
|
||||
|
||||
# ops.scaled_fp8_quant supports both dynamic and static quant.
|
||||
# If dynamic, layer.input_scale is None and x_s computed from x.
|
||||
# If static, layer.input_scale is scalar and x_s is input_scale.
|
||||
# View input as 2D matrix for fp8 methods
|
||||
x_2d = x.view(-1, x.shape[-1])
|
||||
output_shape = [*x.shape[:-1], w.shape[1]]
|
||||
out_dtype = x.dtype if maybe_out_dtype is None else maybe_out_dtype
|
||||
qa = as_quantized_activation(x, self.input_quant_key())
|
||||
if qa is not None:
|
||||
x_data, x_s = qa.data, qa.scale
|
||||
orig_shape, orig_dtype = qa.orig_shape, qa.orig_dtype
|
||||
assert x_data.dtype == fp8_dtype
|
||||
else:
|
||||
assert isinstance(x, torch.Tensor)
|
||||
x_data = x
|
||||
orig_shape, orig_dtype = x.shape, x.dtype
|
||||
|
||||
x_2d = x_data.view(-1, x_data.shape[-1])
|
||||
output_shape = [*orig_shape[:-1], w.shape[1]]
|
||||
out_dtype = orig_dtype if maybe_out_dtype is None else maybe_out_dtype
|
||||
|
||||
# If input not quantized
|
||||
# TODO(luka) remove this path if not used anymore
|
||||
x_2d_q = x_2d
|
||||
if x.dtype != fp8_dtype:
|
||||
x_2d_q, x_s = self.quant_fp8(
|
||||
x_2d,
|
||||
x_s,
|
||||
x_s_ub,
|
||||
)
|
||||
if qa is None:
|
||||
x_2d_q, x_s = self.quant_fp8(x_2d, x_s, x_s_ub)
|
||||
return self.apply_scaled_mm(
|
||||
A=x_2d_q,
|
||||
B=w,
|
||||
|
||||
@@ -11,6 +11,8 @@ from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils import replace_parameter
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
CUTLASS_BLOCK_FP8_SUPPORTED,
|
||||
@@ -171,6 +173,13 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""Only static per-tensor activation quantization is supported for external
|
||||
quantization."""
|
||||
if self.config.activation_quant_key == kFp8StaticTensorSym:
|
||||
return kFp8StaticTensorSym
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _pad_to_alignment(
|
||||
x: torch.Tensor, dim: int, alignment: int, value: float = 0.0
|
||||
|
||||
@@ -12,6 +12,8 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
@@ -62,6 +64,11 @@ class FlashInferFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
|
||||
return True, None
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
if self.config.activation_quant_key == kFp8StaticTensorSym:
|
||||
return kFp8StaticTensorSym
|
||||
return None
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user