Add the QuantizedActivation linear-kernel contract (#44260)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,8 @@ from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
|
||||
@dataclass
|
||||
class NvFp4LinearLayerConfig:
|
||||
@@ -33,6 +35,12 @@ class NvFp4LinearKernel(ABC):
|
||||
assert self.is_supported()[0]
|
||||
self.config = config
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""Return the input quantization key supported by this kernel. If the kernel
|
||||
does not support input quantization outside of the kernel, return None.
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def is_supported(
|
||||
|
||||
@@ -4,12 +4,20 @@
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
as_quantized_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
pad_nvfp4_activation_for_cutlass,
|
||||
pad_nvfp4_weight_for_cutlass,
|
||||
slice_nvfp4_output,
|
||||
swizzle_blockscale,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_scaled_fp4_mm,
|
||||
@@ -23,6 +31,11 @@ from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
|
||||
class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
"""NVFP4 GEMM via FlashInfer's CUTLASS wrapper."""
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""This kernel supports dynamic quantization of the input. By
|
||||
convention, pre-quantized blockscales must use the swizzled layout."""
|
||||
return kNvfp4Dynamic
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
@@ -56,21 +69,29 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
x: torch.Tensor | QuantizedActivation,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="flashinfer-cutlass",
|
||||
padded_n=x.shape[-1] + weights_padding_bytes * 2,
|
||||
)
|
||||
qa = as_quantized_activation(x, self.input_quant_key())
|
||||
if qa is not None:
|
||||
x_fp4, x_blockscale = qa.data, qa.scale
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(x_fp4, weights_padding_bytes)
|
||||
output_dtype = qa.orig_dtype
|
||||
output_shape = [*qa.orig_shape[:-1], output_size]
|
||||
else:
|
||||
assert isinstance(x, torch.Tensor)
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="flashinfer-cutlass",
|
||||
padded_n=x.shape[-1] + weights_padding_bytes * 2,
|
||||
)
|
||||
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
x_fp4,
|
||||
|
||||
Reference in New Issue
Block a user