Add the QuantizedActivation linear-kernel contract (#44260)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-12 13:48:15 -07:00
committed by GitHub
co-authored by Claude
parent 9eaacb23ec
commit c90650088d
13 changed files with 327 additions and 27 deletions
@@ -0,0 +1,71 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
A QuantizedActivation is a pre-quantized activation produced by a fused kernel
and consumed directly by a linear layer, letting the layer skip its own input
quantization. A linear advertises the key its kernel can consume via
expose_input_quant_key; the kernel validates and reads the activation via
as_quantized_activation.
"""
from dataclasses import dataclass
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
@dataclass
class QuantizedActivation:
"""A quantized activation paired with its scale and original metadata.
The quant_key describes how data and scale are to be interpreted (dtype,
scale granularity, value packing). Details the key does not capture, such
as blockscale layout or activation padding, must follow the consumer
kernel's convention.
TODO(mgoin): Encode layout and padding requirements in the contract so
producers can match consumer kernels without relying on convention.
"""
data: torch.Tensor
scale: torch.Tensor
orig_dtype: torch.dtype
orig_shape: torch.Size
quant_key: QuantKey
def expose_input_quant_key(layer: torch.nn.Module, kernel) -> None:
"""Advertise the kernel's pre-quantized input key on the layer, if any.
This is the bridge from a kernel's input_quant_key() to the
layer.input_quant_key attribute that fusion call sites read. The attribute
is left unset when the kernel quantizes its own input, so non-supporting
backends never receive a QuantizedActivation.
TODO(mgoin): Producers also need the consumer's quantization scales (e.g.
static input scale, global scale). Expose those here as well so producers
do not reach into kernel-specific layer attributes.
"""
key = kernel.input_quant_key()
if key is not None:
layer.input_quant_key = key
def as_quantized_activation(
x: "torch.Tensor | QuantizedActivation", expected_key: QuantKey | None
) -> "QuantizedActivation | None":
"""Validate and narrow a pre-quantized activation for a consumer kernel.
Returns the QuantizedActivation when x is one whose key matches the
kernel's declared expected_key, and None when x is a plain tensor (the
caller quantizes in-kernel). Raises on a key mismatch so a wrongly routed
activation fails loudly instead of being silently re-quantized.
"""
if not isinstance(x, QuantizedActivation):
return None
assert x.quant_key == expected_key, (
f"QuantizedActivation key {x.quant_key} != consumer kernel "
f"input_quant_key {expected_key}"
)
return x
@@ -7,6 +7,9 @@ from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
from vllm.model_executor.layers.fusion.quant_activation import (
expose_input_quant_key,
)
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsScheme,
)
@@ -87,6 +90,8 @@ class CompressedTensorsW4A4Fp4(CompressedTensorsScheme):
)
layer.register_parameter("input_global_scale", input_global_scale)
expose_input_quant_key(layer, self.kernel)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Rename CT checkpoint names to standardized names
layer.weight = layer.weight_packed
@@ -13,6 +13,10 @@ from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
init_fp8_linear_kernel,
)
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
expose_input_quant_key,
)
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsScheme,
)
@@ -143,6 +147,8 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
module_name=self.__class__.__name__,
)
expose_input_quant_key(layer, self.fp8_linear)
def process_weights_after_loading(self, layer) -> None:
if self.strategy == QuantizationStrategy.TENSOR:
weight, weight_scale, input_scale = process_fp8_weight_tensor_strategy(
@@ -191,7 +197,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
x: torch.Tensor | QuantizedActivation,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.fp8_linear.apply_weights(layer, x, bias)
@@ -42,6 +42,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
make_nvfp4_moe_quant_config,
select_nvfp4_moe_backend,
)
from vllm.model_executor.layers.fusion.quant_activation import (
expose_input_quant_key,
)
from vllm.model_executor.layers.linear import (
LinearBase,
LinearMethodBase,
@@ -1191,6 +1194,8 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
layer.register_parameter("weight_scale", weight_scale)
expose_input_quant_key(layer, self.kernel)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if (
torch.unique(layer.input_scale).numel() != 1