[Quantization] Add ModelOpt IQ2_XS, IQ2_XXS, and Q8_0 support via b12x (#59604)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Luke Alonso <lalonso@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
Andrii Skliar
2026-10-13 09:06:33 -07:00
committed by GitHub
co-authored by Andrii Skliar Luke Alonso
parent ac71b17517
commit 6849ca696a
15 changed files with 1238 additions and 25 deletions
@@ -135,6 +135,7 @@ from vllm.model_executor.kernels.linear.nvfp4 import (
)
from vllm.model_executor.kernels.linear.nvfp4.b12x import (
B12xNvFp4LinearKernel,
B12xNvFp4W4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.nvfp4.cutlass import (
CutlassNvFp4LinearKernel,
@@ -275,6 +276,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
B12xMxFp4LinearKernel,
B12xMxfp8LinearKernel,
B12xNvFp4LinearKernel,
B12xNvFp4W4A16LinearKernel,
B12xTensorFP8ScaledMMLinearKernel,
},
"cutlass": {
@@ -582,6 +584,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
FlashInferCudnnNvFp4LinearKernel,
FbgemmNvFp4LinearKernel,
B12xNvFp4LinearKernel,
B12xNvFp4W4A16LinearKernel,
TorchNvFp4LinearKernel,
EmulationNvFp4LinearKernel,
HummingNvFp4LinearKernel,
@@ -1127,6 +1130,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
current platform."""
config = NvFp4LinearLayerConfig()
a16_kernels = (
B12xNvFp4W4A16LinearKernel,
FlashInferCuteDslNvFp4W4A16LinearKernel,
MarlinNvFp4LinearKernel,
HummingNvFp4LinearKernel,
@@ -1341,6 +1345,7 @@ __all__ = [
"B12xMxfp8LinearKernel",
"B12xMxFp4LinearKernel",
"B12xNvFp4LinearKernel",
"B12xNvFp4W4A16LinearKernel",
"init_mxfp4_linear_kernel",
"MxFp4LinearKernel",
"MxFp4LinearLayerConfig",
@@ -138,4 +138,64 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel):
)
__all__ = ["B12xNvFp4LinearKernel"]
class B12xNvFp4W4A16LinearKernel(B12xNvFp4LinearKernel):
"""BF16 x NVFP4 GEMM without activation quantization."""
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
blockscaled = _import_b12x_blockscaled()
assert blockscaled is not None
layer.b12x_nvfp4_packed_weight = blockscaled.pack_weight(
layer.weight,
layer.weight_scale,
recipe="nvfp4",
global_scale=layer.weight_global_scale,
)
def get_b12x_warmup_unit(
self,
layer: torch.nn.Module,
token_counts: tuple[int, ...],
output_dtype: torch.dtype,
) -> B12xWarmupUnit:
def compile() -> None:
for tokens in token_counts:
source = torch.zeros(
(tokens, layer.weight.shape[1] * 2),
dtype=output_dtype,
device=layer.weight.device,
)
self.apply_weights(layer, source)
return B12xWarmupUnit(
name="NVFP4 W4A16",
key=(
type(self),
layer.weight.device,
tuple(layer.weight.shape),
output_dtype,
),
compile=compile,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if x.dtype != torch.bfloat16:
raise ValueError(f"b12x NVFP4 W4A16 requires BF16 input, got {x.dtype}")
blockscaled = _import_b12x_blockscaled()
assert blockscaled is not None
output = blockscaled.mm(
x.reshape(-1, x.shape[-1]).contiguous(),
layer.b12x_nvfp4_packed_weight,
required_mode="a16",
)
if bias is not None:
output = output + bias
return output.view(*x.shape[:-1], layer.weight.shape[0])
__all__ = ["B12xNvFp4LinearKernel", "B12xNvFp4W4A16LinearKernel"]