[Quantization] Add ModelOpt IQ2_XS, IQ2_XXS, and Q8_0 support via b12x (#59604)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Luke Alonso <lalonso@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
co-authored by
Andrii Skliar
Luke Alonso
parent
ac71b17517
commit
6849ca696a
@@ -135,6 +135,7 @@ from vllm.model_executor.kernels.linear.nvfp4 import (
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4.b12x import (
|
||||
B12xNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4.cutlass import (
|
||||
CutlassNvFp4LinearKernel,
|
||||
@@ -275,6 +276,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
B12xMxFp4LinearKernel,
|
||||
B12xMxfp8LinearKernel,
|
||||
B12xNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
B12xTensorFP8ScaledMMLinearKernel,
|
||||
},
|
||||
"cutlass": {
|
||||
@@ -582,6 +584,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
|
||||
FlashInferCudnnNvFp4LinearKernel,
|
||||
FbgemmNvFp4LinearKernel,
|
||||
B12xNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
TorchNvFp4LinearKernel,
|
||||
EmulationNvFp4LinearKernel,
|
||||
HummingNvFp4LinearKernel,
|
||||
@@ -1127,6 +1130,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
current platform."""
|
||||
config = NvFp4LinearLayerConfig()
|
||||
a16_kernels = (
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
MarlinNvFp4LinearKernel,
|
||||
HummingNvFp4LinearKernel,
|
||||
@@ -1341,6 +1345,7 @@ __all__ = [
|
||||
"B12xMxfp8LinearKernel",
|
||||
"B12xMxFp4LinearKernel",
|
||||
"B12xNvFp4LinearKernel",
|
||||
"B12xNvFp4W4A16LinearKernel",
|
||||
"init_mxfp4_linear_kernel",
|
||||
"MxFp4LinearKernel",
|
||||
"MxFp4LinearLayerConfig",
|
||||
|
||||
@@ -138,4 +138,64 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["B12xNvFp4LinearKernel"]
|
||||
class B12xNvFp4W4A16LinearKernel(B12xNvFp4LinearKernel):
|
||||
"""BF16 x NVFP4 GEMM without activation quantization."""
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
blockscaled = _import_b12x_blockscaled()
|
||||
assert blockscaled is not None
|
||||
layer.b12x_nvfp4_packed_weight = blockscaled.pack_weight(
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
recipe="nvfp4",
|
||||
global_scale=layer.weight_global_scale,
|
||||
)
|
||||
|
||||
def get_b12x_warmup_unit(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
token_counts: tuple[int, ...],
|
||||
output_dtype: torch.dtype,
|
||||
) -> B12xWarmupUnit:
|
||||
def compile() -> None:
|
||||
for tokens in token_counts:
|
||||
source = torch.zeros(
|
||||
(tokens, layer.weight.shape[1] * 2),
|
||||
dtype=output_dtype,
|
||||
device=layer.weight.device,
|
||||
)
|
||||
self.apply_weights(layer, source)
|
||||
|
||||
return B12xWarmupUnit(
|
||||
name="NVFP4 W4A16",
|
||||
key=(
|
||||
type(self),
|
||||
layer.weight.device,
|
||||
tuple(layer.weight.shape),
|
||||
output_dtype,
|
||||
),
|
||||
compile=compile,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if x.dtype != torch.bfloat16:
|
||||
raise ValueError(f"b12x NVFP4 W4A16 requires BF16 input, got {x.dtype}")
|
||||
blockscaled = _import_b12x_blockscaled()
|
||||
assert blockscaled is not None
|
||||
output = blockscaled.mm(
|
||||
x.reshape(-1, x.shape[-1]).contiguous(),
|
||||
layer.b12x_nvfp4_packed_weight,
|
||||
required_mode="a16",
|
||||
)
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
return output.view(*x.shape[:-1], layer.weight.shape[0])
|
||||
|
||||
|
||||
__all__ = ["B12xNvFp4LinearKernel", "B12xNvFp4W4A16LinearKernel"]
|
||||
|
||||
Reference in New Issue
Block a user