[Kernel] Add B12X dense linear backends (#52016)

Signed-off-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
Luke Alonso
2026-08-13 19:48:33 -07:00
committed by GitHub
parent fe4c5dcd4c
commit 3c79b1a8bf
17 changed files with 2753 additions and 3 deletions
@@ -0,0 +1,176 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
import torch
from vllm._custom_ops import scaled_fp4_quant
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.b12x import (
b12x_warmup_token_counts,
)
from vllm.utils.b12x import (
get_b12x_blockscaled as _import_b12x_blockscaled,
)
from vllm.utils.b12x import get_b12x_intrinsics as _import_b12x_intrinsics
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
def _apply_b12x_nvfp4_linear(
x: torch.Tensor,
weight: torch.Tensor,
weight_scale_storage: torch.Tensor,
input_global_scale_inv: torch.Tensor,
alpha: torch.Tensor,
bias: torch.Tensor | None,
) -> torch.Tensor:
blockscaled = _import_b12x_blockscaled()
assert blockscaled is not None
output_size = int(weight.shape[0])
output_shape = [*x.shape[:-1], output_size]
x_2d = x.reshape(-1, x.shape[-1])
x_packed, x_scale_swizzled = scaled_fp4_quant(
x_2d,
input_global_scale_inv,
is_sf_swizzled_layout=True,
)
output = blockscaled.mm_nvfp4(
x_packed,
x_scale_swizzled,
weight,
weight_scale_storage,
alpha,
out_dtype=x.dtype,
)
if bias is not None:
output = output + bias
return output.view(*output_shape)
def warmup_b12x_nvfp4_linear(
model: torch.nn.Module,
*,
max_tokens: int,
cudagraph_capture_sizes: Iterable[int] = (),
output_dtype: torch.dtype = torch.bfloat16,
) -> int:
if not current_platform.is_cuda():
return 0
if not current_platform.is_device_capability_family(120):
return 0
if output_dtype not in (torch.bfloat16, torch.float16):
output_dtype = torch.bfloat16
layer_map: dict[tuple[Any, ...], torch.nn.Module] = {}
for layer in model.modules():
if not getattr(layer, "b12x_nvfp4_linear", False):
continue
weight = layer.weight
weight_scale = layer.weight_scale
n, packed_k = map(int, weight.shape)
signature = (
weight.device,
n,
packed_k * 2,
weight.dtype,
weight_scale.dtype,
output_dtype,
)
layer_map.setdefault(signature, layer)
if not layer_map:
return 0
if _import_b12x_blockscaled() is None:
return 0
token_counts = b12x_warmup_token_counts(
max_tokens=max_tokens,
cudagraph_capture_sizes=cudagraph_capture_sizes,
)
warmed = 0
last_device: torch.device | None = None
with torch.inference_mode():
for signature, layer in layer_map.items():
weight = layer.weight
weight_scale = layer.weight_scale
k = signature[2]
last_device = weight.device
for tokens in token_counts:
source = torch.zeros(
(tokens, k),
dtype=output_dtype,
device=weight.device,
)
_apply_b12x_nvfp4_linear(
source,
weight,
weight_scale,
layer.input_global_scale_inv,
layer.alpha,
None,
)
warmed += 1
if warmed > 0 and last_device is not None and last_device.type == "cuda":
torch.accelerator.synchronize(last_device)
return warmed
class B12xNvFp4LinearKernel(NvFp4LinearKernel):
"""ModelOpt NVFP4 linear through the native B12X SM120 dense GEMM."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
del compute_capability
if not current_platform.is_cuda():
return False, "B12X NVFP4 kernels are only available on CUDA"
if not current_platform.is_device_capability_family(120):
return False, "B12X NVFP4 kernels require a Blackwell 12x device"
blockscaled = _import_b12x_blockscaled()
if blockscaled is None or _import_b12x_intrinsics() is None:
return False, "Install the B12X backend with `pip install vllm[b12x]`"
if not blockscaled.is_supported():
return False, "b12x native NVFP4 GEMM is not supported"
return True, None
@classmethod
def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
del config
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
intrinsics = _import_b12x_intrinsics()
assert intrinsics is not None
replace_parameter(
layer,
"weight_scale",
intrinsics.swizzle_block_scale(layer.weight_scale.data),
)
layer.b12x_nvfp4_linear = True
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return _apply_b12x_nvfp4_linear(
x,
layer.weight,
layer.weight_scale,
layer.input_global_scale_inv,
layer.alpha,
bias,
)
__all__ = ["B12xNvFp4LinearKernel", "warmup_b12x_nvfp4_linear"]