[Kernel] Add B12X dense linear backends (#52016)
Signed-off-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
@@ -0,0 +1,176 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.b12x import (
|
||||
b12x_warmup_token_counts,
|
||||
)
|
||||
from vllm.utils.b12x import (
|
||||
get_b12x_blockscaled as _import_b12x_blockscaled,
|
||||
)
|
||||
from vllm.utils.b12x import get_b12x_intrinsics as _import_b12x_intrinsics
|
||||
|
||||
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
|
||||
|
||||
|
||||
def _apply_b12x_nvfp4_linear(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale_storage: torch.Tensor,
|
||||
input_global_scale_inv: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
blockscaled = _import_b12x_blockscaled()
|
||||
assert blockscaled is not None
|
||||
|
||||
output_size = int(weight.shape[0])
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
x_packed, x_scale_swizzled = scaled_fp4_quant(
|
||||
x_2d,
|
||||
input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
)
|
||||
output = blockscaled.mm_nvfp4(
|
||||
x_packed,
|
||||
x_scale_swizzled,
|
||||
weight,
|
||||
weight_scale_storage,
|
||||
alpha,
|
||||
out_dtype=x.dtype,
|
||||
)
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
return output.view(*output_shape)
|
||||
|
||||
|
||||
def warmup_b12x_nvfp4_linear(
|
||||
model: torch.nn.Module,
|
||||
*,
|
||||
max_tokens: int,
|
||||
cudagraph_capture_sizes: Iterable[int] = (),
|
||||
output_dtype: torch.dtype = torch.bfloat16,
|
||||
) -> int:
|
||||
if not current_platform.is_cuda():
|
||||
return 0
|
||||
if not current_platform.is_device_capability_family(120):
|
||||
return 0
|
||||
if output_dtype not in (torch.bfloat16, torch.float16):
|
||||
output_dtype = torch.bfloat16
|
||||
layer_map: dict[tuple[Any, ...], torch.nn.Module] = {}
|
||||
for layer in model.modules():
|
||||
if not getattr(layer, "b12x_nvfp4_linear", False):
|
||||
continue
|
||||
weight = layer.weight
|
||||
weight_scale = layer.weight_scale
|
||||
n, packed_k = map(int, weight.shape)
|
||||
signature = (
|
||||
weight.device,
|
||||
n,
|
||||
packed_k * 2,
|
||||
weight.dtype,
|
||||
weight_scale.dtype,
|
||||
output_dtype,
|
||||
)
|
||||
layer_map.setdefault(signature, layer)
|
||||
if not layer_map:
|
||||
return 0
|
||||
if _import_b12x_blockscaled() is None:
|
||||
return 0
|
||||
|
||||
token_counts = b12x_warmup_token_counts(
|
||||
max_tokens=max_tokens,
|
||||
cudagraph_capture_sizes=cudagraph_capture_sizes,
|
||||
)
|
||||
warmed = 0
|
||||
last_device: torch.device | None = None
|
||||
|
||||
with torch.inference_mode():
|
||||
for signature, layer in layer_map.items():
|
||||
weight = layer.weight
|
||||
weight_scale = layer.weight_scale
|
||||
k = signature[2]
|
||||
last_device = weight.device
|
||||
for tokens in token_counts:
|
||||
source = torch.zeros(
|
||||
(tokens, k),
|
||||
dtype=output_dtype,
|
||||
device=weight.device,
|
||||
)
|
||||
_apply_b12x_nvfp4_linear(
|
||||
source,
|
||||
weight,
|
||||
weight_scale,
|
||||
layer.input_global_scale_inv,
|
||||
layer.alpha,
|
||||
None,
|
||||
)
|
||||
warmed += 1
|
||||
|
||||
if warmed > 0 and last_device is not None and last_device.type == "cuda":
|
||||
torch.accelerator.synchronize(last_device)
|
||||
|
||||
return warmed
|
||||
|
||||
|
||||
class B12xNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
"""ModelOpt NVFP4 linear through the native B12X SM120 dense GEMM."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
del compute_capability
|
||||
if not current_platform.is_cuda():
|
||||
return False, "B12X NVFP4 kernels are only available on CUDA"
|
||||
if not current_platform.is_device_capability_family(120):
|
||||
return False, "B12X NVFP4 kernels require a Blackwell 12x device"
|
||||
blockscaled = _import_b12x_blockscaled()
|
||||
if blockscaled is None or _import_b12x_intrinsics() is None:
|
||||
return False, "Install the B12X backend with `pip install vllm[b12x]`"
|
||||
if not blockscaled.is_supported():
|
||||
return False, "b12x native NVFP4 GEMM is not supported"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
del config
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
intrinsics = _import_b12x_intrinsics()
|
||||
assert intrinsics is not None
|
||||
replace_parameter(
|
||||
layer,
|
||||
"weight_scale",
|
||||
intrinsics.swizzle_block_scale(layer.weight_scale.data),
|
||||
)
|
||||
layer.b12x_nvfp4_linear = True
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return _apply_b12x_nvfp4_linear(
|
||||
x,
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
layer.input_global_scale_inv,
|
||||
layer.alpha,
|
||||
bias,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["B12xNvFp4LinearKernel", "warmup_b12x_nvfp4_linear"]
|
||||
Reference in New Issue
Block a user