[Refactor] Simplify B12X linear kernels and warmup (#52368)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Misha Goin
2026-08-17 09:45:45 -07:00
committed by GitHub
parent 017e9f4448
commit 4ab5e5012a
15 changed files with 969 additions and 1617 deletions
@@ -3,9 +3,6 @@
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -13,9 +10,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.b12x import (
b12x_warmup_token_counts,
)
from vllm.utils.b12x import B12xWarmupUnit
from vllm.utils.b12x import (
get_b12x_blockscaled as _import_b12x_blockscaled,
)
@@ -51,73 +46,6 @@ def _apply_b12x_mxfp4_linear(
return output.view(*output_shape)
def warmup_b12x_mxfp4_linear(
model: torch.nn.Module,
*,
max_tokens: int,
cudagraph_capture_sizes: Iterable[int] = (),
output_dtype: torch.dtype = torch.bfloat16,
) -> int:
if not current_platform.is_cuda():
return 0
if not current_platform.is_device_capability_family(120):
return 0
if output_dtype not in (torch.bfloat16, torch.float16):
output_dtype = torch.bfloat16
layer_map: dict[tuple[Any, ...], torch.nn.Module] = {}
for layer in model.modules():
if not getattr(layer, "b12x_mxfp4_linear", False):
continue
weight = layer.weight
weight_scale = layer.weight_scale
n, packed_k = map(int, weight.shape)
signature = (
weight.device,
n,
packed_k * 2,
weight.dtype,
weight_scale.dtype,
output_dtype,
)
layer_map.setdefault(signature, layer)
if not layer_map:
return 0
if _import_b12x_blockscaled() is None:
return 0
token_counts = b12x_warmup_token_counts(
max_tokens=max_tokens,
cudagraph_capture_sizes=cudagraph_capture_sizes,
)
warmed = 0
last_device: torch.device | None = None
with torch.inference_mode():
for signature, layer in layer_map.items():
weight = layer.weight
weight_scale = layer.weight_scale
k = signature[2]
last_device = weight.device
for tokens in token_counts:
source = torch.zeros(
(tokens, k),
dtype=output_dtype,
device=weight.device,
)
_apply_b12x_mxfp4_linear(
source,
weight,
weight_scale,
None,
)
warmed += 1
if warmed > 0 and last_device is not None and last_device.type == "cuda":
torch.accelerator.synchronize(last_device)
return warmed
class B12xMxFp4LinearKernel(MxFp4LinearKernel):
"""MXFP4 linear through the native B12X SM120 dense GEMM."""
@@ -151,7 +79,39 @@ class B12xMxFp4LinearKernel(MxFp4LinearKernel):
"weight_scale",
intrinsics.swizzle_block_scale(layer.weight_scale.data),
)
layer.b12x_mxfp4_linear = True
layer.b12x_warmup_provider = self
def get_b12x_warmup_unit(
self,
layer: torch.nn.Module,
token_counts: tuple[int, ...],
output_dtype: torch.dtype,
) -> B12xWarmupUnit:
weight = layer.weight
weight_scale = layer.weight_scale
n, packed_k = map(int, weight.shape)
k = packed_k * 2
def compile() -> None:
for tokens in token_counts:
source = torch.zeros(
(tokens, k), dtype=output_dtype, device=weight.device
)
_apply_b12x_mxfp4_linear(source, weight, weight_scale, None)
return B12xWarmupUnit(
name="MXFP4",
key=(
type(self),
weight.device,
n,
k,
weight.dtype,
weight_scale.dtype,
output_dtype,
),
compile=compile,
)
def apply_weights(
self,
@@ -167,4 +127,4 @@ class B12xMxFp4LinearKernel(MxFp4LinearKernel):
)
__all__ = ["B12xMxFp4LinearKernel", "warmup_b12x_mxfp4_linear"]
__all__ = ["B12xMxFp4LinearKernel"]