152 lines
5.2 KiB
Python
152 lines
5.2 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
|
|
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
|
MXFP8_BLOCK_SIZE,
|
|
MXFP8_SCALE_DTYPE,
|
|
MXFP8_VALUE_DTYPE,
|
|
)
|
|
from vllm.model_executor.utils import replace_parameter
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.b12x import B12xWarmupUnit, reuse_packed_weight_storage
|
|
from vllm.utils.b12x import (
|
|
get_b12x_mxfp8_linear as _import_b12x_mxfp8,
|
|
)
|
|
from vllm.utils.torch_utils import current_stream
|
|
|
|
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
|
|
|
|
|
def _apply_b12x_mxfp8_packed_linear(
|
|
layer: torch.nn.Module,
|
|
x: torch.Tensor,
|
|
bias: torch.Tensor | None,
|
|
) -> torch.Tensor:
|
|
packed_weight = layer.b12x_mxfp8_packed_weight
|
|
|
|
input_2d = x.reshape(-1, x.shape[-1]).contiguous()
|
|
output_shape = [*x.shape[:-1], int(packed_weight.out_features)]
|
|
|
|
mxfp8 = _import_b12x_mxfp8()
|
|
assert mxfp8 is not None
|
|
output = mxfp8.mm(
|
|
input_2d,
|
|
packed_weight,
|
|
bias=bias,
|
|
expected_m=max(1, int(input_2d.shape[0])),
|
|
)
|
|
return output.view(*output_shape)
|
|
|
|
|
|
class B12xMxfp8LinearKernel(Mxfp8LinearKernel):
|
|
"""ModelOpt MXFP8 linear through the native b12x SM120 dense GEMM path."""
|
|
|
|
@classmethod
|
|
def is_supported(
|
|
cls,
|
|
compute_capability: int | None = None,
|
|
) -> tuple[bool, str | None]:
|
|
del compute_capability
|
|
if not current_platform.is_cuda():
|
|
return False, "b12x MXFP8 kernels are only available on CUDA"
|
|
if not current_platform.is_device_capability_family(120):
|
|
return False, "b12x MXFP8 kernels require a Blackwell 12x device"
|
|
mxfp8 = _import_b12x_mxfp8()
|
|
if mxfp8 is None:
|
|
return False, "Install the B12X backend with `pip install vllm[b12x]`"
|
|
if not mxfp8.is_supported():
|
|
return False, "b12x.gemm.mxfp8_linear is not supported"
|
|
return True, None
|
|
|
|
@classmethod
|
|
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
|
del c
|
|
return True, None
|
|
|
|
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
|
weight = layer.weight.data
|
|
assert weight.dtype == MXFP8_VALUE_DTYPE, (
|
|
f"b12x MXFP8 requires {MXFP8_VALUE_DTYPE}, got {weight.dtype}"
|
|
)
|
|
assert weight.ndim == 2, f"b12x MXFP8 weight must be 2D, got {weight.ndim}D"
|
|
assert hasattr(layer, "weight_scale"), "b12x MXFP8 linear requires weight_scale"
|
|
|
|
out_features, in_features = map(int, weight.shape)
|
|
assert in_features % MXFP8_BLOCK_SIZE == 0, (
|
|
"b12x MXFP8 requires input features divisible by "
|
|
f"{MXFP8_BLOCK_SIZE}, got {in_features}"
|
|
)
|
|
weight_scale = layer.weight_scale.data
|
|
assert weight_scale.dtype == MXFP8_SCALE_DTYPE, (
|
|
f"b12x MXFP8 requires {MXFP8_SCALE_DTYPE} weight_scale, "
|
|
f"got {weight_scale.dtype}"
|
|
)
|
|
assert weight_scale.ndim == 2, (
|
|
f"b12x MXFP8 weight_scale must be 2D, got {weight_scale.ndim}D"
|
|
)
|
|
|
|
mxfp8 = _import_b12x_mxfp8()
|
|
assert mxfp8 is not None
|
|
scale_k = in_features // MXFP8_BLOCK_SIZE
|
|
packed_weight = mxfp8.pack_weight(
|
|
weight[:out_features, :in_features].detach(),
|
|
weight_scale[:out_features, :scale_k].detach(),
|
|
)
|
|
layer.b12x_mxfp8_packed_weight = reuse_packed_weight_storage(
|
|
getattr(layer, "b12x_mxfp8_packed_weight", None),
|
|
packed_weight,
|
|
)
|
|
replace_parameter(layer, "weight", weight.new_empty((0,)))
|
|
replace_parameter(layer, "weight_scale", weight_scale.new_empty((0,)))
|
|
layer.b12x_warmup_provider = self
|
|
|
|
def get_b12x_warmup_unit(
|
|
self,
|
|
layer: torch.nn.Module,
|
|
token_counts: tuple[int, ...],
|
|
output_dtype: torch.dtype,
|
|
) -> B12xWarmupUnit:
|
|
packed_weight = layer.b12x_mxfp8_packed_weight
|
|
device = torch.device(packed_weight.weight.values.device)
|
|
|
|
def compile() -> None:
|
|
mxfp8 = _import_b12x_mxfp8()
|
|
assert mxfp8 is not None
|
|
for tokens in token_counts:
|
|
source = torch.zeros(
|
|
(tokens, int(packed_weight.in_features)),
|
|
dtype=output_dtype,
|
|
device=device,
|
|
)
|
|
mxfp8.mm(
|
|
source,
|
|
packed_weight,
|
|
expected_m=max(1, int(tokens)),
|
|
stream=current_stream().cuda_stream,
|
|
)
|
|
|
|
return B12xWarmupUnit(
|
|
name="MXFP8",
|
|
key=(
|
|
type(self),
|
|
device,
|
|
int(packed_weight.in_features),
|
|
int(packed_weight.padded_in_features),
|
|
int(packed_weight.out_features),
|
|
output_dtype,
|
|
),
|
|
compile=compile,
|
|
)
|
|
|
|
def apply_weights(
|
|
self,
|
|
layer: torch.nn.Module,
|
|
x: torch.Tensor,
|
|
bias: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
return _apply_b12x_mxfp8_packed_linear(layer, x, bias)
|