[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel (#39129)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
+4
-19
@@ -6,15 +6,10 @@ import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
NvFp4LinearBackend,
|
||||
apply_nvfp4_linear,
|
||||
convert_to_nvfp4_linear_kernel_format,
|
||||
select_nvfp4_linear_backend,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
@@ -29,13 +24,9 @@ __all__ = ["CompressedTensorsW4A4Fp4"]
|
||||
|
||||
class CompressedTensorsW4A4Fp4(CompressedTensorsScheme):
|
||||
def __init__(self):
|
||||
self.backend = select_nvfp4_linear_backend()
|
||||
self.kernel = init_nvfp4_linear_kernel()
|
||||
self.group_size = 16
|
||||
|
||||
self.swizzle = None
|
||||
if self.backend == NvFp4LinearBackend.EMULATION:
|
||||
self.swizzle = False
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 75
|
||||
@@ -130,7 +121,7 @@ class CompressedTensorsW4A4Fp4(CompressedTensorsScheme):
|
||||
)
|
||||
|
||||
# Convert layer to NVFP4 linear kernel format
|
||||
convert_to_nvfp4_linear_kernel_format(self.backend, layer)
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -138,10 +129,4 @@ class CompressedTensorsW4A4Fp4(CompressedTensorsScheme):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return apply_nvfp4_linear(
|
||||
backend=self.backend,
|
||||
layer=layer,
|
||||
x=x,
|
||||
bias=bias,
|
||||
swizzle=self.swizzle,
|
||||
)
|
||||
return self.kernel.apply_weights(layer=layer, x=x, bias=bias)
|
||||
|
||||
@@ -10,7 +10,10 @@ from torch.nn.parameter import Parameter
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
init_fp8_linear_kernel,
|
||||
init_nvfp4_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
@@ -70,12 +73,6 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
Mxfp8LinearOp,
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
NvFp4LinearBackend,
|
||||
apply_nvfp4_linear,
|
||||
convert_to_nvfp4_linear_kernel_format,
|
||||
select_nvfp4_linear_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
create_fp8_quant_key,
|
||||
@@ -1090,11 +1087,7 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
|
||||
def __init__(self, quant_config: ModelOptNvFp4Config) -> None:
|
||||
self.quant_config = quant_config
|
||||
self.marlin_input_dtype = None
|
||||
self.backend = select_nvfp4_linear_backend()
|
||||
|
||||
self.swizzle = None
|
||||
if self.backend == NvFp4LinearBackend.EMULATION:
|
||||
self.swizzle = False
|
||||
self.kernel = init_nvfp4_linear_kernel()
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -1201,7 +1194,7 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
|
||||
)
|
||||
|
||||
# Convert layer to NVFP4 linear kernel format
|
||||
convert_to_nvfp4_linear_kernel_format(self.backend, layer)
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -1209,13 +1202,7 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return apply_nvfp4_linear(
|
||||
backend=self.backend,
|
||||
layer=layer,
|
||||
x=x,
|
||||
bias=bias,
|
||||
swizzle=self.swizzle,
|
||||
)
|
||||
return self.kernel.apply_weights(layer=layer, x=x, bias=bias)
|
||||
|
||||
|
||||
class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
|
||||
@@ -1,352 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from enum import Enum
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._custom_ops import (
|
||||
cutlass_scaled_fp4_mm,
|
||||
cutlass_scaled_mm_supports_fp4,
|
||||
scaled_fp4_quant,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
apply_fp4_marlin_linear,
|
||||
is_fp4_marlin_supported,
|
||||
prepare_fp4_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
kE2M1ToFloat_handle,
|
||||
run_nvfp4_emulations,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import flashinfer_scaled_fp4_mm, has_flashinfer
|
||||
from vllm.utils.import_utils import has_fbgemm_gpu
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# NOTE: This is ordered by preferred backend.
|
||||
# Example: if both are available, FLASHINFER_CUTLASS is preferred to VLLM_CUTLASS.
|
||||
class NvFp4LinearBackend(Enum):
|
||||
FLASHINFER_CUTLASS = "flashinfer-cutlass"
|
||||
VLLM_CUTLASS = "cutlass"
|
||||
MARLIN = "marlin"
|
||||
FLASHINFER_TRTLLM = "flashinfer-trtllm"
|
||||
FLASHINFER_CUDNN = "flashinfer-cudnn"
|
||||
FBGEMM = "fbgemm"
|
||||
EMULATION = "emulation"
|
||||
|
||||
|
||||
NVFP4_LINEAR_BACKENDS = list(NvFp4LinearBackend)
|
||||
|
||||
|
||||
def is_backend_supported(backend: NvFp4LinearBackend) -> tuple[bool, str | None]:
|
||||
reason = None
|
||||
supported = True
|
||||
|
||||
if backend == NvFp4LinearBackend.FLASHINFER_CUTLASS:
|
||||
# cutlass_fp4_supported() checks that the vLLM NVFP4 kernels (both
|
||||
# quantization and GEMM) were compiled for the current SM version.
|
||||
# FlashInfer backends still rely on the vLLM quantization kernels,
|
||||
# so we gate them on the same check.
|
||||
supported = (
|
||||
cutlass_fp4_supported()
|
||||
and current_platform.has_device_capability(100)
|
||||
and has_flashinfer()
|
||||
)
|
||||
|
||||
if not supported:
|
||||
reason = "FlashInfer is required, >=sm_100 is required"
|
||||
elif backend == NvFp4LinearBackend.VLLM_CUTLASS:
|
||||
supported = cutlass_fp4_supported()
|
||||
if not supported:
|
||||
reason = "Cutlass is required"
|
||||
elif backend == NvFp4LinearBackend.MARLIN:
|
||||
supported = is_fp4_marlin_supported()
|
||||
if not supported:
|
||||
reason = "Marlin is required"
|
||||
elif backend in [
|
||||
NvFp4LinearBackend.FLASHINFER_TRTLLM,
|
||||
NvFp4LinearBackend.FLASHINFER_CUDNN,
|
||||
]:
|
||||
supported = has_flashinfer()
|
||||
if not supported:
|
||||
reason = "FlashInfer is required"
|
||||
elif backend == NvFp4LinearBackend.FBGEMM:
|
||||
supported = has_fbgemm_gpu()
|
||||
if not supported:
|
||||
reason = "fbgemm_gpu is required"
|
||||
elif backend == NvFp4LinearBackend.EMULATION:
|
||||
# e.g. AMD Instinct does not support native NVFP4.
|
||||
unsupported_reasons = {}
|
||||
for other_backend in NVFP4_LINEAR_BACKENDS:
|
||||
if other_backend == NvFp4LinearBackend.EMULATION:
|
||||
continue
|
||||
other_supported, other_reason = is_backend_supported(other_backend)
|
||||
if not other_supported:
|
||||
unsupported_reasons[other_backend] = other_reason
|
||||
|
||||
if unsupported_reasons:
|
||||
unsupported_reasons_str = "\n - ".join(
|
||||
[f"{b.value}: {r}" for b, r in unsupported_reasons.items()]
|
||||
)
|
||||
logger.warning_once(
|
||||
f"NVFP4 linear falling back to the slow and unoptimized "
|
||||
f"backend=NvFp4LinearBackend.EMULATION as no optimized backend is "
|
||||
f"available (unavailable reasons:\n - {unsupported_reasons_str}\n). "
|
||||
"In case you expect one of these backend to be used, "
|
||||
"please verify your environment."
|
||||
)
|
||||
|
||||
return supported, reason
|
||||
|
||||
|
||||
def select_nvfp4_linear_backend() -> NvFp4LinearBackend:
|
||||
"""
|
||||
Select the best available NVFP4 GEMM backend based on environment
|
||||
configuration and platform capabilities.
|
||||
"""
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
logger.info_once(
|
||||
"VLLM_BATCH_INVARIANT forces NVFP4 linear to use the emulation "
|
||||
"backend for deterministic execution."
|
||||
)
|
||||
return NvFp4LinearBackend.EMULATION
|
||||
|
||||
selected_backend: NvFp4LinearBackend | None = None
|
||||
|
||||
if envs.VLLM_USE_FBGEMM:
|
||||
try:
|
||||
import fbgemm_gpu # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"Backend fbgemm requires fbgemm.f4f4bf16 operator, "
|
||||
"Please install with: pip install fbgemm-gpu-genai"
|
||||
) from exc
|
||||
selected_backend = NvFp4LinearBackend.FBGEMM
|
||||
elif envs.VLLM_USE_NVFP4_CT_EMULATIONS:
|
||||
selected_backend = NvFp4LinearBackend.EMULATION
|
||||
elif envs.VLLM_NVFP4_GEMM_BACKEND is None:
|
||||
for backend in NVFP4_LINEAR_BACKENDS:
|
||||
supported, reason = is_backend_supported(backend)
|
||||
if supported:
|
||||
selected_backend = backend
|
||||
break
|
||||
else:
|
||||
selected_backend = NvFp4LinearBackend(envs.VLLM_NVFP4_GEMM_BACKEND)
|
||||
|
||||
if selected_backend is None:
|
||||
raise ValueError(
|
||||
f"No NVFP4 GEMM backend selected, "
|
||||
f"available backends: {NVFP4_LINEAR_BACKENDS}"
|
||||
)
|
||||
|
||||
supported, reason = is_backend_supported(selected_backend)
|
||||
|
||||
if not supported:
|
||||
raise ValueError(
|
||||
f"The selected backend={selected_backend} is not supported in current "
|
||||
f"environment. Reason: {reason}. Current environment: "
|
||||
f"{envs.VLLM_USE_FBGEMM=}, {envs.VLLM_USE_NVFP4_CT_EMULATIONS=}, "
|
||||
f"{envs.VLLM_NVFP4_GEMM_BACKEND}."
|
||||
)
|
||||
|
||||
logger.info_once(f"Using {selected_backend} for NVFP4 GEMM")
|
||||
return selected_backend
|
||||
|
||||
|
||||
def prepare_weights_for_nvfp4_flashinfer_trtllm(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Prepare weights and scales for FlashInfer TRTLLM FP4 GEMM."""
|
||||
from flashinfer import shuffle_matrix_a, shuffle_matrix_sf_a
|
||||
|
||||
epilogue_tile_m = 128
|
||||
shuffled_weight = shuffle_matrix_a(weight.view(torch.uint8), epilogue_tile_m)
|
||||
shuffled_weight_scale = (
|
||||
shuffle_matrix_sf_a(weight_scale.view(torch.uint8), epilogue_tile_m)
|
||||
.reshape(weight_scale.shape)
|
||||
.view(torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
return shuffled_weight, shuffled_weight_scale
|
||||
|
||||
|
||||
def prepare_weights_for_nvfp4_cutlass(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, int]:
|
||||
"""
|
||||
Prepare weights and scales for CUTLASS/FlashInfer-CUTLASS FP4 GEMM.
|
||||
This involves padding weights for alignment (K and N divisible by 32)
|
||||
"""
|
||||
swizzled_weight_scale = swizzle_blockscale(weight_scale)
|
||||
padded_weight, weights_padding_cols = pad_nvfp4_weight_for_cutlass(weight)
|
||||
return padded_weight, swizzled_weight_scale, weights_padding_cols
|
||||
|
||||
|
||||
def prepare_weights_for_nvfp4_fbgemm(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Prepare weights and scales for FBGEMM FP4 GEMM."""
|
||||
swizzled_weight_scale = swizzle_blockscale(weight_scale)
|
||||
swizzled_weight_scale = swizzled_weight_scale.view(-1).view(torch.uint8)
|
||||
return weight, swizzled_weight_scale
|
||||
|
||||
|
||||
def convert_to_nvfp4_linear_kernel_format(
|
||||
backend: NvFp4LinearBackend,
|
||||
layer: torch.nn.Module,
|
||||
) -> None:
|
||||
"""Convert layer to NVFP4 linear kernel format."""
|
||||
|
||||
assert layer.weight_scale.dtype == torch.float8_e4m3fn, (
|
||||
"Weight Block scale must be represented as FP8-E4M3"
|
||||
)
|
||||
|
||||
# Default to no padding
|
||||
layer.weights_padding_cols = 0
|
||||
|
||||
if backend == NvFp4LinearBackend.MARLIN:
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP4 computation but "
|
||||
"FP4 quantization is being used. Weight-only FP4 compression "
|
||||
"will be used leveraging the Marlin kernel. This may degrade "
|
||||
"performance for compute-heavy workloads."
|
||||
)
|
||||
prepare_fp4_layer_for_marlin(layer)
|
||||
elif backend == NvFp4LinearBackend.FLASHINFER_TRTLLM:
|
||||
weight, weight_scale = prepare_weights_for_nvfp4_flashinfer_trtllm(
|
||||
layer.weight.data, layer.weight_scale.data
|
||||
)
|
||||
layer.weight = torch.nn.Parameter(weight, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
|
||||
elif backend == NvFp4LinearBackend.FBGEMM:
|
||||
weight, weight_scale = prepare_weights_for_nvfp4_fbgemm(
|
||||
layer.weight.data, layer.weight_scale.data
|
||||
)
|
||||
layer.weight = torch.nn.Parameter(weight, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
|
||||
elif backend in (
|
||||
NvFp4LinearBackend.VLLM_CUTLASS,
|
||||
NvFp4LinearBackend.FLASHINFER_CUTLASS,
|
||||
NvFp4LinearBackend.FLASHINFER_CUDNN,
|
||||
):
|
||||
weight, weight_scale, weights_padding_cols = prepare_weights_for_nvfp4_cutlass(
|
||||
layer.weight.data, layer.weight_scale.data
|
||||
)
|
||||
layer.weight = torch.nn.Parameter(weight, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
|
||||
layer.weights_padding_cols = weights_padding_cols
|
||||
elif backend == NvFp4LinearBackend.EMULATION:
|
||||
# We can not call `.to(device)` during cuda graph capture - do it here instead.
|
||||
# (operation not permitted when stream is capturing)
|
||||
kE2M1ToFloat_handle.val = kE2M1ToFloat_handle.val.to(layer.weight.device)
|
||||
|
||||
|
||||
def apply_nvfp4_linear(
|
||||
backend: NvFp4LinearBackend,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
swizzle: bool | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply NVFP4 linear transformation using the specified backend.
|
||||
"""
|
||||
weight = layer.weight
|
||||
weight_scale = layer.weight_scale
|
||||
weight_global_scale = layer.weight_global_scale
|
||||
input_global_scale_inv = layer.input_global_scale_inv
|
||||
alpha = layer.alpha
|
||||
output_size = layer.output_size_per_partition
|
||||
input_size = layer.input_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
|
||||
if backend == NvFp4LinearBackend.MARLIN:
|
||||
return apply_fp4_marlin_linear(
|
||||
input=x,
|
||||
weight=weight,
|
||||
weight_scale=weight_scale,
|
||||
weight_global_scale=weight_global_scale,
|
||||
workspace=layer.workspace,
|
||||
size_n=output_size,
|
||||
size_k=input_size,
|
||||
bias=bias,
|
||||
)
|
||||
elif backend == NvFp4LinearBackend.EMULATION:
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
out = run_nvfp4_emulations(
|
||||
x=x_2d,
|
||||
input_global_scale=input_global_scale_inv,
|
||||
weight=weight,
|
||||
weight_scale_swizzled=weight_scale,
|
||||
weight_global_scale=weight_global_scale,
|
||||
swizzle=swizzle,
|
||||
)
|
||||
out = out[:, :output_size]
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
return out.view(*output_shape)
|
||||
|
||||
# Quantize BF16 or FP16 to (FP4 and interleaved block scale)
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x, input_global_scale_inv, is_sf_swizzled_layout=True, backend=backend.value
|
||||
)
|
||||
|
||||
# Validate dtypes
|
||||
assert x_fp4.dtype == torch.uint8
|
||||
assert weight.dtype == torch.uint8
|
||||
assert x_blockscale.dtype == torch.float8_e4m3fn
|
||||
# weight_scale is fp8 for most backends, but uint8 for fbgemm
|
||||
assert weight_scale.dtype in (torch.float8_e4m3fn, torch.uint8)
|
||||
assert alpha.dtype == torch.float32
|
||||
|
||||
# Pad activations to match weight K-dimension padding
|
||||
weights_padding_cols = getattr(layer, "weights_padding_cols", 0)
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(x_fp4, weights_padding_cols)
|
||||
|
||||
# Prepare args for the matmul
|
||||
mm_args = (
|
||||
x_fp4,
|
||||
weight,
|
||||
x_blockscale,
|
||||
weight_scale,
|
||||
alpha,
|
||||
output_dtype,
|
||||
)
|
||||
|
||||
# Call the appropriate backend
|
||||
if backend.value.startswith("flashinfer-"):
|
||||
backend_name = backend.value[len("flashinfer-") :]
|
||||
out = flashinfer_scaled_fp4_mm(*mm_args, backend=backend_name)
|
||||
elif backend == NvFp4LinearBackend.FBGEMM:
|
||||
out = torch.ops.fbgemm.f4f4bf16(
|
||||
x_fp4,
|
||||
weight,
|
||||
x_blockscale.view(-1).view(torch.uint8),
|
||||
weight_scale,
|
||||
alpha,
|
||||
use_mx=False,
|
||||
).to(output_dtype)
|
||||
else:
|
||||
assert backend == NvFp4LinearBackend.VLLM_CUTLASS
|
||||
out = cutlass_scaled_fp4_mm(*mm_args)
|
||||
|
||||
# Slice output to remove N-dimension padding
|
||||
out = slice_nvfp4_output(out, output_size)
|
||||
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
|
||||
return out.view(*output_shape)
|
||||
|
||||
|
||||
def swizzle_blockscale(scale: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user