[MoE Refactor][16/N] Apply Refactor to NVFP4 (#31692)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Pavani Majety <pmajety@nvidia.com>
This commit is contained in:
co-authored by
Robert Shaw
Michael Goin
Pavani Majety
parent
8dd2419fa9
commit
9f6dcb71ae
@@ -2,10 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
@@ -20,12 +23,23 @@ from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_prepare_finalize import ( # noqa: E501
|
||||
create_flashinfer_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
swizzle_blockscale,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
has_flashinfer_cutedsl_grouped_gemm_nt_masked,
|
||||
has_flashinfer_cutlass_fused_moe,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"is_flashinfer_fp4_cutlass_moe_available",
|
||||
"is_flashinfer_fp4_cutedsl_moe_available",
|
||||
@@ -273,10 +287,9 @@ def flashinfer_trtllm_fp4_moe(
|
||||
hidden_states_fp4, hidden_states_scale_linear_fp4 = x
|
||||
else:
|
||||
# hidden_states is the already quantized
|
||||
a1_gscale = layer.w13_input_scale_quant
|
||||
(hidden_states_fp4, hidden_states_scale_linear_fp4) = flashinfer.fp4_quantize(
|
||||
x,
|
||||
a1_gscale,
|
||||
layer.a1_gscale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
|
||||
@@ -369,10 +382,9 @@ def flashinfer_trtllm_fp4_routed_moe(
|
||||
hidden_states_fp4, hidden_states_scale_linear_fp4 = x
|
||||
else:
|
||||
# Quantize input to FP4
|
||||
a1_gscale = layer.w13_input_scale_quant
|
||||
(hidden_states_fp4, hidden_states_scale_linear_fp4) = flashinfer.fp4_quantize(
|
||||
x,
|
||||
a1_gscale,
|
||||
layer.a1_gscale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
|
||||
@@ -410,3 +422,93 @@ def flashinfer_trtllm_fp4_routed_moe(
|
||||
)[0]
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
backend: "NvFp4MoeBackend",
|
||||
layer: torch.nn.Module,
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w13_scale_2: torch.Tensor,
|
||||
a13_scale: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w2_scale_2: torch.Tensor,
|
||||
a2_scale: torch.Tensor,
|
||||
is_act_and_mul: bool,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
]:
|
||||
# Delayed import for circular dependency avoidance.
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
is_global_sf_supported_for_nvfp4_backend,
|
||||
)
|
||||
|
||||
assert backend in [
|
||||
NvFp4MoeBackend.VLLM_CUTLASS,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
]
|
||||
|
||||
# Reorder [w1, w3] to [w3, w1] for FI NVFP4 MoE kernels.
|
||||
if is_act_and_mul and backend in [
|
||||
NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
]:
|
||||
w13, w13_scale = reorder_w1w3_to_w3w1(w13, w13_scale)
|
||||
|
||||
# For some FI kernels, the input scales are shared by all experts.
|
||||
if is_global_sf_supported_for_nvfp4_backend(backend):
|
||||
num_experts = w13.shape[0]
|
||||
a13_scale = a13_scale.max().to(torch.float32).expand(num_experts)
|
||||
a2_scale = a2_scale.max().to(torch.float32).expand(num_experts)
|
||||
else:
|
||||
a13_scale = a13_scale.max(dim=1).values.to(torch.float32)
|
||||
|
||||
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
|
||||
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
|
||||
w13, w13_scale, w2, w2_scale = prepare_static_weights_for_trtllm_fp4_moe(
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w2.size(-2), # hidden_size
|
||||
w13.size(-2) // 2, # intermediate_size
|
||||
w13.size(0), # num_experts
|
||||
)
|
||||
|
||||
# We do not need to make this a parameter, because
|
||||
# it is not used during the weight (re)-loading process.
|
||||
layer.g1_scale_c = a13_scale * w13_scale_2 / a2_scale
|
||||
layer.a1_gscale = 1.0 / a13_scale
|
||||
layer.g1_alphas = a13_scale * w13_scale_2
|
||||
layer.g2_alphas = a2_scale * w2_scale_2
|
||||
else:
|
||||
# Swizzle the block scales for other FI NVFP4 MoE kernels.
|
||||
w13_scale = swizzle_blockscale(w13_scale)
|
||||
|
||||
# Apply padding if needed.
|
||||
pad_size = w13_scale.size(1) - w13.size(1)
|
||||
if pad_size > 0:
|
||||
if is_act_and_mul:
|
||||
raise NotImplementedError(
|
||||
"Intermediate size padding for w1 and w3, for %s "
|
||||
"NvFp4 backend, but this is not currently supported",
|
||||
backend.value,
|
||||
)
|
||||
w13 = torch.nn.functional.pad(w13, (0, 0, 0, pad_size))
|
||||
w2 = torch.nn.functional.pad(w2, (0, pad_size // 2, 0, 0))
|
||||
w2_scale = torch.nn.functional.pad(w2_scale, (0, pad_size // 16))
|
||||
|
||||
w2_scale = swizzle_blockscale(w2_scale)
|
||||
|
||||
return w13, w13_scale, w13_scale_2, a13_scale, w2, w2_scale, w2_scale_2, a2_scale
|
||||
|
||||
@@ -8,6 +8,7 @@ import vllm._custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
marlin_permute_bias,
|
||||
marlin_permute_scales,
|
||||
@@ -226,6 +227,106 @@ def prepare_fp4_layer_for_marlin(
|
||||
return
|
||||
|
||||
|
||||
def prepare_nvfp4_moe_layer_for_marlin(
|
||||
layer: torch.nn.Module,
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w13_scale_2: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w2_scale_2: torch.Tensor,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
||||
]:
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP4 computation but "
|
||||
"FP4 quantization is being used. Weight-only FP4 compression will "
|
||||
"be used leveraging the Marlin kernel. This may degrade "
|
||||
"performance for compute-heavy workloads."
|
||||
)
|
||||
|
||||
input_dtype = get_marlin_input_dtype(prefix="")
|
||||
if input_dtype is not None and input_dtype.itemsize == 1:
|
||||
raise RuntimeError("NVFP4 weight + INT8/FP8 activation is not supported.")
|
||||
|
||||
GROUP_SIZE = 16
|
||||
E = layer.num_experts
|
||||
K = layer.hidden_size
|
||||
N = layer.intermediate_size_per_partition
|
||||
|
||||
device = w13.device
|
||||
param_dtype = layer.params_dtype
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
tensor_list = []
|
||||
if "w13" in name:
|
||||
size_n, size_k = N * 2, K
|
||||
else:
|
||||
size_n, size_k = K, N
|
||||
|
||||
assert weight.shape == (E, size_n, size_k // 2)
|
||||
|
||||
for i in range(E):
|
||||
qweight = weight[i].view(torch.int32).T.contiguous()
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=4,
|
||||
is_a_8bit=is_a_8bit,
|
||||
)
|
||||
tensor_list.append(marlin_qweight)
|
||||
|
||||
return torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
|
||||
|
||||
w13 = repack_weight(w13, "w13")
|
||||
w2 = repack_weight(w2, "w2")
|
||||
|
||||
# WEIGHT SCALES
|
||||
# Permute scales
|
||||
def premute_scales(
|
||||
scales: torch.Tensor, g_scales: torch.Tensor, name: str
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
scales = scales.to(param_dtype)
|
||||
g_scales = g_scales.to(param_dtype)
|
||||
|
||||
tensor_list = []
|
||||
if "w13" in name:
|
||||
size_n, size_k = N * 2, K
|
||||
else:
|
||||
size_n, size_k = K, N
|
||||
|
||||
for i in range(E):
|
||||
scale = scales[i].T
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scale,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
group_size=GROUP_SIZE,
|
||||
is_a_8bit=is_a_8bit,
|
||||
)
|
||||
marlin_scales = nvfp4_marlin_process_scales(marlin_scales)
|
||||
tensor_list.append(marlin_scales)
|
||||
|
||||
scales = torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
|
||||
g_scales = nvfp4_marlin_process_global_scale(g_scales)
|
||||
return scales, g_scales
|
||||
|
||||
w13_scale, w13_scale_2 = premute_scales(w13_scale, w13_scale_2, "w13")
|
||||
w2_scale, w2_scale_2 = premute_scales(w2_scale, w2_scale_2, "w2")
|
||||
|
||||
return w13, w13_scale, w13_scale_2, w2, w2_scale, w2_scale_2
|
||||
|
||||
|
||||
def prepare_moe_fp4_layer_for_marlin(
|
||||
layer: torch.nn.Module, input_dtype: torch.dtype | None = None
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user