[Core][Kernel] Share persistent workspaces for Marlin and Humming (#57421)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: ErinYin <liang.yin@daocloud.io>
This commit is contained in:
@@ -8,7 +8,6 @@ import numpy
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
@@ -23,6 +22,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from .quant_utils import pack_cols, unpack_cols
|
||||
|
||||
@@ -32,6 +32,7 @@ GPTQ_MARLIN_TILE = 16
|
||||
GPTQ_MARLIN_MIN_THREAD_N = 64
|
||||
GPTQ_MARLIN_MIN_THREAD_K = 128
|
||||
GPTQ_MARLIN_MAX_PARALLEL = 16
|
||||
MARLIN_MAX_BLOCKS_PER_SM = 4
|
||||
|
||||
MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
|
||||
|
||||
@@ -407,29 +408,96 @@ def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tenso
|
||||
def marlin_make_workspace_new(
|
||||
device: torch.device,
|
||||
max_blocks_per_sm: int = 1,
|
||||
existing: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# In the new marlin kernel, we use the num of threadblocks as workspace
|
||||
# size. The num of threadblocks is sms_count * max_blocks_per_sm.
|
||||
sms = num_compute_units(device.index)
|
||||
size = sms * max_blocks_per_sm
|
||||
# On weight reload, reuse the existing storage so the workspace address
|
||||
# captured by CUDA graphs stays valid.
|
||||
if existing is not None:
|
||||
if (
|
||||
existing.device != device
|
||||
or existing.dtype != torch.int
|
||||
or existing.numel() != size
|
||||
):
|
||||
raise ValueError(
|
||||
f"Existing Marlin workspace is incompatible "
|
||||
f"(device={existing.device}, dtype={existing.dtype}, "
|
||||
f"numel={existing.numel()}; expected device={device}, "
|
||||
f"dtype={torch.int}, numel={size}). Reload must reuse the "
|
||||
f"workspace storage captured by CUDA graphs."
|
||||
)
|
||||
return existing.zero_()
|
||||
return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False)
|
||||
return torch.zeros(sms * max_blocks_per_sm, dtype=torch.int, device=device)
|
||||
|
||||
|
||||
def get_marlin_workspace(device: torch.device) -> torch.Tensor:
|
||||
"""Get zero-initialized locks for the current stream, ubatch, and lane.
|
||||
|
||||
Marlin resets its locks after each call, so ordered calls can reuse them.
|
||||
"""
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.worker.workspace import (
|
||||
current_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
if not is_workspace_manager_initialized():
|
||||
return marlin_make_workspace_new(device, MARLIN_MAX_BLOCKS_PER_SM)
|
||||
|
||||
return current_workspace_manager().get_persistent(
|
||||
("marlin_locks", current_stream()),
|
||||
(num_compute_units(device.index) * MARLIN_MAX_BLOCKS_PER_SM,),
|
||||
torch.int,
|
||||
zero_init=True,
|
||||
)
|
||||
|
||||
|
||||
def _marlin_gemm(
|
||||
a: torch.Tensor,
|
||||
b_q_weight: torch.Tensor,
|
||||
b_bias: torch.Tensor | None,
|
||||
b_scales: torch.Tensor,
|
||||
a_scales: torch.Tensor | None,
|
||||
global_scale: torch.Tensor | None,
|
||||
b_zeros: torch.Tensor | None,
|
||||
workspace: torch.Tensor | None,
|
||||
b_q_type_id: int,
|
||||
size_m: int,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
use_atomic_add: bool = False,
|
||||
use_fp32_reduce: bool = False,
|
||||
is_zp_float: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if workspace is None:
|
||||
workspace = get_marlin_workspace(a.device)
|
||||
return torch.ops._C.marlin_gemm(
|
||||
a,
|
||||
None,
|
||||
b_q_weight,
|
||||
b_bias,
|
||||
b_scales,
|
||||
a_scales,
|
||||
global_scale,
|
||||
b_zeros,
|
||||
workspace,
|
||||
b_q_type_id,
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
is_zp_float,
|
||||
)
|
||||
|
||||
|
||||
def _marlin_gemm_fake(
|
||||
a: torch.Tensor,
|
||||
b_q_weight: torch.Tensor,
|
||||
b_bias: torch.Tensor | None,
|
||||
b_scales: torch.Tensor,
|
||||
a_scales: torch.Tensor | None,
|
||||
global_scale: torch.Tensor | None,
|
||||
b_zeros: torch.Tensor | None,
|
||||
workspace: torch.Tensor | None,
|
||||
b_q_type_id: int,
|
||||
size_m: int,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
use_atomic_add: bool = False,
|
||||
use_fp32_reduce: bool = False,
|
||||
is_zp_float: bool = False,
|
||||
) -> torch.Tensor:
|
||||
dtype = a.dtype if a.dtype in (torch.float16, torch.bfloat16) else b_scales.dtype
|
||||
return torch.empty((size_m, size_n), device=a.device, dtype=dtype)
|
||||
|
||||
|
||||
direct_register_custom_op("marlin_gemm", _marlin_gemm, fake_impl=_marlin_gemm_fake)
|
||||
|
||||
|
||||
def marlin_repeat_scales_on_all_ranks(group_size: int, is_row_parallel: bool) -> bool:
|
||||
@@ -686,7 +754,7 @@ def apply_gptq_marlin_linear(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_zp: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
workspace: torch.Tensor | None,
|
||||
wtype: ScalarType,
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
@@ -723,9 +791,8 @@ def apply_gptq_marlin_linear(
|
||||
|
||||
reshaped_x, a_scales = marlin_quant_input(reshaped_x, input_dtype)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
output = torch.ops.vllm.marlin_gemm(
|
||||
reshaped_x,
|
||||
None,
|
||||
weight,
|
||||
bias,
|
||||
weight_scale,
|
||||
@@ -733,7 +800,7 @@ def apply_gptq_marlin_linear(
|
||||
None,
|
||||
weight_zp,
|
||||
workspace,
|
||||
wtype,
|
||||
wtype.id,
|
||||
size_m=reshaped_x.shape[0],
|
||||
size_n=padded_n,
|
||||
size_k=padded_k,
|
||||
|
||||
@@ -10,7 +10,6 @@ from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
marlin_pad_dim,
|
||||
marlin_pad_qweight,
|
||||
marlin_pad_scales,
|
||||
@@ -161,7 +160,7 @@ def apply_fp4_marlin_linear(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_global_scale: torch.Tensor | None,
|
||||
workspace: torch.Tensor,
|
||||
workspace: torch.Tensor | None,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: torch.Tensor | None = None,
|
||||
@@ -196,9 +195,8 @@ def apply_fp4_marlin_linear(
|
||||
|
||||
inputs, a_scales = marlin_quant_input(inputs, torch.float8_e4m3fn)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
output = torch.ops.vllm.marlin_gemm(
|
||||
a=inputs,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
@@ -206,7 +204,7 @@ def apply_fp4_marlin_linear(
|
||||
global_scale=weight_global_scale,
|
||||
b_zeros=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float4_e2m1f,
|
||||
b_q_type_id=scalar_types.float4_e2m1f.id,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=padded_n,
|
||||
size_k=padded_k,
|
||||
@@ -237,13 +235,6 @@ def prepare_fp4_layer_for_marlin(
|
||||
|
||||
assert layer.weight.shape == (part_size_n, part_size_k // 2)
|
||||
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
qweight = layer.weight.view(torch.int32).T.contiguous()
|
||||
@@ -389,15 +380,9 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
return x
|
||||
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
|
||||
|
||||
device = w13.device
|
||||
param_dtype = layer.params_dtype
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
@@ -481,12 +466,7 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
k = layer.moe_config.hidden_dim
|
||||
n = layer.moe_config.intermediate_size_per_partition
|
||||
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
param_dtype = layer.params_dtype
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
# WEIGHT
|
||||
|
||||
@@ -9,7 +9,6 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_pad_dim,
|
||||
marlin_pad_qweight,
|
||||
@@ -50,7 +49,7 @@ def apply_fp8_marlin_linear(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
workspace: torch.Tensor | None,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: torch.Tensor | None,
|
||||
@@ -80,9 +79,8 @@ def apply_fp8_marlin_linear(
|
||||
# inputs, a_scales = marlin_quant_input(inputs, torch.float8_e4m3fn)
|
||||
raise RuntimeError("Marlin W8A8 is not supported.")
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
output = torch.ops.vllm.marlin_gemm(
|
||||
a=inputs,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
@@ -90,7 +88,7 @@ def apply_fp8_marlin_linear(
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
b_q_type_id=scalar_types.float8_e4m3fn.id,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=padded_n,
|
||||
size_k=padded_k,
|
||||
@@ -127,13 +125,6 @@ def prepare_fp8_layer_for_marlin(
|
||||
else:
|
||||
assert layer.weight.shape == (part_size_n, part_size_k)
|
||||
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
qweight = pack_fp8_to_int32(layer.weight, size_k_first)
|
||||
@@ -241,12 +232,7 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Shuffle weights and scales into marlin format.
|
||||
|
||||
Note that this function has the side effect of adding a `workspace`
|
||||
attribute to the layer. This `workspace` does not need to be
|
||||
registered as a Parameter as it is not used during weight reloading.
|
||||
"""
|
||||
"""Shuffle weights and scales into marlin format."""
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP8 computation but "
|
||||
"FP8 quantization is being used. Weight-only FP8 compression will "
|
||||
@@ -272,12 +258,6 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
w13_weight = _moe_pad_shard_rows(w13_weight, n, padded_n)
|
||||
w2_weight = _moe_pad_last(w2_weight, n, padded_n)
|
||||
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
def repack_weight(name: str, weight: torch.Tensor) -> torch.Tensor:
|
||||
@@ -396,7 +376,7 @@ def apply_mxfp8_marlin_linear(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
workspace: torch.Tensor | None,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: torch.Tensor | None = None,
|
||||
@@ -416,9 +396,8 @@ def apply_mxfp8_marlin_linear(
|
||||
dtype=input.dtype,
|
||||
)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
output = torch.ops.vllm.marlin_gemm(
|
||||
a=reshaped_x,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
@@ -426,7 +405,7 @@ def apply_mxfp8_marlin_linear(
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
b_q_type_id=scalar_types.float8_e4m3fn.id,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=padded_n,
|
||||
size_k=padded_k,
|
||||
@@ -451,13 +430,6 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
group_size = 32 # MX standard block size
|
||||
padded_n, padded_k = marlin_padded_nk(part_size_n, part_size_k, group_size)
|
||||
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT - repack FP8 weights to Marlin format
|
||||
qweight = pack_fp8_to_int32(layer.weight, size_k_first=False)
|
||||
qweight = qweight.T.contiguous()
|
||||
@@ -512,7 +484,7 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
"""Repack MXFP8 MoE weights and scales into Marlin kernel format.
|
||||
|
||||
Args:
|
||||
layer: MoE layer (used to read params_dtype and attach workspace).
|
||||
layer: MoE layer.
|
||||
w13: [E, 2*N, K] float8_e4m3fn weights.
|
||||
w2: [E, K, N] float8_e4m3fn weights.
|
||||
w13_scale: [E, 2*N, K//32] uint8 e8m0 scales.
|
||||
@@ -537,13 +509,8 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
n = padded_n
|
||||
w13_n = w13.shape[1]
|
||||
|
||||
device = w13.device
|
||||
param_dtype = torch.get_default_dtype()
|
||||
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
if "w13" in name:
|
||||
size_n, size_k = w13_n, k
|
||||
|
||||
Reference in New Issue
Block a user