[Core][Kernel] Share persistent workspaces for Marlin and Humming (#57421)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: ErinYin <liang.yin@daocloud.io>
This commit is contained in:
Misha Goin
2026-09-20 17:28:33 +08:00
committed by GitHub
co-authored by ErinYin
parent 0e110f696d
commit e5fce7b56b
21 changed files with 598 additions and 211 deletions
@@ -48,7 +48,6 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supports_layer,
check_moe_marlin_supports_layer,
get_marlin_input_dtype,
marlin_make_workspace_new,
verify_marlin_supported,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -642,9 +641,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
layer.register_parameter("w2_qzeros", w2_qzeros)
set_weight_attrs(w2_qzeros, extra_weight_attrs)
device = layer.w13_qweight.device
layer.workspace = marlin_make_workspace_new(device, 4)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
@@ -16,7 +16,6 @@ from vllm.model_executor.kernels.linear import (
)
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEExpertsModular,
FusedMoEMethodBase,
FusedMoEQuantConfig,
FusedMoeWeightScaleSupported,
@@ -46,7 +45,6 @@ from vllm.model_executor.layers.quantization.utils.gptq_utils import (
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_layer,
get_marlin_input_dtype,
marlin_make_workspace_new,
marlin_repeat_scales_on_all_ranks,
verify_marlin_supported,
)
@@ -601,12 +599,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
if self.experts_cls is not None and issubclass(
self.experts_cls, FusedMoEExpertsModular
):
device = layer.w13_qweight.device
layer.workspace = marlin_make_workspace_new(device, 4)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
def replace_or_register(name: str, val: torch.Tensor | None):
if val is None:
@@ -11,7 +11,6 @@ from compressed_tensors.quantization import (
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEExpertsModular,
RoutedExperts,
SharedExperts,
)
@@ -36,7 +35,6 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compress
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_config,
get_marlin_input_dtype,
marlin_make_workspace_new,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
@@ -466,18 +464,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
# Marlin workspace — only needed for Marlin-family backends, not emulation.
if (
self.experts_cls is not None
and issubclass(self.experts_cls, FusedMoEExpertsModular)
and self.wna16_backend != WNA16MoEBackend.EMULATION
):
layer.workspace = marlin_make_workspace_new(
layer.w13_weight_packed.device,
4,
existing=getattr(layer, "workspace", None),
)
# Alias packed weights to w13_weight/w2_weight for the modular kernel interface
layer.w13_weight = layer.w13_weight_packed
layer.w2_weight = layer.w2_weight_packed
@@ -8,7 +8,6 @@ import numpy
import torch
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.distributed.utils import verify_group_size_divides_partition
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -23,6 +22,7 @@ from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType, scalar_types
from vllm.utils.math_utils import round_up
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import direct_register_custom_op
from .quant_utils import pack_cols, unpack_cols
@@ -32,6 +32,7 @@ GPTQ_MARLIN_TILE = 16
GPTQ_MARLIN_MIN_THREAD_N = 64
GPTQ_MARLIN_MIN_THREAD_K = 128
GPTQ_MARLIN_MAX_PARALLEL = 16
MARLIN_MAX_BLOCKS_PER_SM = 4
MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
@@ -407,29 +408,96 @@ def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tenso
def marlin_make_workspace_new(
device: torch.device,
max_blocks_per_sm: int = 1,
existing: torch.Tensor | None = None,
) -> torch.Tensor:
# In the new marlin kernel, we use the num of threadblocks as workspace
# size. The num of threadblocks is sms_count * max_blocks_per_sm.
sms = num_compute_units(device.index)
size = sms * max_blocks_per_sm
# On weight reload, reuse the existing storage so the workspace address
# captured by CUDA graphs stays valid.
if existing is not None:
if (
existing.device != device
or existing.dtype != torch.int
or existing.numel() != size
):
raise ValueError(
f"Existing Marlin workspace is incompatible "
f"(device={existing.device}, dtype={existing.dtype}, "
f"numel={existing.numel()}; expected device={device}, "
f"dtype={torch.int}, numel={size}). Reload must reuse the "
f"workspace storage captured by CUDA graphs."
)
return existing.zero_()
return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False)
return torch.zeros(sms * max_blocks_per_sm, dtype=torch.int, device=device)
def get_marlin_workspace(device: torch.device) -> torch.Tensor:
"""Get zero-initialized locks for the current stream, ubatch, and lane.
Marlin resets its locks after each call, so ordered calls can reuse them.
"""
from vllm.utils.torch_utils import current_stream
from vllm.v1.worker.workspace import (
current_workspace_manager,
is_workspace_manager_initialized,
)
if not is_workspace_manager_initialized():
return marlin_make_workspace_new(device, MARLIN_MAX_BLOCKS_PER_SM)
return current_workspace_manager().get_persistent(
("marlin_locks", current_stream()),
(num_compute_units(device.index) * MARLIN_MAX_BLOCKS_PER_SM,),
torch.int,
zero_init=True,
)
def _marlin_gemm(
a: torch.Tensor,
b_q_weight: torch.Tensor,
b_bias: torch.Tensor | None,
b_scales: torch.Tensor,
a_scales: torch.Tensor | None,
global_scale: torch.Tensor | None,
b_zeros: torch.Tensor | None,
workspace: torch.Tensor | None,
b_q_type_id: int,
size_m: int,
size_n: int,
size_k: int,
use_atomic_add: bool = False,
use_fp32_reduce: bool = False,
is_zp_float: bool = False,
) -> torch.Tensor:
if workspace is None:
workspace = get_marlin_workspace(a.device)
return torch.ops._C.marlin_gemm(
a,
None,
b_q_weight,
b_bias,
b_scales,
a_scales,
global_scale,
b_zeros,
workspace,
b_q_type_id,
size_m,
size_n,
size_k,
use_atomic_add,
use_fp32_reduce,
is_zp_float,
)
def _marlin_gemm_fake(
a: torch.Tensor,
b_q_weight: torch.Tensor,
b_bias: torch.Tensor | None,
b_scales: torch.Tensor,
a_scales: torch.Tensor | None,
global_scale: torch.Tensor | None,
b_zeros: torch.Tensor | None,
workspace: torch.Tensor | None,
b_q_type_id: int,
size_m: int,
size_n: int,
size_k: int,
use_atomic_add: bool = False,
use_fp32_reduce: bool = False,
is_zp_float: bool = False,
) -> torch.Tensor:
dtype = a.dtype if a.dtype in (torch.float16, torch.bfloat16) else b_scales.dtype
return torch.empty((size_m, size_n), device=a.device, dtype=dtype)
direct_register_custom_op("marlin_gemm", _marlin_gemm, fake_impl=_marlin_gemm_fake)
def marlin_repeat_scales_on_all_ranks(group_size: int, is_row_parallel: bool) -> bool:
@@ -686,7 +754,7 @@ def apply_gptq_marlin_linear(
weight: torch.Tensor,
weight_scale: torch.Tensor,
weight_zp: torch.Tensor,
workspace: torch.Tensor,
workspace: torch.Tensor | None,
wtype: ScalarType,
output_size_per_partition: int,
input_size_per_partition: int,
@@ -723,9 +791,8 @@ def apply_gptq_marlin_linear(
reshaped_x, a_scales = marlin_quant_input(reshaped_x, input_dtype)
output = ops.marlin_gemm(
output = torch.ops.vllm.marlin_gemm(
reshaped_x,
None,
weight,
bias,
weight_scale,
@@ -733,7 +800,7 @@ def apply_gptq_marlin_linear(
None,
weight_zp,
workspace,
wtype,
wtype.id,
size_m=reshaped_x.shape[0],
size_n=padded_n,
size_k=padded_k,
@@ -10,7 +10,6 @@ from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
USE_FP32_REDUCE_DEFAULT,
get_marlin_input_dtype,
marlin_make_workspace_new,
marlin_pad_dim,
marlin_pad_qweight,
marlin_pad_scales,
@@ -161,7 +160,7 @@ def apply_fp4_marlin_linear(
weight: torch.Tensor,
weight_scale: torch.Tensor,
weight_global_scale: torch.Tensor | None,
workspace: torch.Tensor,
workspace: torch.Tensor | None,
size_n: int,
size_k: int,
bias: torch.Tensor | None = None,
@@ -196,9 +195,8 @@ def apply_fp4_marlin_linear(
inputs, a_scales = marlin_quant_input(inputs, torch.float8_e4m3fn)
output = ops.marlin_gemm(
output = torch.ops.vllm.marlin_gemm(
a=inputs,
c=None,
b_q_weight=weight,
b_bias=bias,
b_scales=weight_scale,
@@ -206,7 +204,7 @@ def apply_fp4_marlin_linear(
global_scale=weight_global_scale,
b_zeros=None,
workspace=workspace,
b_q_type=scalar_types.float4_e2m1f,
b_q_type_id=scalar_types.float4_e2m1f.id,
size_m=reshaped_x.size(0),
size_n=padded_n,
size_k=padded_k,
@@ -237,13 +235,6 @@ def prepare_fp4_layer_for_marlin(
assert layer.weight.shape == (part_size_n, part_size_k // 2)
device = layer.weight.device
# WORKSPACE
layer.workspace = marlin_make_workspace_new(
device, existing=getattr(layer, "workspace", None)
)
# WEIGHT
# Repack weights to marlin format
qweight = layer.weight.view(torch.int32).T.contiguous()
@@ -389,15 +380,9 @@ def prepare_nvfp4_moe_layer_for_marlin(
return x
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
device = w13.device
param_dtype = layer.params_dtype
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
# WORKSPACE
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
# WEIGHT
# Repack weights to marlin format
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
@@ -481,12 +466,7 @@ def prepare_moe_fp4_layer_for_marlin(
k = layer.moe_config.hidden_dim
n = layer.moe_config.intermediate_size_per_partition
# WORKSPACE
device = layer.w13_weight.device
param_dtype = layer.params_dtype
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
# WEIGHT
@@ -9,7 +9,6 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
USE_FP32_REDUCE_DEFAULT,
get_marlin_input_dtype,
marlin_make_workspace_new,
marlin_moe_padded_intermediate,
marlin_pad_dim,
marlin_pad_qweight,
@@ -50,7 +49,7 @@ def apply_fp8_marlin_linear(
input: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
workspace: torch.Tensor,
workspace: torch.Tensor | None,
size_n: int,
size_k: int,
bias: torch.Tensor | None,
@@ -80,9 +79,8 @@ def apply_fp8_marlin_linear(
# inputs, a_scales = marlin_quant_input(inputs, torch.float8_e4m3fn)
raise RuntimeError("Marlin W8A8 is not supported.")
output = ops.marlin_gemm(
output = torch.ops.vllm.marlin_gemm(
a=inputs,
c=None,
b_q_weight=weight,
b_bias=bias,
b_scales=weight_scale,
@@ -90,7 +88,7 @@ def apply_fp8_marlin_linear(
global_scale=None,
b_zeros=None,
workspace=workspace,
b_q_type=scalar_types.float8_e4m3fn,
b_q_type_id=scalar_types.float8_e4m3fn.id,
size_m=reshaped_x.size(0),
size_n=padded_n,
size_k=padded_k,
@@ -127,13 +125,6 @@ def prepare_fp8_layer_for_marlin(
else:
assert layer.weight.shape == (part_size_n, part_size_k)
device = layer.weight.device
# WORKSPACE
layer.workspace = marlin_make_workspace_new(
device, existing=getattr(layer, "workspace", None)
)
# WEIGHT
# Repack weights to marlin format
qweight = pack_fp8_to_int32(layer.weight, size_k_first)
@@ -241,12 +232,7 @@ def prepare_fp8_moe_layer_for_marlin(
w13_weight_scale: torch.Tensor,
w2_weight_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Shuffle weights and scales into marlin format.
Note that this function has the side effect of adding a `workspace`
attribute to the layer. This `workspace` does not need to be
registered as a Parameter as it is not used during weight reloading.
"""
"""Shuffle weights and scales into marlin format."""
logger.warning_once(
"Your GPU does not have native support for FP8 computation but "
"FP8 quantization is being used. Weight-only FP8 compression will "
@@ -272,12 +258,6 @@ def prepare_fp8_moe_layer_for_marlin(
w13_weight = _moe_pad_shard_rows(w13_weight, n, padded_n)
w2_weight = _moe_pad_last(w2_weight, n, padded_n)
# WORKSPACE
device = layer.w13_weight.device
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
# WEIGHT
# Repack weights to marlin format
def repack_weight(name: str, weight: torch.Tensor) -> torch.Tensor:
@@ -396,7 +376,7 @@ def apply_mxfp8_marlin_linear(
input: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
workspace: torch.Tensor,
workspace: torch.Tensor | None,
size_n: int,
size_k: int,
bias: torch.Tensor | None = None,
@@ -416,9 +396,8 @@ def apply_mxfp8_marlin_linear(
dtype=input.dtype,
)
output = ops.marlin_gemm(
output = torch.ops.vllm.marlin_gemm(
a=reshaped_x,
c=None,
b_q_weight=weight,
b_bias=bias,
b_scales=weight_scale,
@@ -426,7 +405,7 @@ def apply_mxfp8_marlin_linear(
global_scale=None,
b_zeros=None,
workspace=workspace,
b_q_type=scalar_types.float8_e4m3fn,
b_q_type_id=scalar_types.float8_e4m3fn.id,
size_m=reshaped_x.size(0),
size_n=padded_n,
size_k=padded_k,
@@ -451,13 +430,6 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
group_size = 32 # MX standard block size
padded_n, padded_k = marlin_padded_nk(part_size_n, part_size_k, group_size)
device = layer.weight.device
# WORKSPACE
layer.workspace = marlin_make_workspace_new(
device, existing=getattr(layer, "workspace", None)
)
# WEIGHT - repack FP8 weights to Marlin format
qweight = pack_fp8_to_int32(layer.weight, size_k_first=False)
qweight = qweight.T.contiguous()
@@ -512,7 +484,7 @@ def prepare_mxfp8_moe_layer_for_marlin(
"""Repack MXFP8 MoE weights and scales into Marlin kernel format.
Args:
layer: MoE layer (used to read params_dtype and attach workspace).
layer: MoE layer.
w13: [E, 2*N, K] float8_e4m3fn weights.
w2: [E, K, N] float8_e4m3fn weights.
w13_scale: [E, 2*N, K//32] uint8 e8m0 scales.
@@ -537,13 +509,8 @@ def prepare_mxfp8_moe_layer_for_marlin(
n = padded_n
w13_n = w13.shape[1]
device = w13.device
param_dtype = torch.get_default_dtype()
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
if "w13" in name:
size_n, size_k = w13_n, k