[Core][Kernel] Share persistent workspaces for Marlin and Humming (#57421)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: ErinYin <liang.yin@daocloud.io>
This commit is contained in:
@@ -10,7 +10,6 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
apply_gptq_marlin_linear,
|
||||
marlin_act_int8_process_scales,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_pad_dim,
|
||||
marlin_pad_qweight,
|
||||
marlin_pad_scales,
|
||||
@@ -91,11 +90,6 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
size_k, size_n = c.partition_weight_shape
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, c.group_size)
|
||||
|
||||
# Allocate marlin workspace, reusing existing storage on reload.
|
||||
self.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(self, "workspace", None)
|
||||
)
|
||||
|
||||
# Default name since marlin requires empty parameter for zp,
|
||||
# TODO: remove this requirement from marlin (allow optional tensors)
|
||||
if self.w_zp_name is None:
|
||||
@@ -197,7 +191,7 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
weight=w_q,
|
||||
weight_scale=w_s,
|
||||
weight_zp=w_zp, # type: ignore
|
||||
workspace=self.workspace,
|
||||
workspace=None,
|
||||
wtype=c.weight_type,
|
||||
input_size_per_partition=c.partition_weight_shape[0],
|
||||
output_size_per_partition=c.partition_weight_shape[1],
|
||||
|
||||
@@ -58,7 +58,7 @@ class MarlinMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
weight_global_scale=None,
|
||||
workspace=layer.workspace,
|
||||
workspace=None,
|
||||
size_n=layer.output_size_per_partition,
|
||||
size_k=layer.input_size_per_partition,
|
||||
bias=bias,
|
||||
|
||||
@@ -46,7 +46,7 @@ class MarlinMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
workspace=layer.workspace,
|
||||
workspace=None,
|
||||
size_n=layer.output_size_per_partition,
|
||||
size_k=layer.input_size_per_partition,
|
||||
bias=bias,
|
||||
|
||||
@@ -50,7 +50,7 @@ class MarlinNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
weight_global_scale=layer.weight_global_scale,
|
||||
workspace=layer.workspace,
|
||||
workspace=None,
|
||||
size_n=layer.output_size_per_partition,
|
||||
size_k=layer.input_size_per_partition,
|
||||
bias=bias,
|
||||
|
||||
@@ -93,7 +93,7 @@ class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=weight_scale,
|
||||
workspace=layer.workspace,
|
||||
workspace=None,
|
||||
size_n=layer.output_size_per_partition,
|
||||
size_k=layer.input_size_per_partition,
|
||||
input_dtype=self.marlin_input_dtype,
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
from vllm.model_executor.layers.fused_moe.moe_fused_mul_sum import moe_fused_mul_sum
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
MoEPermuteScratch,
|
||||
get_moe_permute_scratch,
|
||||
moe_permute,
|
||||
moe_permute_unpermute_supported,
|
||||
moe_unpermute,
|
||||
@@ -176,7 +177,6 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=num_dispatchers,
|
||||
)
|
||||
self._permute_scratch: dict[int, MoEPermuteScratch] = {}
|
||||
|
||||
def init_humming_moe(self):
|
||||
from vllm.utils.humming import get_heuristics_config
|
||||
@@ -259,24 +259,20 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
if not moe_permute_unpermute_supported():
|
||||
return None
|
||||
|
||||
scratch = self._permute_scratch.get(topk)
|
||||
if scratch is None:
|
||||
max_expanded_rows = (
|
||||
self.moe_config.max_num_tokens
|
||||
* self.moe_config.dp_size
|
||||
* self.moe_config.experts_per_token
|
||||
)
|
||||
scratch = MoEPermuteScratch(
|
||||
max_num_tokens=math.ceil(max_expanded_rows / topk),
|
||||
topk=topk,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
hidden_size=self.moe_config.hidden_dim,
|
||||
hidden_dtype=self.moe_config.in_dtype,
|
||||
)
|
||||
self._permute_scratch[topk] = scratch
|
||||
return scratch
|
||||
max_expanded_rows = (
|
||||
self.moe_config.max_num_tokens
|
||||
* self.moe_config.dp_size
|
||||
* self.moe_config.experts_per_token
|
||||
)
|
||||
return get_moe_permute_scratch(
|
||||
max_num_tokens=math.ceil(max_expanded_rows / topk),
|
||||
topk=topk,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
hidden_size=self.moe_config.hidden_dim,
|
||||
hidden_dtype=self.moe_config.in_dtype,
|
||||
)
|
||||
|
||||
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
|
||||
ctx = get_forward_context()
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
get_marlin_workspace,
|
||||
marlin_moe_intermediate_size,
|
||||
marlin_quant_input,
|
||||
)
|
||||
@@ -95,7 +95,7 @@ def _fused_marlin_moe(
|
||||
N = marlin_moe_intermediate_size(w1, w2)
|
||||
w13_num_shards = 2 if activation.is_gated else 1
|
||||
if workspace is None:
|
||||
workspace = marlin_make_workspace_new(hidden_states.device, 4)
|
||||
workspace = get_marlin_workspace(hidden_states.device)
|
||||
|
||||
if intermediate_cache13 is None:
|
||||
intermediate_cache13 = torch.empty(
|
||||
|
||||
@@ -5,6 +5,11 @@ from dataclasses import dataclass, field
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.worker.workspace import (
|
||||
current_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MoEPermuteScratch:
|
||||
@@ -102,6 +107,48 @@ class MoEPermuteScratch:
|
||||
return topk_ids_int32
|
||||
|
||||
|
||||
def get_moe_permute_scratch(
|
||||
*,
|
||||
max_num_tokens: int,
|
||||
topk: int,
|
||||
num_experts: int,
|
||||
num_local_experts: int,
|
||||
device: torch.device,
|
||||
hidden_size: int | None = None,
|
||||
hidden_dtype: torch.dtype | None = None,
|
||||
) -> MoEPermuteScratch:
|
||||
"""Share scratch across sequential layers in the current ubatch and lane.
|
||||
|
||||
Without a workspace manager, allocate new scratch for each call.
|
||||
"""
|
||||
|
||||
def create_scratch() -> MoEPermuteScratch:
|
||||
return MoEPermuteScratch(
|
||||
max_num_tokens=max_num_tokens,
|
||||
topk=topk,
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
device=device,
|
||||
hidden_size=hidden_size,
|
||||
hidden_dtype=hidden_dtype,
|
||||
)
|
||||
|
||||
if not is_workspace_manager_initialized():
|
||||
return create_scratch()
|
||||
|
||||
key = (
|
||||
MoEPermuteScratch,
|
||||
max_num_tokens,
|
||||
topk,
|
||||
num_experts,
|
||||
num_local_experts,
|
||||
device,
|
||||
hidden_size,
|
||||
hidden_dtype,
|
||||
)
|
||||
return current_workspace_manager().get_persistent_resource(key, create_scratch)
|
||||
|
||||
|
||||
def moe_permute(
|
||||
hidden_states: torch.Tensor,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
|
||||
@@ -48,7 +48,6 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_marlin_supports_layer,
|
||||
check_moe_marlin_supports_layer,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
verify_marlin_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -642,9 +641,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
layer.register_parameter("w2_qzeros", w2_qzeros)
|
||||
set_weight_attrs(w2_qzeros, extra_weight_attrs)
|
||||
|
||||
device = layer.w13_qweight.device
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_moe_backend,
|
||||
|
||||
@@ -16,7 +16,6 @@ from vllm.model_executor.kernels.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEExpertsModular,
|
||||
FusedMoEMethodBase,
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoeWeightScaleSupported,
|
||||
@@ -46,7 +45,6 @@ from vllm.model_executor.layers.quantization.utils.gptq_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_layer,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
marlin_repeat_scales_on_all_ranks,
|
||||
verify_marlin_supported,
|
||||
)
|
||||
@@ -601,12 +599,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
layer.register_parameter("w2_bias", w2_bias)
|
||||
set_weight_attrs(w2_bias, extra_weight_attrs)
|
||||
|
||||
if self.experts_cls is not None and issubclass(
|
||||
self.experts_cls, FusedMoEExpertsModular
|
||||
):
|
||||
device = layer.w13_qweight.device
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
def replace_or_register(name: str, val: torch.Tensor | None):
|
||||
if val is None:
|
||||
|
||||
-14
@@ -11,7 +11,6 @@ from compressed_tensors.quantization import (
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEExpertsModular,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
@@ -36,7 +35,6 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compress
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_config,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
@@ -466,18 +464,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
|
||||
)
|
||||
|
||||
# Marlin workspace — only needed for Marlin-family backends, not emulation.
|
||||
if (
|
||||
self.experts_cls is not None
|
||||
and issubclass(self.experts_cls, FusedMoEExpertsModular)
|
||||
and self.wna16_backend != WNA16MoEBackend.EMULATION
|
||||
):
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
layer.w13_weight_packed.device,
|
||||
4,
|
||||
existing=getattr(layer, "workspace", None),
|
||||
)
|
||||
|
||||
# Alias packed weights to w13_weight/w2_weight for the modular kernel interface
|
||||
layer.w13_weight = layer.w13_weight_packed
|
||||
layer.w2_weight = layer.w2_weight_packed
|
||||
|
||||
@@ -8,7 +8,6 @@ import numpy
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
@@ -23,6 +22,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from .quant_utils import pack_cols, unpack_cols
|
||||
|
||||
@@ -32,6 +32,7 @@ GPTQ_MARLIN_TILE = 16
|
||||
GPTQ_MARLIN_MIN_THREAD_N = 64
|
||||
GPTQ_MARLIN_MIN_THREAD_K = 128
|
||||
GPTQ_MARLIN_MAX_PARALLEL = 16
|
||||
MARLIN_MAX_BLOCKS_PER_SM = 4
|
||||
|
||||
MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
|
||||
|
||||
@@ -407,29 +408,96 @@ def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tenso
|
||||
def marlin_make_workspace_new(
|
||||
device: torch.device,
|
||||
max_blocks_per_sm: int = 1,
|
||||
existing: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# In the new marlin kernel, we use the num of threadblocks as workspace
|
||||
# size. The num of threadblocks is sms_count * max_blocks_per_sm.
|
||||
sms = num_compute_units(device.index)
|
||||
size = sms * max_blocks_per_sm
|
||||
# On weight reload, reuse the existing storage so the workspace address
|
||||
# captured by CUDA graphs stays valid.
|
||||
if existing is not None:
|
||||
if (
|
||||
existing.device != device
|
||||
or existing.dtype != torch.int
|
||||
or existing.numel() != size
|
||||
):
|
||||
raise ValueError(
|
||||
f"Existing Marlin workspace is incompatible "
|
||||
f"(device={existing.device}, dtype={existing.dtype}, "
|
||||
f"numel={existing.numel()}; expected device={device}, "
|
||||
f"dtype={torch.int}, numel={size}). Reload must reuse the "
|
||||
f"workspace storage captured by CUDA graphs."
|
||||
)
|
||||
return existing.zero_()
|
||||
return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False)
|
||||
return torch.zeros(sms * max_blocks_per_sm, dtype=torch.int, device=device)
|
||||
|
||||
|
||||
def get_marlin_workspace(device: torch.device) -> torch.Tensor:
|
||||
"""Get zero-initialized locks for the current stream, ubatch, and lane.
|
||||
|
||||
Marlin resets its locks after each call, so ordered calls can reuse them.
|
||||
"""
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.worker.workspace import (
|
||||
current_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
if not is_workspace_manager_initialized():
|
||||
return marlin_make_workspace_new(device, MARLIN_MAX_BLOCKS_PER_SM)
|
||||
|
||||
return current_workspace_manager().get_persistent(
|
||||
("marlin_locks", current_stream()),
|
||||
(num_compute_units(device.index) * MARLIN_MAX_BLOCKS_PER_SM,),
|
||||
torch.int,
|
||||
zero_init=True,
|
||||
)
|
||||
|
||||
|
||||
def _marlin_gemm(
|
||||
a: torch.Tensor,
|
||||
b_q_weight: torch.Tensor,
|
||||
b_bias: torch.Tensor | None,
|
||||
b_scales: torch.Tensor,
|
||||
a_scales: torch.Tensor | None,
|
||||
global_scale: torch.Tensor | None,
|
||||
b_zeros: torch.Tensor | None,
|
||||
workspace: torch.Tensor | None,
|
||||
b_q_type_id: int,
|
||||
size_m: int,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
use_atomic_add: bool = False,
|
||||
use_fp32_reduce: bool = False,
|
||||
is_zp_float: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if workspace is None:
|
||||
workspace = get_marlin_workspace(a.device)
|
||||
return torch.ops._C.marlin_gemm(
|
||||
a,
|
||||
None,
|
||||
b_q_weight,
|
||||
b_bias,
|
||||
b_scales,
|
||||
a_scales,
|
||||
global_scale,
|
||||
b_zeros,
|
||||
workspace,
|
||||
b_q_type_id,
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
is_zp_float,
|
||||
)
|
||||
|
||||
|
||||
def _marlin_gemm_fake(
|
||||
a: torch.Tensor,
|
||||
b_q_weight: torch.Tensor,
|
||||
b_bias: torch.Tensor | None,
|
||||
b_scales: torch.Tensor,
|
||||
a_scales: torch.Tensor | None,
|
||||
global_scale: torch.Tensor | None,
|
||||
b_zeros: torch.Tensor | None,
|
||||
workspace: torch.Tensor | None,
|
||||
b_q_type_id: int,
|
||||
size_m: int,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
use_atomic_add: bool = False,
|
||||
use_fp32_reduce: bool = False,
|
||||
is_zp_float: bool = False,
|
||||
) -> torch.Tensor:
|
||||
dtype = a.dtype if a.dtype in (torch.float16, torch.bfloat16) else b_scales.dtype
|
||||
return torch.empty((size_m, size_n), device=a.device, dtype=dtype)
|
||||
|
||||
|
||||
direct_register_custom_op("marlin_gemm", _marlin_gemm, fake_impl=_marlin_gemm_fake)
|
||||
|
||||
|
||||
def marlin_repeat_scales_on_all_ranks(group_size: int, is_row_parallel: bool) -> bool:
|
||||
@@ -686,7 +754,7 @@ def apply_gptq_marlin_linear(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_zp: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
workspace: torch.Tensor | None,
|
||||
wtype: ScalarType,
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
@@ -723,9 +791,8 @@ def apply_gptq_marlin_linear(
|
||||
|
||||
reshaped_x, a_scales = marlin_quant_input(reshaped_x, input_dtype)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
output = torch.ops.vllm.marlin_gemm(
|
||||
reshaped_x,
|
||||
None,
|
||||
weight,
|
||||
bias,
|
||||
weight_scale,
|
||||
@@ -733,7 +800,7 @@ def apply_gptq_marlin_linear(
|
||||
None,
|
||||
weight_zp,
|
||||
workspace,
|
||||
wtype,
|
||||
wtype.id,
|
||||
size_m=reshaped_x.shape[0],
|
||||
size_n=padded_n,
|
||||
size_k=padded_k,
|
||||
|
||||
@@ -10,7 +10,6 @@ from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
marlin_pad_dim,
|
||||
marlin_pad_qweight,
|
||||
marlin_pad_scales,
|
||||
@@ -161,7 +160,7 @@ def apply_fp4_marlin_linear(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_global_scale: torch.Tensor | None,
|
||||
workspace: torch.Tensor,
|
||||
workspace: torch.Tensor | None,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: torch.Tensor | None = None,
|
||||
@@ -196,9 +195,8 @@ def apply_fp4_marlin_linear(
|
||||
|
||||
inputs, a_scales = marlin_quant_input(inputs, torch.float8_e4m3fn)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
output = torch.ops.vllm.marlin_gemm(
|
||||
a=inputs,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
@@ -206,7 +204,7 @@ def apply_fp4_marlin_linear(
|
||||
global_scale=weight_global_scale,
|
||||
b_zeros=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float4_e2m1f,
|
||||
b_q_type_id=scalar_types.float4_e2m1f.id,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=padded_n,
|
||||
size_k=padded_k,
|
||||
@@ -237,13 +235,6 @@ def prepare_fp4_layer_for_marlin(
|
||||
|
||||
assert layer.weight.shape == (part_size_n, part_size_k // 2)
|
||||
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
qweight = layer.weight.view(torch.int32).T.contiguous()
|
||||
@@ -389,15 +380,9 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
return x
|
||||
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
|
||||
|
||||
device = w13.device
|
||||
param_dtype = layer.params_dtype
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
@@ -481,12 +466,7 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
k = layer.moe_config.hidden_dim
|
||||
n = layer.moe_config.intermediate_size_per_partition
|
||||
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
param_dtype = layer.params_dtype
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
# WEIGHT
|
||||
|
||||
@@ -9,7 +9,6 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_pad_dim,
|
||||
marlin_pad_qweight,
|
||||
@@ -50,7 +49,7 @@ def apply_fp8_marlin_linear(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
workspace: torch.Tensor | None,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: torch.Tensor | None,
|
||||
@@ -80,9 +79,8 @@ def apply_fp8_marlin_linear(
|
||||
# inputs, a_scales = marlin_quant_input(inputs, torch.float8_e4m3fn)
|
||||
raise RuntimeError("Marlin W8A8 is not supported.")
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
output = torch.ops.vllm.marlin_gemm(
|
||||
a=inputs,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
@@ -90,7 +88,7 @@ def apply_fp8_marlin_linear(
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
b_q_type_id=scalar_types.float8_e4m3fn.id,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=padded_n,
|
||||
size_k=padded_k,
|
||||
@@ -127,13 +125,6 @@ def prepare_fp8_layer_for_marlin(
|
||||
else:
|
||||
assert layer.weight.shape == (part_size_n, part_size_k)
|
||||
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
qweight = pack_fp8_to_int32(layer.weight, size_k_first)
|
||||
@@ -241,12 +232,7 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Shuffle weights and scales into marlin format.
|
||||
|
||||
Note that this function has the side effect of adding a `workspace`
|
||||
attribute to the layer. This `workspace` does not need to be
|
||||
registered as a Parameter as it is not used during weight reloading.
|
||||
"""
|
||||
"""Shuffle weights and scales into marlin format."""
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP8 computation but "
|
||||
"FP8 quantization is being used. Weight-only FP8 compression will "
|
||||
@@ -272,12 +258,6 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
w13_weight = _moe_pad_shard_rows(w13_weight, n, padded_n)
|
||||
w2_weight = _moe_pad_last(w2_weight, n, padded_n)
|
||||
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
def repack_weight(name: str, weight: torch.Tensor) -> torch.Tensor:
|
||||
@@ -396,7 +376,7 @@ def apply_mxfp8_marlin_linear(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
workspace: torch.Tensor | None,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: torch.Tensor | None = None,
|
||||
@@ -416,9 +396,8 @@ def apply_mxfp8_marlin_linear(
|
||||
dtype=input.dtype,
|
||||
)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
output = torch.ops.vllm.marlin_gemm(
|
||||
a=reshaped_x,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
@@ -426,7 +405,7 @@ def apply_mxfp8_marlin_linear(
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
b_q_type_id=scalar_types.float8_e4m3fn.id,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=padded_n,
|
||||
size_k=padded_k,
|
||||
@@ -451,13 +430,6 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
group_size = 32 # MX standard block size
|
||||
padded_n, padded_k = marlin_padded_nk(part_size_n, part_size_k, group_size)
|
||||
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT - repack FP8 weights to Marlin format
|
||||
qweight = pack_fp8_to_int32(layer.weight, size_k_first=False)
|
||||
qweight = qweight.T.contiguous()
|
||||
@@ -512,7 +484,7 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
"""Repack MXFP8 MoE weights and scales into Marlin kernel format.
|
||||
|
||||
Args:
|
||||
layer: MoE layer (used to read params_dtype and attach workspace).
|
||||
layer: MoE layer.
|
||||
w13: [E, 2*N, K] float8_e4m3fn weights.
|
||||
w2: [E, K, N] float8_e4m3fn weights.
|
||||
w13_scale: [E, 2*N, K//32] uint8 e8m0 scales.
|
||||
@@ -537,13 +509,8 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
n = padded_n
|
||||
w13_n = w13.shape[1]
|
||||
|
||||
device = w13.device
|
||||
param_dtype = torch.get_default_dtype()
|
||||
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
if "w13" in name:
|
||||
size_n, size_k = w13_n, k
|
||||
|
||||
Reference in New Issue
Block a user