[Kernel] Enable 8-bit weights in Fused Marlin MoE (#8032)
Co-authored-by: Dipika <dipikasikka1@gmail.com>
This commit is contained in:
@@ -7,18 +7,21 @@ import torch
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
fused_topk, moe_align_block_size, try_get_optimal_moe_config)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
|
||||
def single_marlin_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
g_idx: torch.Tensor,
|
||||
perm: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
override_config: Optional[Dict[str, Any]] = None) -> torch.Tensor:
|
||||
hidden_states: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
g_idx: torch.Tensor,
|
||||
perm: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
override_config: Optional[Dict[str, Any]] = None,
|
||||
num_bits: int = 8,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function computes the multiplication of hidden_states with expert
|
||||
weights used in Marlin MoE, using weights w and top-k gating mechanism.
|
||||
@@ -36,6 +39,7 @@ def single_marlin_moe(
|
||||
- renormalize (bool): If True, renormalize the top-k weights to sum to 1.
|
||||
- override_config (Optional[Dict[str, Any]]): Optional override
|
||||
for the kernel configuration.
|
||||
- num_bits (bool): The number of bits in expert weights quantization.
|
||||
|
||||
Returns:
|
||||
- torch.Tensor: The output tensor after applying the MoE layer.
|
||||
@@ -48,10 +52,11 @@ def single_marlin_moe(
|
||||
assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
|
||||
assert w.is_contiguous(), "Expert weights must be contiguous"
|
||||
assert hidden_states.dtype == torch.float16
|
||||
assert num_bits in [4, 8]
|
||||
|
||||
M, K = hidden_states.shape
|
||||
E = w.shape[0]
|
||||
N = w.shape[2] // 2
|
||||
N = w.shape[2] // (num_bits // 2)
|
||||
|
||||
topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
|
||||
renormalize)
|
||||
@@ -76,10 +81,13 @@ def single_marlin_moe(
|
||||
device="cuda",
|
||||
requires_grad=False)
|
||||
|
||||
scalar_type = (scalar_types.uint4b8
|
||||
if num_bits == 4 else scalar_types.uint8b128)
|
||||
|
||||
intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
|
||||
hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
|
||||
g_idx, perm, workspace, M, N, K, True, E, topk, block_size_m, True,
|
||||
False)
|
||||
g_idx, perm, workspace, scalar_type, M, N, K, True, E, topk,
|
||||
block_size_m, True, False)
|
||||
|
||||
return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
|
||||
|
||||
@@ -98,6 +106,7 @@ def fused_marlin_moe(
|
||||
override_config: Optional[Dict[str, Any]] = None,
|
||||
w1_scale: Optional[torch.Tensor] = None,
|
||||
w2_scale: Optional[torch.Tensor] = None,
|
||||
num_bits: int = 8,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function computes a Mixture of Experts (MoE) layer using two sets of
|
||||
@@ -122,6 +131,7 @@ def fused_marlin_moe(
|
||||
w1.
|
||||
- w2_scale (Optional[torch.Tensor]): Optional scale to be used for
|
||||
w2.
|
||||
- num_bits (bool): The number of bits in expert weights quantization.
|
||||
|
||||
Returns:
|
||||
- torch.Tensor: The output tensor after applying the MoE layer.
|
||||
@@ -131,13 +141,14 @@ def fused_marlin_moe(
|
||||
0], "Number of tokens mismatch"
|
||||
assert hidden_states.shape[
|
||||
1] == w1.shape[1] * 16, "Hidden size mismatch w1"
|
||||
assert hidden_states.shape[
|
||||
1] == w2.shape[2] // 2, "Hidden size mismatch w2"
|
||||
assert hidden_states.shape[1] == w2.shape[2] // (
|
||||
num_bits // 2), "Hidden size mismatch w2"
|
||||
assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
|
||||
assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
|
||||
assert w1.is_contiguous(), "Expert weights1 must be contiguous"
|
||||
assert w2.is_contiguous(), "Expert weights2 must be contiguous"
|
||||
assert hidden_states.dtype == torch.float16
|
||||
assert num_bits in [4, 8]
|
||||
|
||||
M, K = hidden_states.shape
|
||||
E = w1.shape[0]
|
||||
@@ -165,6 +176,9 @@ def fused_marlin_moe(
|
||||
device="cuda",
|
||||
requires_grad=False)
|
||||
|
||||
scalar_type = (scalar_types.uint4b8
|
||||
if num_bits == 4 else scalar_types.uint8b128)
|
||||
|
||||
intermediate_cache2 = torch.empty(
|
||||
(M * topk_ids.shape[1], N),
|
||||
device=hidden_states.device,
|
||||
@@ -181,6 +195,7 @@ def fused_marlin_moe(
|
||||
g_idx1,
|
||||
perm1,
|
||||
workspace,
|
||||
scalar_type,
|
||||
M,
|
||||
2 * N,
|
||||
K,
|
||||
@@ -204,6 +219,7 @@ def fused_marlin_moe(
|
||||
g_idx2,
|
||||
perm2,
|
||||
workspace,
|
||||
scalar_type,
|
||||
M,
|
||||
K,
|
||||
N,
|
||||
|
||||
@@ -445,7 +445,7 @@ def grouped_topk(hidden_states: torch.Tensor,
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
return topk_weights, topk_ids.to(torch.int32)
|
||||
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
def get_config_dtype_str(dtype: torch.dtype,
|
||||
|
||||
+6
-2
@@ -6,6 +6,8 @@ import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
WNA16_SUPPORTED_BITS)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
CompressionFormat)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
@@ -38,10 +40,11 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
if not (self.quant_config.quant_format
|
||||
== CompressionFormat.pack_quantized.value
|
||||
and self.num_bits == 4):
|
||||
and self.num_bits in WNA16_SUPPORTED_BITS):
|
||||
raise ValueError("For Fused MoE layers, only ",
|
||||
f"{CompressionFormat.pack_quantized.value} ",
|
||||
"is supported for 4 bits")
|
||||
"is supported for the following bits: ",
|
||||
f"{WNA16_SUPPORTED_BITS}")
|
||||
|
||||
def create_weights(self, layer: torch.nn.Module, num_experts: int,
|
||||
hidden_size: int, intermediate_size: int,
|
||||
@@ -292,4 +295,5 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
topk_ids,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
num_bits=self.num_bits,
|
||||
)
|
||||
|
||||
@@ -611,4 +611,5 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
|
||||
topk_ids,
|
||||
w1_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
num_bits=self.quant_config.quant_type.size_bits,
|
||||
).to(orig_dtype)
|
||||
|
||||
@@ -23,13 +23,7 @@ def get_model_architecture(
|
||||
architectures = getattr(model_config.hf_config, "architectures", [])
|
||||
# Special handling for quantized Mixtral.
|
||||
# FIXME(woosuk): This is a temporary hack.
|
||||
mixtral_supported = ["fp8", "compressed-tensors"]
|
||||
# for gptq_marlin, only run fused MoE for int4
|
||||
if model_config.quantization == "gptq_marlin":
|
||||
hf_quant_config = getattr(model_config.hf_config,
|
||||
"quantization_config", None)
|
||||
if hf_quant_config and hf_quant_config.get("bits") == 4:
|
||||
mixtral_supported.append("gptq_marlin")
|
||||
mixtral_supported = ["fp8", "compressed-tensors", "gptq_marlin"]
|
||||
|
||||
if (model_config.quantization is not None
|
||||
and model_config.quantization not in mixtral_supported
|
||||
|
||||
Reference in New Issue
Block a user