[MoE Refactor] Combine CompressedTensorsWNA16MarlinMoEMethod with CompressedTensorsWNA16MoEMethod (#44570)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Dobrzyniewicz, Agata <agata.dobrzyniewicz@intel.com> Signed-off-by: Tushar Jain <tushar00jain@users.noreply.github.com> Signed-off-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: Agata Dobrzyniewicz <160237065+adobrzyn@users.noreply.github.com> Co-authored-by: Tushar Jain <8455015+tushar00jain@users.noreply.github.com> Co-authored-by: Tushar Jain <tushar00jain@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Claude
Taneem Ibrahim
Agata Dobrzyniewicz
Tushar Jain
Tushar Jain
Wentao Ye
Jiangyun Zhu
parent
0c4dc7c488
commit
454ea5b526
@@ -12,6 +12,7 @@ from vllm import _custom_ops as ops
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
@@ -351,10 +352,12 @@ def marlin_moe_padded_intermediate(intermediate_size: int, group_size: int = -1)
|
||||
return padded
|
||||
|
||||
|
||||
def check_moe_marlin_supports_layer(
|
||||
layer: RoutedExperts, group_size: int, allow_tile_padding: bool = False
|
||||
def check_moe_marlin_supports_config(
|
||||
config: FusedMoEConfig,
|
||||
group_size: int,
|
||||
allow_tile_padding: bool = False,
|
||||
) -> bool:
|
||||
"""Whether the fused MoE Marlin kernel supports ``layer``.
|
||||
"""Whether the fused MoE Marlin kernel supports ``config``.
|
||||
|
||||
Callers without act-order may pass ``allow_tile_padding=True``: a
|
||||
tile-misaligned intermediate size is then zero-padded to a valid thread
|
||||
@@ -364,15 +367,11 @@ def check_moe_marlin_supports_layer(
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
return False
|
||||
hidden_size = layer.hidden_size
|
||||
hidden_size = config.hidden_dim
|
||||
# The layer has not rounded intermediate_size yet; use the stable unpadded
|
||||
# size. gate-up needs n=2*intermediate % 128, down needs k=intermediate % 64.
|
||||
intermediate_size_per_partition = (
|
||||
layer.moe_config.intermediate_size_per_partition_unpadded
|
||||
)
|
||||
intermediate_size_per_partition = config.intermediate_size_per_partition_unpadded
|
||||
assert intermediate_size_per_partition is not None
|
||||
# apply_router_weight_on_input is not supported for moe marlin
|
||||
supports_router_weight = not layer.apply_router_weight_on_input
|
||||
|
||||
if allow_tile_padding:
|
||||
supports_shape = hidden_size % 128 == 0 and (
|
||||
@@ -384,7 +383,17 @@ def check_moe_marlin_supports_layer(
|
||||
and intermediate_size_per_partition % max(64, group_size) == 0
|
||||
)
|
||||
supports_group_size = group_size in [-1, 32, 64, 128]
|
||||
return supports_shape and supports_group_size and supports_router_weight
|
||||
return supports_shape and supports_group_size
|
||||
|
||||
|
||||
def check_moe_marlin_supports_layer(
|
||||
layer: RoutedExperts,
|
||||
group_size: int,
|
||||
allow_tile_padding: bool = False,
|
||||
) -> bool:
|
||||
return check_moe_marlin_supports_config(
|
||||
layer.moe_config, group_size, allow_tile_padding
|
||||
)
|
||||
|
||||
|
||||
def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tensor):
|
||||
|
||||
Reference in New Issue
Block a user