[Misc] Add unit tests for MoE ModularKernel combinations + Profiling utility (#20449)
Signed-off-by: Varun Sundar Rabindranath <vsundarr@redhat.com> Co-authored-by: Varun Sundar Rabindranath <vsundarr@redhat.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
parent
6fb162447b
commit
53fa457391
@@ -37,7 +37,6 @@ class BatchedTritonOrDeepGemmExperts(mk.FusedMoEPermuteExpertsUnpermute):
|
||||
block_shape=block_shape,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
))
|
||||
self.allow_deep_gemm = allow_deep_gemm
|
||||
|
||||
self.batched_triton_experts = BatchedTritonExperts(
|
||||
max_num_tokens=max_num_tokens,
|
||||
|
||||
@@ -81,13 +81,12 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
params_dtype: torch.dtype, **extra_weight_attrs):
|
||||
raise NotImplementedError
|
||||
|
||||
def init_prepare_finalize(self, moe: FusedMoEConfig,
|
||||
quant_config: Optional[QuantizationConfig]):
|
||||
@staticmethod
|
||||
def maybe_make_prepare_finalize(
|
||||
moe: FusedMoEConfig) -> Optional[FusedMoEPrepareAndFinalize]:
|
||||
all2all_manager = get_ep_group().device_communicator.all2all_manager
|
||||
assert all2all_manager is not None
|
||||
|
||||
self.moe = moe
|
||||
|
||||
prepare_finalize: Optional[FusedMoEPrepareAndFinalize] = None
|
||||
|
||||
if moe.use_pplx_kernels:
|
||||
@@ -160,8 +159,6 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
and moe.quant_config.block_shape
|
||||
== DEEPEP_QUANT_BLOCK_SHAPE)
|
||||
|
||||
# Note (varun): Whether to use FP8 dispatch or not needs some
|
||||
# profiling. Turning it off for now.
|
||||
prepare_finalize = DeepEPLLPrepareAndFinalize(
|
||||
handle,
|
||||
max_tokens_per_rank=moe.max_num_tokens,
|
||||
@@ -169,11 +166,18 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
use_fp8_dispatch=use_fp8_dispatch,
|
||||
)
|
||||
|
||||
return prepare_finalize
|
||||
|
||||
def init_prepare_finalize(self, moe: FusedMoEConfig):
|
||||
self.moe = moe
|
||||
prepare_finalize = FusedMoEMethodBase.maybe_make_prepare_finalize(
|
||||
self.moe)
|
||||
|
||||
self.topk_indices_dtype = None
|
||||
if prepare_finalize is not None:
|
||||
logger.debug("%s", prepare_finalize.__class__.__name__)
|
||||
self.topk_indices_dtype = prepare_finalize.topk_indices_dtype()
|
||||
experts = self.select_gemm_impl(prepare_finalize, moe)
|
||||
experts = self.select_gemm_impl(prepare_finalize, self.moe)
|
||||
self.fused_experts = FusedMoEModularKernel(
|
||||
prepare_finalize,
|
||||
experts,
|
||||
|
||||
@@ -7,7 +7,8 @@ import torch
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_moe import (
|
||||
DeepGemmExperts, _valid_deep_gemm, _valid_deep_gemm_shape)
|
||||
DeepGemmExperts, _valid_deep_gemm, _valid_deep_gemm_shape,
|
||||
deep_gemm_block_shape)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts
|
||||
from vllm.utils.deep_gemm import is_blackwell_deep_gemm_used
|
||||
|
||||
@@ -44,8 +45,10 @@ class TritonOrDeepGemmExperts(mk.FusedMoEPermuteExpertsUnpermute):
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
self.allow_deep_gemm = (allow_deep_gemm and not per_act_token_quant
|
||||
and use_fp8_w8a8)
|
||||
|
||||
self.allow_deep_gemm = (allow_deep_gemm and use_fp8_w8a8 and
|
||||
self.block_shape == deep_gemm_block_shape())
|
||||
|
||||
self.deep_gemm_expert = DeepGemmExperts(
|
||||
) if self.allow_deep_gemm else None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user