[Kernel] Enable fp8 support for pplx and BatchedTritonExperts. (#18864)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
+46
-2
@@ -573,6 +573,41 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts
|
||||
self.fused_experts_func = fused_experts
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: FusedMoEPrepareAndFinalize,
|
||||
moe: FusedMoEConfig,
|
||||
) -> FusedMoEPermuteExpertsUnpermute:
|
||||
from vllm.model_executor.layers.fused_moe import TritonExperts
|
||||
from vllm.model_executor.layers.fused_moe.fused_batched_moe import (
|
||||
BatchedTritonExperts)
|
||||
|
||||
assert not self.rocm_aiter_moe_enabled and not self.use_marlin
|
||||
|
||||
logger.debug("BatchedTritonExperts(%s)", self.__class__.__name__)
|
||||
|
||||
if (prepare_finalize.activation_format ==
|
||||
FusedMoEActivationFormat.BatchedExperts):
|
||||
max_num_tokens_per_rank = prepare_finalize.max_num_tokens_per_rank(
|
||||
)
|
||||
assert max_num_tokens_per_rank is not None
|
||||
|
||||
return BatchedTritonExperts(
|
||||
max_num_tokens=max_num_tokens_per_rank,
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
use_fp8_w8a8=True,
|
||||
block_shape=self.quant_config.weight_block_size,
|
||||
per_act_token_quant=(
|
||||
self.input_quant.strategy == QuantizationStrategy.TOKEN),
|
||||
)
|
||||
else:
|
||||
return TritonExperts(
|
||||
use_fp8_w8a8=True,
|
||||
block_shape=self.quant_config.weight_block_size,
|
||||
per_act_token_quant=(
|
||||
self.input_quant.strategy == QuantizationStrategy.TOKEN),
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -610,7 +645,9 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
num_expert_group=num_expert_group,
|
||||
custom_routing_function=custom_routing_function,
|
||||
scoring_func=scoring_func,
|
||||
e_score_correction_bias=e_score_correction_bias)
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
indices_type=self.topk_indices_dtype,
|
||||
)
|
||||
|
||||
if self.rocm_aiter_moe_enabled:
|
||||
return self.rocm_aiter_fused_experts_func(
|
||||
@@ -832,18 +869,25 @@ class CompressedTensorsW8A8Fp8MoECutlassMethod(CompressedTensorsMoEMethod):
|
||||
use_batched_format = (prepare_finalize.activation_format ==
|
||||
FusedMoEActivationFormat.BatchedExperts)
|
||||
|
||||
num_dispatchers = prepare_finalize.num_dispatchers()
|
||||
|
||||
num_experts = (moe.num_local_experts
|
||||
if use_batched_format else moe.num_experts)
|
||||
|
||||
logger.debug("CutlassExpertsFp8(%s)", self.__class__.__name__)
|
||||
|
||||
experts = CutlassExpertsFp8(
|
||||
num_experts,
|
||||
moe.in_dtype,
|
||||
self.input_quant.strategy == QuantizationStrategy.TOKEN,
|
||||
self.weight_quant.strategy == QuantizationStrategy.CHANNEL,
|
||||
num_dispatchers=num_dispatchers,
|
||||
use_batched_format=use_batched_format,
|
||||
)
|
||||
|
||||
self.disable_expert_map = not experts.supports_expert_map()
|
||||
self.disable_expert_map = (num_dispatchers > 1
|
||||
or not experts.supports_expert_map())
|
||||
|
||||
return experts
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -802,10 +802,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
self.quant_config.weight_block_size, False)
|
||||
return BatchedTritonOrDeepGemmExperts(
|
||||
max_num_tokens=max_num_tokens_per_rank,
|
||||
world_size=prepare_finalize.
|
||||
world_size, # type: ignore [attr-defined]
|
||||
dp_size=prepare_finalize.
|
||||
dp_size, # type: ignore [attr-defined]
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
use_fp8_w8a8=True,
|
||||
block_shape=self.quant_config.weight_block_size,
|
||||
per_act_token_quant=False,
|
||||
|
||||
Reference in New Issue
Block a user