[ROCm][CI] Add GELU activation for AiterExperts in the modular-kernel coverage (#58030)
Signed-off-by: Divakar Verma <divakar.verma@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
415e11c6f1
commit
d1f63f2af9
@@ -84,6 +84,8 @@ class Config:
|
||||
|
||||
world_size: int
|
||||
|
||||
activation: MoEActivation = MoEActivation.SILU
|
||||
|
||||
torch_trace_dir_path: str | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
@@ -311,6 +313,16 @@ class Config:
|
||||
f"block={self.quant_block_shape})"
|
||||
)
|
||||
|
||||
# Check activation support; NotImplementedError means no opinion.
|
||||
try:
|
||||
if not self.fused_experts_type._supports_activation(self.activation):
|
||||
return False, (
|
||||
f"FE {self.fused_experts_type.__name__} does not support "
|
||||
f"activation {self.activation}"
|
||||
)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
|
||||
# Check block quantization support
|
||||
is_block_quantized = self.quant_block_shape is not None
|
||||
if is_block_quantized and self.quant_dtype is None:
|
||||
@@ -605,6 +617,7 @@ def reference_moe_impl(
|
||||
topk_ids=rank_tensors.topk_ids,
|
||||
global_num_experts=config.E,
|
||||
expert_map=None,
|
||||
activation=config.activation,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a_scale,
|
||||
@@ -648,7 +661,7 @@ def make_modular_kernel(
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
in_dtype=config.dtype,
|
||||
max_num_tokens=next_power_of_2(config.M),
|
||||
activation=MoEActivation.SILU,
|
||||
activation=config.activation,
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=RoutingMethodType.DeepSeekV3,
|
||||
)
|
||||
@@ -774,7 +787,7 @@ def run_modular_kernel(
|
||||
"w2": rank_weights.w2,
|
||||
"topk_weights": rank_tensors.topk_weights,
|
||||
"topk_ids": topk_ids,
|
||||
"activation": MoEActivation.SILU,
|
||||
"activation": config.activation,
|
||||
"expert_map": rank_tensors.expert_map,
|
||||
"global_num_experts": config.E,
|
||||
"apply_router_weight_on_input": config.topk == 1
|
||||
|
||||
Reference in New Issue
Block a user