[Feature] Support sequence parallel without the need for DP, 1.9%~5.0% E2E Throughput Improvement (#47070)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
@@ -1038,7 +1038,7 @@ class FusedMoEParallelConfig:
|
||||
|
||||
@property
|
||||
def use_all2all_kernels(self):
|
||||
return self.dp_size > 1 and self.use_ep
|
||||
return self.use_ep and (self.dp_size > 1 or self.is_sequence_parallel)
|
||||
|
||||
@property
|
||||
def use_deepep_ht_kernels(self):
|
||||
|
||||
@@ -725,8 +725,8 @@ class MoERunner(MoERunnerInterface):
|
||||
@property
|
||||
def do_naive_dispatch_combine(self) -> bool:
|
||||
return (
|
||||
self.moe_config.dp_size > 1 and not self._quant_method.supports_internal_mk
|
||||
)
|
||||
self.moe_config.dp_size > 1 or self.moe_config.is_sequence_parallel
|
||||
) and not self._quant_method.supports_internal_mk
|
||||
|
||||
def _maybe_dispatch(
|
||||
self,
|
||||
|
||||
@@ -1305,15 +1305,11 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
residual *= 1.0 / self.routed_scaling_factor
|
||||
|
||||
if self.use_sequence_parallel_moe:
|
||||
sp_remainder = (
|
||||
hidden_states.shape[0] % get_tensor_model_parallel_world_size()
|
||||
)
|
||||
tp_world_size = get_tensor_model_parallel_world_size()
|
||||
# small trick using minus, eg. -17 % 8 = 7
|
||||
sp_pad = (-hidden_states.shape[0]) % tp_world_size
|
||||
# pad if not divisible by world size
|
||||
if sp_remainder:
|
||||
sp_pad = get_tensor_model_parallel_world_size() - sp_remainder
|
||||
hidden_states = torch.nn.functional.pad(
|
||||
hidden_states, (0, 0, 0, sp_pad)
|
||||
)
|
||||
hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
|
||||
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
|
||||
if not input_is_sequence_parallel:
|
||||
residual = sequence_parallel_chunk(residual)
|
||||
|
||||
@@ -192,7 +192,10 @@ class MLPBlock(torch.nn.Module):
|
||||
quant_config = vllm_config.quant_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
|
||||
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
|
||||
self.is_sequence_parallel = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and vllm_config.lora_config is None
|
||||
)
|
||||
|
||||
self.layer_idx = layer_idx
|
||||
self.num_experts = config.num_local_experts
|
||||
|
||||
Reference in New Issue
Block a user