[Feature] Support sequence parallel without the need for DP, 1.9%~5.0% E2E Throughput Improvement (#47070)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
@@ -632,8 +632,8 @@ class ParallelConfig:
|
|||||||
|
|
||||||
# The all_reduce at the end of attention (during o_proj) means that
|
# The all_reduce at the end of attention (during o_proj) means that
|
||||||
# inputs are replicated across each rank of the tensor parallel group.
|
# inputs are replicated across each rank of the tensor parallel group.
|
||||||
# If using expert-parallelism with DeepEP All2All ops, replicated
|
# If using expert-parallelism, replicated tokens results in useless
|
||||||
# tokens results in useless duplicate computation and communication.
|
# duplicate computation and communication.
|
||||||
#
|
#
|
||||||
# In this case, ensure the input to the experts is sequence parallel
|
# In this case, ensure the input to the experts is sequence parallel
|
||||||
# to avoid the excess work.
|
# to avoid the excess work.
|
||||||
@@ -652,7 +652,6 @@ class ParallelConfig:
|
|||||||
)
|
)
|
||||||
and self.enable_expert_parallel
|
and self.enable_expert_parallel
|
||||||
and self.tensor_parallel_size > 1
|
and self.tensor_parallel_size > 1
|
||||||
and self.data_parallel_size > 1
|
|
||||||
)
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|||||||
@@ -175,10 +175,11 @@ class DeviceCommunicatorBase:
|
|||||||
|
|
||||||
config = get_current_vllm_config_or_none()
|
config = get_current_vllm_config_or_none()
|
||||||
if config is not None:
|
if config is not None:
|
||||||
# as long as we use data parallel (coupled data parallel
|
# initialize the all2all manager for DP or sequence-parallel EP.
|
||||||
# where all data parallel ranks execute forward together),
|
use_ep = (
|
||||||
# we initialize the all2all manager used in expert parallel.
|
config.parallel_config.data_parallel_size > 1
|
||||||
use_ep = config.parallel_config.data_parallel_size > 1
|
or config.parallel_config.use_sequence_parallel_moe
|
||||||
|
)
|
||||||
all2all_backend = config.parallel_config.all2all_backend
|
all2all_backend = config.parallel_config.all2all_backend
|
||||||
|
|
||||||
self.is_ep_communicator = unique_name.split(":")[0] == "ep"
|
self.is_ep_communicator = unique_name.split(":")[0] == "ep"
|
||||||
|
|||||||
+15
-3
@@ -83,7 +83,10 @@ class DPMetadata:
|
|||||||
num_tokens_across_dp_cpu: torch.Tensor,
|
num_tokens_across_dp_cpu: torch.Tensor,
|
||||||
) -> "DPMetadata":
|
) -> "DPMetadata":
|
||||||
assert num_tokens_across_dp_cpu is not None
|
assert num_tokens_across_dp_cpu is not None
|
||||||
assert parallel_config.data_parallel_size > 1
|
assert (
|
||||||
|
parallel_config.data_parallel_size > 1
|
||||||
|
or parallel_config.use_sequence_parallel_moe
|
||||||
|
)
|
||||||
assert parallel_config.is_moe_model is not False
|
assert parallel_config.is_moe_model is not False
|
||||||
dp_rank = parallel_config.data_parallel_rank
|
dp_rank = parallel_config.data_parallel_rank
|
||||||
batchsize = num_tokens
|
batchsize = num_tokens
|
||||||
@@ -277,14 +280,20 @@ def set_forward_context(
|
|||||||
|
|
||||||
dp_metadata: DPMetadata | None = None
|
dp_metadata: DPMetadata | None = None
|
||||||
if (
|
if (
|
||||||
vllm_config.parallel_config.data_parallel_size > 1
|
(
|
||||||
|
vllm_config.parallel_config.data_parallel_size > 1
|
||||||
|
or vllm_config.parallel_config.use_sequence_parallel_moe
|
||||||
|
)
|
||||||
and vllm_config.parallel_config.is_moe_model is not False
|
and vllm_config.parallel_config.is_moe_model is not False
|
||||||
and (attn_metadata is not None or num_tokens is not None)
|
and (attn_metadata is not None or num_tokens is not None)
|
||||||
):
|
):
|
||||||
# If num_tokens_across_dp hasn't already been initialized, then
|
# If num_tokens_across_dp hasn't already been initialized, then
|
||||||
# initialize it here. Both DP padding and Microbatching will be
|
# initialize it here. Both DP padding and Microbatching will be
|
||||||
# disabled.
|
# disabled.
|
||||||
if num_tokens_across_dp is None:
|
if (
|
||||||
|
num_tokens_across_dp is None
|
||||||
|
and vllm_config.parallel_config.data_parallel_size > 1
|
||||||
|
):
|
||||||
assert ubatch_slices is None
|
assert ubatch_slices is None
|
||||||
assert num_tokens is not None
|
assert num_tokens is not None
|
||||||
_, num_tokens_across_dp, _ = coordinate_batch_across_dp(
|
_, num_tokens_across_dp, _ = coordinate_batch_across_dp(
|
||||||
@@ -293,6 +302,9 @@ def set_forward_context(
|
|||||||
allow_microbatching=False,
|
allow_microbatching=False,
|
||||||
)
|
)
|
||||||
assert num_tokens_across_dp is not None
|
assert num_tokens_across_dp is not None
|
||||||
|
elif num_tokens_across_dp is None:
|
||||||
|
assert num_tokens is not None
|
||||||
|
num_tokens_across_dp = torch.tensor([num_tokens], dtype=torch.int32)
|
||||||
dp_metadata = DPMetadata.make(
|
dp_metadata = DPMetadata.make(
|
||||||
vllm_config.parallel_config, num_tokens or 0, num_tokens_across_dp
|
vllm_config.parallel_config, num_tokens or 0, num_tokens_across_dp
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1038,7 +1038,7 @@ class FusedMoEParallelConfig:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def use_all2all_kernels(self):
|
def use_all2all_kernels(self):
|
||||||
return self.dp_size > 1 and self.use_ep
|
return self.use_ep and (self.dp_size > 1 or self.is_sequence_parallel)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def use_deepep_ht_kernels(self):
|
def use_deepep_ht_kernels(self):
|
||||||
|
|||||||
@@ -725,8 +725,8 @@ class MoERunner(MoERunnerInterface):
|
|||||||
@property
|
@property
|
||||||
def do_naive_dispatch_combine(self) -> bool:
|
def do_naive_dispatch_combine(self) -> bool:
|
||||||
return (
|
return (
|
||||||
self.moe_config.dp_size > 1 and not self._quant_method.supports_internal_mk
|
self.moe_config.dp_size > 1 or self.moe_config.is_sequence_parallel
|
||||||
)
|
) and not self._quant_method.supports_internal_mk
|
||||||
|
|
||||||
def _maybe_dispatch(
|
def _maybe_dispatch(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -1305,15 +1305,11 @@ class DeepseekV2DecoderLayer(nn.Module):
|
|||||||
residual *= 1.0 / self.routed_scaling_factor
|
residual *= 1.0 / self.routed_scaling_factor
|
||||||
|
|
||||||
if self.use_sequence_parallel_moe:
|
if self.use_sequence_parallel_moe:
|
||||||
sp_remainder = (
|
tp_world_size = get_tensor_model_parallel_world_size()
|
||||||
hidden_states.shape[0] % get_tensor_model_parallel_world_size()
|
# small trick using minus, eg. -17 % 8 = 7
|
||||||
)
|
sp_pad = (-hidden_states.shape[0]) % tp_world_size
|
||||||
# pad if not divisible by world size
|
# pad if not divisible by world size
|
||||||
if sp_remainder:
|
hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
|
||||||
sp_pad = get_tensor_model_parallel_world_size() - sp_remainder
|
|
||||||
hidden_states = torch.nn.functional.pad(
|
|
||||||
hidden_states, (0, 0, 0, sp_pad)
|
|
||||||
)
|
|
||||||
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
|
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
|
||||||
if not input_is_sequence_parallel:
|
if not input_is_sequence_parallel:
|
||||||
residual = sequence_parallel_chunk(residual)
|
residual = sequence_parallel_chunk(residual)
|
||||||
|
|||||||
@@ -192,7 +192,10 @@ class MLPBlock(torch.nn.Module):
|
|||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
parallel_config = vllm_config.parallel_config
|
parallel_config = vllm_config.parallel_config
|
||||||
|
|
||||||
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
|
self.is_sequence_parallel = (
|
||||||
|
parallel_config.use_sequence_parallel_moe
|
||||||
|
and vllm_config.lora_config is None
|
||||||
|
)
|
||||||
|
|
||||||
self.layer_idx = layer_idx
|
self.layer_idx = layer_idx
|
||||||
self.num_experts = config.num_local_experts
|
self.num_experts = config.num_local_experts
|
||||||
|
|||||||
Reference in New Issue
Block a user