[Feature] Support sequence parallel without the need for DP, 1.9%~5.0% E2E Throughput Improvement (#47070)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
@@ -192,7 +192,10 @@ class MLPBlock(torch.nn.Module):
|
||||
quant_config = vllm_config.quant_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
|
||||
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
|
||||
self.is_sequence_parallel = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and vllm_config.lora_config is None
|
||||
)
|
||||
|
||||
self.layer_idx = layer_idx
|
||||
self.num_experts = config.num_local_experts
|
||||
|
||||
Reference in New Issue
Block a user