[Bugfix] Account for PCP in multi-node world size validation (#55111)

Signed-off-by: DebugSy <532151398@qq.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Simon
2026-09-03 10:52:42 +00:00
committed by GitHub
co-authored by Claude Harry Mellor
parent facd9a74a1
commit ee0a4c46ae
2 changed files with 25 additions and 8 deletions
+6 -8
View File
@@ -2121,21 +2121,19 @@ class EngineArgs:
)
inferred_data_parallel_rank = 0
if self.nnodes > 1:
world_size = (
self.data_parallel_size
* self.pipeline_parallel_size
* self.tensor_parallel_size
)
world_size_within_dp = (
self.pipeline_parallel_size * self.tensor_parallel_size
self.pipeline_parallel_size
* self.tensor_parallel_size
* self.prefill_context_parallel_size
)
world_size = self.data_parallel_size * world_size_within_dp
if world_size % self.nnodes != 0:
raise ValueError(
"Invalid data-parallel launch options: "
f"`--nnodes {self.nnodes}` must evenly divide the total "
f"world size ({world_size}). Adjust `--nnodes`, "
"`--data-parallel-size`, `--pipeline-parallel-size`, or "
"`--tensor-parallel-size`."
"`--data-parallel-size`, `--pipeline-parallel-size`, "
"`--tensor-parallel-size`, or `--prefill-context-parallel-size`."
)
if not 0 <= self.node_rank < self.nnodes:
raise ValueError(