[Bugfix] Account for PCP in multi-node world size validation (#55111)
Signed-off-by: DebugSy <532151398@qq.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Harry Mellor
parent
facd9a74a1
commit
ee0a4c46ae
@@ -2121,21 +2121,19 @@ class EngineArgs:
|
||||
)
|
||||
inferred_data_parallel_rank = 0
|
||||
if self.nnodes > 1:
|
||||
world_size = (
|
||||
self.data_parallel_size
|
||||
* self.pipeline_parallel_size
|
||||
* self.tensor_parallel_size
|
||||
)
|
||||
world_size_within_dp = (
|
||||
self.pipeline_parallel_size * self.tensor_parallel_size
|
||||
self.pipeline_parallel_size
|
||||
* self.tensor_parallel_size
|
||||
* self.prefill_context_parallel_size
|
||||
)
|
||||
world_size = self.data_parallel_size * world_size_within_dp
|
||||
if world_size % self.nnodes != 0:
|
||||
raise ValueError(
|
||||
"Invalid data-parallel launch options: "
|
||||
f"`--nnodes {self.nnodes}` must evenly divide the total "
|
||||
f"world size ({world_size}). Adjust `--nnodes`, "
|
||||
"`--data-parallel-size`, `--pipeline-parallel-size`, or "
|
||||
"`--tensor-parallel-size`."
|
||||
"`--data-parallel-size`, `--pipeline-parallel-size`, "
|
||||
"`--tensor-parallel-size`, or `--prefill-context-parallel-size`."
|
||||
)
|
||||
if not 0 <= self.node_rank < self.nnodes:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user