[Core] Check for GPU<->CPU syncs during CI (#43107)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
7b544ecb52
commit
615d4cfade
+27
-18
@@ -1,12 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import nullcontext
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed.parallel_state import get_dp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.v1.worker.ubatch_utils import (
|
||||
check_ubatch_thresholds,
|
||||
is_last_ubatch_empty,
|
||||
@@ -136,27 +139,33 @@ def _synchronize_dp_ranks(
|
||||
parallel_config=parallel_config,
|
||||
)
|
||||
|
||||
# Synchronize cudagraph_mode across ranks first (take min).
|
||||
# This is needed before DP padding decision since we use the synced
|
||||
# cudagraph mode to determine whether DP padding is needed.
|
||||
synced_cudagraph_mode = _post_process_cudagraph_mode(tensor)
|
||||
# Only the NCCL path leaves `tensor` on device. With Gloo -- the default
|
||||
# under async scheduling -- the all-reduce runs on CPU, so the reads below
|
||||
# are host-side and the check should stay armed.
|
||||
with (
|
||||
nullcontext()
|
||||
if parallel_config.disable_nccl_for_dp_synchronization
|
||||
else gpu_sync_allowed()
|
||||
):
|
||||
# Synchronize cudagraph_mode across ranks first (take min).
|
||||
# This is needed before DP padding decision since we use the synced
|
||||
# cudagraph mode to determine whether DP padding is needed.
|
||||
synced_cudagraph_mode = _post_process_cudagraph_mode(tensor)
|
||||
|
||||
# Check conditions for microbatching
|
||||
should_ubatch = _post_process_ubatch(tensor, parallel_config.num_ubatches)
|
||||
# Check conditions for microbatching
|
||||
should_ubatch = _post_process_ubatch(tensor, parallel_config.num_ubatches)
|
||||
|
||||
# DP padding is needed when cudagraph is enabled (synced across ranks)
|
||||
# or when ubatching/DBO is active (ubatching requires uniform batch
|
||||
# sizes across DP ranks currently).
|
||||
# Use the synced runtime cudagraph mode rather than the compilation config
|
||||
# so we can avoid padding when cudagraph is not enabled for this step.
|
||||
should_dp_pad = synced_cudagraph_mode != 0 or should_ubatch
|
||||
# DP padding is needed when cudagraph is enabled (synced across ranks)
|
||||
# or when ubatching/DBO is active (ubatching requires uniform batch
|
||||
# sizes across DP ranks currently).
|
||||
# Use the synced runtime cudagraph mode rather than the compilation
|
||||
# config so we can avoid padding when cudagraph is not enabled for
|
||||
# this step.
|
||||
should_dp_pad = synced_cudagraph_mode != 0 or should_ubatch
|
||||
|
||||
# Pad all DP ranks up to the maximum token count across ranks if
|
||||
# should_dp_pad is True
|
||||
num_tokens_after_padding = _post_process_dp_padding(
|
||||
tensor,
|
||||
should_dp_pad,
|
||||
)
|
||||
# Pad all DP ranks up to the maximum token count across ranks if
|
||||
# should_dp_pad is True
|
||||
num_tokens_after_padding = _post_process_dp_padding(tensor, should_dp_pad)
|
||||
|
||||
return should_ubatch, num_tokens_after_padding, synced_cudagraph_mode
|
||||
|
||||
|
||||
Reference in New Issue
Block a user