[Core] Check for GPU<->CPU syncs during CI (#43107)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
7b544ecb52
commit
615d4cfade
@@ -33,6 +33,7 @@ from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
|
||||
from vllm.distributed.utils import is_weak_contiguous
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -222,10 +223,11 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
|
||||
|
||||
# Wait for all D2H copies to finish
|
||||
# before issuing gloo batch_isend_irecv operations.
|
||||
if self._cuda_stream is not None:
|
||||
self._cuda_stream.synchronize()
|
||||
else:
|
||||
torch.cuda.current_stream().synchronize()
|
||||
with gpu_sync_allowed():
|
||||
if self._cuda_stream is not None:
|
||||
self._cuda_stream.synchronize()
|
||||
else:
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
reqs = batch_isend_irecv(p2p_ops)
|
||||
for req in reqs:
|
||||
|
||||
@@ -47,6 +47,7 @@ from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
from .async_worker import start_async_worker
|
||||
from .eplb_communicator import EplbCommunicator, create_eplb_communicator
|
||||
@@ -596,10 +597,11 @@ class EplbState:
|
||||
num_tokens_per_rank
|
||||
)
|
||||
|
||||
# Just to make type checker happy
|
||||
tokens_tensors: list[float] = torch.stack(
|
||||
[avg_tokens_tensor, max_tokens_tensor]
|
||||
).tolist()
|
||||
# This gpu/cpu sync only happens with expert stat logging enabled.
|
||||
with gpu_sync_allowed():
|
||||
tokens_tensors: list[float] = torch.stack(
|
||||
[avg_tokens_tensor, max_tokens_tensor]
|
||||
).tolist()
|
||||
avg_tokens, max_tokens = tokens_tensors
|
||||
balancedness = avg_tokens / max_tokens if max_tokens > 0 else 0.0
|
||||
|
||||
@@ -825,15 +827,17 @@ class EplbState:
|
||||
self.model_states.values(), global_expert_load_windows
|
||||
):
|
||||
if not self.is_async or is_profile:
|
||||
# Get new expert mappings for the model
|
||||
new_physical_to_logical_map = self.policy.rebalance_experts(
|
||||
global_expert_load_window.cpu(),
|
||||
num_replicas,
|
||||
num_groups,
|
||||
num_nodes,
|
||||
num_gpus,
|
||||
eplb_model_state.physical_to_logical_map.cpu(),
|
||||
)
|
||||
# Get new expert mappings for the model. The policy runs on the
|
||||
# host, so the load window and current map have to come back.
|
||||
with gpu_sync_allowed():
|
||||
new_physical_to_logical_map = self.policy.rebalance_experts(
|
||||
global_expert_load_window.cpu(),
|
||||
num_replicas,
|
||||
num_groups,
|
||||
num_nodes,
|
||||
num_gpus,
|
||||
eplb_model_state.physical_to_logical_map.cpu(),
|
||||
)
|
||||
|
||||
skip_rearrange = False
|
||||
if (
|
||||
@@ -1248,7 +1252,9 @@ def _pad_out_tensor(src: torch.Tensor, dst: torch.Tensor) -> None:
|
||||
src_padding = dst.shape[-1] - src.shape[-1]
|
||||
assert src_padding >= 0
|
||||
new_src = torch.nn.functional.pad(src, (0, src_padding), value=-1)
|
||||
dst.copy_(new_src)
|
||||
# The map is committed from the host once per layer per rearrangement.
|
||||
with gpu_sync_allowed():
|
||||
dst.copy_(new_src)
|
||||
|
||||
|
||||
def _commit_eplb_maps_for_layer(
|
||||
@@ -1276,10 +1282,7 @@ def _commit_eplb_maps_for_layer(
|
||||
num_logical_experts = model_state.logical_to_physical_map.shape[1]
|
||||
new_logical, new_replica_count = compute_logical_maps(src, num_logical_experts)
|
||||
# Commit logical_to_physical_map
|
||||
_pad_out_tensor(
|
||||
src=new_logical,
|
||||
dst=model_state.logical_to_physical_map[layer],
|
||||
)
|
||||
_pad_out_tensor(src=new_logical, dst=model_state.logical_to_physical_map[layer])
|
||||
|
||||
# Commit logical_replica_count
|
||||
src = new_replica_count
|
||||
|
||||
@@ -16,6 +16,7 @@ from torch.distributed import ProcessGroup, all_gather
|
||||
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
|
||||
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -590,8 +591,11 @@ def rearrange_expert_weights_inplace(
|
||||
|
||||
weights_buffer = list(expert_buffer)
|
||||
|
||||
old_global_expert_indices_cpu = old_global_expert_indices.cpu().numpy()
|
||||
new_global_expert_indices_cpu = new_global_expert_indices.cpu().numpy()
|
||||
# The per-layer transfer plan is built in Python, so both maps have to
|
||||
# come back to the host. Once per rearrangement.
|
||||
with gpu_sync_allowed():
|
||||
old_global_expert_indices_cpu = old_global_expert_indices.cpu().numpy()
|
||||
new_global_expert_indices_cpu = new_global_expert_indices.cpu().numpy()
|
||||
|
||||
for layer_idx in range(num_moe_layers):
|
||||
transfer_metadata = move_to_buffer(
|
||||
|
||||
Reference in New Issue
Block a user