[Core] Check for GPU<->CPU syncs during CI (#43107)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-08-14 17:42:05 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 7b544ecb52
commit 615d4cfade
44 changed files with 600 additions and 357 deletions
+6 -4
View File
@@ -33,6 +33,7 @@ from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
from vllm.distributed.utils import is_weak_contiguous
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
logger = init_logger(__name__)
@@ -222,10 +223,11 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
# Wait for all D2H copies to finish
# before issuing gloo batch_isend_irecv operations.
if self._cuda_stream is not None:
self._cuda_stream.synchronize()
else:
torch.cuda.current_stream().synchronize()
with gpu_sync_allowed():
if self._cuda_stream is not None:
self._cuda_stream.synchronize()
else:
torch.cuda.current_stream().synchronize()
reqs = batch_isend_irecv(p2p_ops)
for req in reqs:
+21 -18
View File
@@ -47,6 +47,7 @@ from vllm.distributed.utils import StatelessProcessGroup
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.platforms import current_platform
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from .async_worker import start_async_worker
from .eplb_communicator import EplbCommunicator, create_eplb_communicator
@@ -596,10 +597,11 @@ class EplbState:
num_tokens_per_rank
)
# Just to make type checker happy
tokens_tensors: list[float] = torch.stack(
[avg_tokens_tensor, max_tokens_tensor]
).tolist()
# This gpu/cpu sync only happens with expert stat logging enabled.
with gpu_sync_allowed():
tokens_tensors: list[float] = torch.stack(
[avg_tokens_tensor, max_tokens_tensor]
).tolist()
avg_tokens, max_tokens = tokens_tensors
balancedness = avg_tokens / max_tokens if max_tokens > 0 else 0.0
@@ -825,15 +827,17 @@ class EplbState:
self.model_states.values(), global_expert_load_windows
):
if not self.is_async or is_profile:
# Get new expert mappings for the model
new_physical_to_logical_map = self.policy.rebalance_experts(
global_expert_load_window.cpu(),
num_replicas,
num_groups,
num_nodes,
num_gpus,
eplb_model_state.physical_to_logical_map.cpu(),
)
# Get new expert mappings for the model. The policy runs on the
# host, so the load window and current map have to come back.
with gpu_sync_allowed():
new_physical_to_logical_map = self.policy.rebalance_experts(
global_expert_load_window.cpu(),
num_replicas,
num_groups,
num_nodes,
num_gpus,
eplb_model_state.physical_to_logical_map.cpu(),
)
skip_rearrange = False
if (
@@ -1248,7 +1252,9 @@ def _pad_out_tensor(src: torch.Tensor, dst: torch.Tensor) -> None:
src_padding = dst.shape[-1] - src.shape[-1]
assert src_padding >= 0
new_src = torch.nn.functional.pad(src, (0, src_padding), value=-1)
dst.copy_(new_src)
# The map is committed from the host once per layer per rearrangement.
with gpu_sync_allowed():
dst.copy_(new_src)
def _commit_eplb_maps_for_layer(
@@ -1276,10 +1282,7 @@ def _commit_eplb_maps_for_layer(
num_logical_experts = model_state.logical_to_physical_map.shape[1]
new_logical, new_replica_count = compute_logical_maps(src, num_logical_experts)
# Commit logical_to_physical_map
_pad_out_tensor(
src=new_logical,
dst=model_state.logical_to_physical_map[layer],
)
_pad_out_tensor(src=new_logical, dst=model_state.logical_to_physical_map[layer])
# Commit logical_replica_count
src = new_replica_count
+6 -2
View File
@@ -16,6 +16,7 @@ from torch.distributed import ProcessGroup, all_gather
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent
from vllm.logger import init_logger
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
logger = init_logger(__name__)
@@ -590,8 +591,11 @@ def rearrange_expert_weights_inplace(
weights_buffer = list(expert_buffer)
old_global_expert_indices_cpu = old_global_expert_indices.cpu().numpy()
new_global_expert_indices_cpu = new_global_expert_indices.cpu().numpy()
# The per-layer transfer plan is built in Python, so both maps have to
# come back to the host. Once per rearrangement.
with gpu_sync_allowed():
old_global_expert_indices_cpu = old_global_expert_indices.cpu().numpy()
new_global_expert_indices_cpu = new_global_expert_indices.cpu().numpy()
for layer_idx in range(num_moe_layers):
transfer_metadata = move_to_buffer(
@@ -15,6 +15,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.hashing import safe_hash
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
@@ -243,7 +244,8 @@ class ExampleConnector(KVConnectorBase_V1):
layer_name, request.token_ids, request.mm_hashes
)
kv_cache = extract_kv_from_layer(kv_layer, request.slot_mapping)
tensors = {"kv_cache": kv_cache.detach().cpu()}
with gpu_sync_allowed():
tensors = {"kv_cache": kv_cache.detach().cpu()}
safetensors.torch.save_file(tensors, filename)
def wait_for_save(self):
@@ -67,6 +67,7 @@ from vllm.distributed.parallel_state import (
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.network_utils import make_zmq_path
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.utils import get_kv_cache_layout
@@ -1882,14 +1883,16 @@ class NixlBaseConnectorWorker:
local_block_ids = meta.local_physical_block_ids
# TODO (NickLucche) D2H<>H2D ops could benefit from coalescing io across groups
for group_block_ids in local_block_ids:
self.copy_blocks(
self.host_xfer_buffers,
self.device_kv_caches,
group_block_ids,
group_block_ids,
"h2d",
)
# The h2d block copies below are intentionally synchronous.
with gpu_sync_allowed():
for group_block_ids in local_block_ids:
self.copy_blocks(
self.host_xfer_buffers,
self.device_kv_caches,
group_block_ids,
group_block_ids,
"h2d",
)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"synced recved kv of request[%s] to device kv buffer,"
@@ -1903,26 +1906,28 @@ class NixlBaseConnectorWorker:
assert self.use_host_buffer
assert self.copy_blocks is not None
for req_id, meta in metadata.reqs_to_save.items():
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"save_load_kv for request[%s] to host xfer buffer."
"local_block_ids: %s. ",
req_id,
",".join(map(str, meta.local_physical_block_ids)),
)
# blocking
for group_block_ids in meta.local_physical_block_ids:
self.copy_blocks(
self.device_kv_caches,
self.host_xfer_buffers,
group_block_ids,
group_block_ids,
"d2h",
# The d2h block copies below are intentionally synchronous.
with gpu_sync_allowed():
for req_id, meta in metadata.reqs_to_save.items():
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"save_load_kv for request[%s] to host xfer buffer."
"local_block_ids: %s. ",
req_id,
",".join(map(str, meta.local_physical_block_ids)),
)
# blocking
for group_block_ids in meta.local_physical_block_ids:
self.copy_blocks(
self.device_kv_caches,
self.host_xfer_buffers,
group_block_ids,
group_block_ids,
"d2h",
)
@cached_property
def _attention_kv_caches(self) -> list[torch.Tensor]: