[Core] Check for GPU<->CPU syncs during CI (#43107)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
7b544ecb52
commit
615d4cfade
@@ -15,6 +15,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.hashing import safe_hash
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
@@ -243,7 +244,8 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
layer_name, request.token_ids, request.mm_hashes
|
||||
)
|
||||
kv_cache = extract_kv_from_layer(kv_layer, request.slot_mapping)
|
||||
tensors = {"kv_cache": kv_cache.detach().cpu()}
|
||||
with gpu_sync_allowed():
|
||||
tensors = {"kv_cache": kv_cache.detach().cpu()}
|
||||
safetensors.torch.save_file(tensors, filename)
|
||||
|
||||
def wait_for_save(self):
|
||||
|
||||
@@ -67,6 +67,7 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
||||
@@ -1882,14 +1883,16 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
# TODO (NickLucche) D2H<>H2D ops could benefit from coalescing io across groups
|
||||
for group_block_ids in local_block_ids:
|
||||
self.copy_blocks(
|
||||
self.host_xfer_buffers,
|
||||
self.device_kv_caches,
|
||||
group_block_ids,
|
||||
group_block_ids,
|
||||
"h2d",
|
||||
)
|
||||
# The h2d block copies below are intentionally synchronous.
|
||||
with gpu_sync_allowed():
|
||||
for group_block_ids in local_block_ids:
|
||||
self.copy_blocks(
|
||||
self.host_xfer_buffers,
|
||||
self.device_kv_caches,
|
||||
group_block_ids,
|
||||
group_block_ids,
|
||||
"h2d",
|
||||
)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
"synced recved kv of request[%s] to device kv buffer,"
|
||||
@@ -1903,26 +1906,28 @@ class NixlBaseConnectorWorker:
|
||||
assert self.use_host_buffer
|
||||
assert self.copy_blocks is not None
|
||||
|
||||
for req_id, meta in metadata.reqs_to_save.items():
|
||||
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
|
||||
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
"save_load_kv for request[%s] to host xfer buffer."
|
||||
"local_block_ids: %s. ",
|
||||
req_id,
|
||||
",".join(map(str, meta.local_physical_block_ids)),
|
||||
)
|
||||
# blocking
|
||||
for group_block_ids in meta.local_physical_block_ids:
|
||||
self.copy_blocks(
|
||||
self.device_kv_caches,
|
||||
self.host_xfer_buffers,
|
||||
group_block_ids,
|
||||
group_block_ids,
|
||||
"d2h",
|
||||
# The d2h block copies below are intentionally synchronous.
|
||||
with gpu_sync_allowed():
|
||||
for req_id, meta in metadata.reqs_to_save.items():
|
||||
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
|
||||
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
"save_load_kv for request[%s] to host xfer buffer."
|
||||
"local_block_ids: %s. ",
|
||||
req_id,
|
||||
",".join(map(str, meta.local_physical_block_ids)),
|
||||
)
|
||||
# blocking
|
||||
for group_block_ids in meta.local_physical_block_ids:
|
||||
self.copy_blocks(
|
||||
self.device_kv_caches,
|
||||
self.host_xfer_buffers,
|
||||
group_block_ids,
|
||||
group_block_ids,
|
||||
"d2h",
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def _attention_kv_caches(self) -> list[torch.Tensor]:
|
||||
|
||||
Reference in New Issue
Block a user