From 1696c864b90afb9cec925312d0cf12c396517007 Mon Sep 17 00:00:00 2001 From: Zhewen Li Date: Tue, 14 Apr 2026 14:13:58 -0700 Subject: [PATCH] [Bugfix][Mooncake] Fix thread-local CUDA context for NVLink transfers in _send_blocks (#39548) Signed-off-by: Zhewen Li Co-authored-by: Zhewen Li --- .../kv_connector/v1/mooncake/mooncake_connector.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index b49a016641..6c17bf82ac 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -41,6 +41,7 @@ from vllm.distributed.parallel_state import ( ) from vllm.forward_context import ForwardContext from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.utils.network_utils import get_ip, make_zmq_path, make_zmq_socket from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.attention.backends.utils import get_kv_cache_layout @@ -645,6 +646,10 @@ class MooncakeConnectorWorker: logger.info("Initializing Mooncake Transfer Engine worker %s", engine_id) self.vllm_config = vllm_config + # Capture device BEFORE TransferEngine init — MNNVL's NVLink allocator + # may change the current CUDA device during engine.initialize(). + self.device_id = torch.accelerator.current_device_index() + current_platform.set_device(self.device_id) self.engine = TransferEngine() self.hostname = get_ip() @@ -705,9 +710,12 @@ class MooncakeConnectorWorker: # For kv_both, we will act both prefiller and decoder. if not self.is_kv_consumer: # Background threads for sending kvcaches to D. + # Each pool thread must be bound to the correct CUDA device + # because CUDA device selection is thread-local. self._sender_executor = ThreadPoolExecutor( max_workers=self.num_sender_workers, thread_name_prefix="vllm-mooncake-sender", + initializer=self._bind_sender_thread_device, ) logger.debug( "Mooncake Prefiller: use %d workers to send kvcaches", @@ -1193,6 +1201,12 @@ class MooncakeConnectorWorker: return src_ptrs, dst_ptrs, lengths, err_reqs, err_msg + def _bind_sender_thread_device(self) -> None: + """ThreadPoolExecutor initializer — binds each pool thread to the + correct CUDA device. CUDA device selection is thread-local, so + without this, NVLink transfers fail for TP ranks > 0.""" + current_platform.set_device(self.device_id) + def _send_blocks( self, remote_session: str,