From bf7eee9d7b2788bafe69f0c21fde6af22a2d3319 Mon Sep 17 00:00:00 2001 From: Hang Gong Date: Tue, 8 Sep 2026 20:44:30 -0700 Subject: [PATCH] [Bugfix][EC Connector] Fix ECExampleConnector load device under TP>1 (#40416) Signed-off-by: Hang Gong Co-authored-by: Claude Co-authored-by: Codex --- .../unit/test_ec_example_connector.py | 83 +++++++++++++++++++ .../ec_connector/example_connector.py | 9 +- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/tests/v1/ec_connector/unit/test_ec_example_connector.py b/tests/v1/ec_connector/unit/test_ec_example_connector.py index dcae0bddad..8edb54d2a6 100644 --- a/tests/v1/ec_connector/unit/test_ec_example_connector.py +++ b/tests/v1/ec_connector/unit/test_ec_example_connector.py @@ -477,6 +477,89 @@ class TestCacheLoading: # Cache should remain empty assert len(encoder_cache) == 0 + def test_start_load_caches_pins_device_to_current_rank( + self, mock_vllm_config_producer, mock_vllm_config_consumer + ): + """Load must pin to the selected device under tensor parallelism.""" + # Produce a real cache file so the load path has something to open. + producer = ECExampleConnector( + vllm_config=mock_vllm_config_producer, + role=ECConnectorRole.WORKER, + ) + mm_hash = "rank_pinning_hash" + producer.save_caches({mm_hash: torch.zeros(4, 8)}, mm_hash) + + consumer = ECExampleConnector( + vllm_config=mock_vllm_config_consumer, + role=ECConnectorRole.WORKER, + ) + metadata = ECExampleConnectorMetadata() + metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100)) + consumer.bind_connector_metadata(metadata) + + mock_platform = Mock() + mock_platform.device_type = "cuda" + mock_platform.is_cuda_alike.return_value = True + mock_platform.current_device.return_value = 2 + + # Simulate running on TP rank 2 without requiring a multi-GPU host. + captured: dict = {} + + def fake_load_file(filename, device): + captured["device"] = device + return {"ec_cache": torch.zeros(4, 8)} + + with ( + patch("vllm.platforms.current_platform", mock_platform), + patch("safetensors.torch.load_file", side_effect=fake_load_file), + ): + encoder_cache: dict[str, torch.Tensor] = {} + consumer.start_load_caches(encoder_cache=encoder_cache) + + assert captured["device"] == "cuda:2", ( + f"Expected device='cuda:2' (pinned to current rank), " + f"got {captured['device']!r}. A bare 'cuda' string makes " + "safetensors load onto cuda:0, which breaks non-rank-0 TP workers." + ) + + def test_start_load_caches_non_cuda_platform_uses_device_type( + self, mock_vllm_config_producer, mock_vllm_config_consumer + ): + """Non-CUDA platforms pass `device_type` through unchanged (no rank suffix).""" + producer = ECExampleConnector( + vllm_config=mock_vllm_config_producer, + role=ECConnectorRole.WORKER, + ) + mm_hash = "cpu_device_hash" + producer.save_caches({mm_hash: torch.zeros(2, 4)}, mm_hash) + + consumer = ECExampleConnector( + vllm_config=mock_vllm_config_consumer, + role=ECConnectorRole.WORKER, + ) + metadata = ECExampleConnectorMetadata() + metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100)) + consumer.bind_connector_metadata(metadata) + + mock_platform = Mock() + mock_platform.device_type = "cpu" + mock_platform.is_cuda_alike.return_value = False + + captured: dict = {} + + def fake_load_file(filename, device): + captured["device"] = device + return {"ec_cache": torch.zeros(2, 4)} + + with ( + patch("vllm.platforms.current_platform", mock_platform), + patch("safetensors.torch.load_file", side_effect=fake_load_file), + ): + encoder_cache: dict[str, torch.Tensor] = {} + consumer.start_load_caches(encoder_cache=encoder_cache) + + assert captured["device"] == "cpu" + class TestFilenameGeneration: """Test filename and path generation.""" diff --git a/vllm/distributed/ec_transfer/ec_connector/example_connector.py b/vllm/distributed/ec_transfer/ec_connector/example_connector.py index 8ea1684fc9..98e6f1f109 100644 --- a/vllm/distributed/ec_transfer/ec_connector/example_connector.py +++ b/vllm/distributed/ec_transfer/ec_connector/example_connector.py @@ -89,14 +89,17 @@ class ECExampleConnector(ECConnectorBase): "In connector.start_load_caches, but the connector metadata is None" ) return + # A bare "cuda" makes safetensors load onto cuda:0. Pin the load to the + # selected device so tensor-parallel workers load the cache locally. + device = current_platform.device_type + if current_platform.is_cuda_alike(): + device = f"{device}:{current_platform.current_device()}" # Load the EC for each mm data for mm_data in metadata.mm_datas: if mm_data.mm_hash in encoder_cache: continue filename = self._generate_filename_debug(mm_data.mm_hash) - ec_cache = safetensors.torch.load_file( - filename, device=current_platform.device_type - )["ec_cache"] + ec_cache = safetensors.torch.load_file(filename, device=device)["ec_cache"] encoder_cache[mm_data.mm_hash] = ec_cache logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)