[Bugfix][EC Connector] Fix ECExampleConnector load device under TP>1 (#40416)

Signed-off-by: Hang Gong <kidogong@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Hang Gong
2026-09-09 11:44:30 +08:00
committed by GitHub
co-authored by Claude Codex
parent fc6b6e1feb
commit bf7eee9d7b
2 changed files with 89 additions and 3 deletions
@@ -477,6 +477,89 @@ class TestCacheLoading:
# Cache should remain empty
assert len(encoder_cache) == 0
def test_start_load_caches_pins_device_to_current_rank(
self, mock_vllm_config_producer, mock_vllm_config_consumer
):
"""Load must pin to the selected device under tensor parallelism."""
# Produce a real cache file so the load path has something to open.
producer = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
mm_hash = "rank_pinning_hash"
producer.save_caches({mm_hash: torch.zeros(4, 8)}, mm_hash)
consumer = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
metadata = ECExampleConnectorMetadata()
metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100))
consumer.bind_connector_metadata(metadata)
mock_platform = Mock()
mock_platform.device_type = "cuda"
mock_platform.is_cuda_alike.return_value = True
mock_platform.current_device.return_value = 2
# Simulate running on TP rank 2 without requiring a multi-GPU host.
captured: dict = {}
def fake_load_file(filename, device):
captured["device"] = device
return {"ec_cache": torch.zeros(4, 8)}
with (
patch("vllm.platforms.current_platform", mock_platform),
patch("safetensors.torch.load_file", side_effect=fake_load_file),
):
encoder_cache: dict[str, torch.Tensor] = {}
consumer.start_load_caches(encoder_cache=encoder_cache)
assert captured["device"] == "cuda:2", (
f"Expected device='cuda:2' (pinned to current rank), "
f"got {captured['device']!r}. A bare 'cuda' string makes "
"safetensors load onto cuda:0, which breaks non-rank-0 TP workers."
)
def test_start_load_caches_non_cuda_platform_uses_device_type(
self, mock_vllm_config_producer, mock_vllm_config_consumer
):
"""Non-CUDA platforms pass `device_type` through unchanged (no rank suffix)."""
producer = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
mm_hash = "cpu_device_hash"
producer.save_caches({mm_hash: torch.zeros(2, 4)}, mm_hash)
consumer = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
metadata = ECExampleConnectorMetadata()
metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100))
consumer.bind_connector_metadata(metadata)
mock_platform = Mock()
mock_platform.device_type = "cpu"
mock_platform.is_cuda_alike.return_value = False
captured: dict = {}
def fake_load_file(filename, device):
captured["device"] = device
return {"ec_cache": torch.zeros(2, 4)}
with (
patch("vllm.platforms.current_platform", mock_platform),
patch("safetensors.torch.load_file", side_effect=fake_load_file),
):
encoder_cache: dict[str, torch.Tensor] = {}
consumer.start_load_caches(encoder_cache=encoder_cache)
assert captured["device"] == "cpu"
class TestFilenameGeneration:
"""Test filename and path generation."""
@@ -89,14 +89,17 @@ class ECExampleConnector(ECConnectorBase):
"In connector.start_load_caches, but the connector metadata is None"
)
return
# A bare "cuda" makes safetensors load onto cuda:0. Pin the load to the
# selected device so tensor-parallel workers load the cache locally.
device = current_platform.device_type
if current_platform.is_cuda_alike():
device = f"{device}:{current_platform.current_device()}"
# Load the EC for each mm data
for mm_data in metadata.mm_datas:
if mm_data.mm_hash in encoder_cache:
continue
filename = self._generate_filename_debug(mm_data.mm_hash)
ec_cache = safetensors.torch.load_file(
filename, device=current_platform.device_type
)["ec_cache"]
ec_cache = safetensors.torch.load_file(filename, device=device)["ec_cache"]
encoder_cache[mm_data.mm_hash] = ec_cache
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)