[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-10-12 12:32:39 +00:00
committed by GitHub
parent aa1d44df16
commit 258b97ff6b
18 changed files with 416 additions and 121 deletions
@@ -22,12 +22,16 @@ class _Pos:
self.offset = offset
self.length = length
def get_num_embeds(self):
return self.length
class _Feature:
def __init__(self, mm_hash, length=1, identifier=None):
self.mm_hash = mm_hash
self.identifier = identifier if identifier is not None else mm_hash
self.mm_position = _Pos(0, length)
self.modality = "image"
class _Request:
@@ -80,7 +84,7 @@ def _make_scheduler(
def _load_ids(meta) -> list[int]:
"""Transfer ids the scheduler dispatched in this step's metadata."""
return [transfer_id for transfer_id, _ in meta.loads.values()]
return [transfer_id for transfer_id, _, _ in meta.loads.values()]
def _load_blocks(meta, mm_hash: str) -> list[int]:
@@ -90,7 +94,7 @@ def _load_blocks(meta, mm_hash: str) -> list[int]:
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
"""Pre-populate a ready cache entry backed by real blocks."""
s._cache.alloc(mm_hash, n_blocks)
s._cache.alloc(mm_hash, n_blocks, (n_blocks, _BS // s._dtype.itemsize))
s._cache.mark_ready(mm_hash)
@@ -97,10 +97,9 @@ def test_second_instance_opens_existing_file_and_shares_memory():
r1.cleanup()
def test_dense_dp_engines_get_distinct_regions(monkeypatch):
def test_dense_dp_engines_get_distinct_regions():
"""Dense DP engines reset data_parallel_rank to 0 but share instance_id,
so the region must be keyed by data_parallel_index."""
monkeypatch.setattr(common, "_get_encoder_cache_hidden_dim", lambda cfg: 8)
instance_id = str(uuid.uuid4())
regions = []
try:
@@ -113,6 +112,7 @@ def test_dense_dp_engines_get_distinct_regions(monkeypatch):
cfg.instance_id = instance_id
cfg.parallel_config = parallel_config
cfg.model_config.dtype = torch.float16
cfg.model_config.get_inputs_embeds_size.return_value = 8
cfg.ec_transfer_config.ec_connector_extra_config = {"ec_cpu_bytes": 64}
regions.append(common.create_ec_shared_region(cfg))
@@ -91,7 +91,9 @@ def _make_region() -> ECSharedRegion:
def _vllm_config(rank: int = 0) -> Mock:
return create_ec_vllm_config(rank=rank, dtype=_DTYPE)
config = create_ec_vllm_config(rank=rank, dtype=_DTYPE)
config.model_config.get_inputs_embeds_size.return_value = _HIDDEN_DIM
return config
def _meta(
@@ -105,7 +107,7 @@ def _meta(
return ECCPUConnectorMetadata(
saves=saves or {},
loads={
mm_hash: (idx, block_ids)
mm_hash: (idx, block_ids, (len(block_ids), _HIDDEN_DIM))
for idx, (mm_hash, block_ids) in enumerate((loads or {}).items())
},
)
@@ -904,21 +906,29 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
# Build scheduler sharing the same region.
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
scheduler = ECCPUScheduler(_vllm_config())
config = _vllm_config()
# DeepStack-wide output: each embedding spans four blocks.
width = 4 * _HIDDEN_DIM
config.ec_transfer_config.mm_encoder_output_widths = {"image": width}
scheduler = ECCPUScheduler(config)
# -- Step 1: scheduler allocates, worker saves --
n_blocks = 3
src = torch.arange(
n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE
).reshape(n_blocks, _HIDDEN_DIM)
n_tokens = 2
src = torch.arange(n_tokens * width, dtype=_DTYPE, device=DEVICE_TYPE).reshape(
n_tokens, width
)
class _Pos:
offset = 0
length = n_blocks
length = n_tokens
def get_num_embeds(self):
return self.length
class _Feature:
identifier = "img_001"
mm_position = _Pos()
modality = "image"
class _Request:
request_id = "req_e2e"
@@ -948,8 +958,9 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
scheduler.update_state_after_alloc(_Request(), 0)
meta_load = scheduler.build_connector_meta(scheduler_output=None)
assert "img_001" in meta_load.loads
_, load_blocks = meta_load.loads["img_001"]
_, load_blocks, shape = meta_load.loads["img_001"]
assert load_blocks == meta_save.saves["img_001"]
assert shape == tuple(src.shape)
load_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(load_cache, meta_load)