[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -22,12 +22,16 @@ class _Pos:
|
||||
self.offset = offset
|
||||
self.length = length
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1, identifier=None):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = identifier if identifier is not None else mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
self.modality = "image"
|
||||
|
||||
|
||||
class _Request:
|
||||
@@ -80,7 +84,7 @@ def _make_scheduler(
|
||||
|
||||
def _load_ids(meta) -> list[int]:
|
||||
"""Transfer ids the scheduler dispatched in this step's metadata."""
|
||||
return [transfer_id for transfer_id, _ in meta.loads.values()]
|
||||
return [transfer_id for transfer_id, _, _ in meta.loads.values()]
|
||||
|
||||
|
||||
def _load_blocks(meta, mm_hash: str) -> list[int]:
|
||||
@@ -90,7 +94,7 @@ def _load_blocks(meta, mm_hash: str) -> list[int]:
|
||||
|
||||
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
|
||||
"""Pre-populate a ready cache entry backed by real blocks."""
|
||||
s._cache.alloc(mm_hash, n_blocks)
|
||||
s._cache.alloc(mm_hash, n_blocks, (n_blocks, _BS // s._dtype.itemsize))
|
||||
s._cache.mark_ready(mm_hash)
|
||||
|
||||
|
||||
|
||||
@@ -97,10 +97,9 @@ def test_second_instance_opens_existing_file_and_shares_memory():
|
||||
r1.cleanup()
|
||||
|
||||
|
||||
def test_dense_dp_engines_get_distinct_regions(monkeypatch):
|
||||
def test_dense_dp_engines_get_distinct_regions():
|
||||
"""Dense DP engines reset data_parallel_rank to 0 but share instance_id,
|
||||
so the region must be keyed by data_parallel_index."""
|
||||
monkeypatch.setattr(common, "_get_encoder_cache_hidden_dim", lambda cfg: 8)
|
||||
instance_id = str(uuid.uuid4())
|
||||
regions = []
|
||||
try:
|
||||
@@ -113,6 +112,7 @@ def test_dense_dp_engines_get_distinct_regions(monkeypatch):
|
||||
cfg.instance_id = instance_id
|
||||
cfg.parallel_config = parallel_config
|
||||
cfg.model_config.dtype = torch.float16
|
||||
cfg.model_config.get_inputs_embeds_size.return_value = 8
|
||||
cfg.ec_transfer_config.ec_connector_extra_config = {"ec_cpu_bytes": 64}
|
||||
regions.append(common.create_ec_shared_region(cfg))
|
||||
|
||||
|
||||
@@ -91,7 +91,9 @@ def _make_region() -> ECSharedRegion:
|
||||
|
||||
|
||||
def _vllm_config(rank: int = 0) -> Mock:
|
||||
return create_ec_vllm_config(rank=rank, dtype=_DTYPE)
|
||||
config = create_ec_vllm_config(rank=rank, dtype=_DTYPE)
|
||||
config.model_config.get_inputs_embeds_size.return_value = _HIDDEN_DIM
|
||||
return config
|
||||
|
||||
|
||||
def _meta(
|
||||
@@ -105,7 +107,7 @@ def _meta(
|
||||
return ECCPUConnectorMetadata(
|
||||
saves=saves or {},
|
||||
loads={
|
||||
mm_hash: (idx, block_ids)
|
||||
mm_hash: (idx, block_ids, (len(block_ids), _HIDDEN_DIM))
|
||||
for idx, (mm_hash, block_ids) in enumerate((loads or {}).items())
|
||||
},
|
||||
)
|
||||
@@ -904,21 +906,29 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
# Build scheduler sharing the same region.
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
|
||||
scheduler = ECCPUScheduler(_vllm_config())
|
||||
config = _vllm_config()
|
||||
# DeepStack-wide output: each embedding spans four blocks.
|
||||
width = 4 * _HIDDEN_DIM
|
||||
config.ec_transfer_config.mm_encoder_output_widths = {"image": width}
|
||||
scheduler = ECCPUScheduler(config)
|
||||
|
||||
# -- Step 1: scheduler allocates, worker saves --
|
||||
n_blocks = 3
|
||||
src = torch.arange(
|
||||
n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE
|
||||
).reshape(n_blocks, _HIDDEN_DIM)
|
||||
n_tokens = 2
|
||||
src = torch.arange(n_tokens * width, dtype=_DTYPE, device=DEVICE_TYPE).reshape(
|
||||
n_tokens, width
|
||||
)
|
||||
|
||||
class _Pos:
|
||||
offset = 0
|
||||
length = n_blocks
|
||||
length = n_tokens
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
class _Feature:
|
||||
identifier = "img_001"
|
||||
mm_position = _Pos()
|
||||
modality = "image"
|
||||
|
||||
class _Request:
|
||||
request_id = "req_e2e"
|
||||
@@ -948,8 +958,9 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
scheduler.update_state_after_alloc(_Request(), 0)
|
||||
meta_load = scheduler.build_connector_meta(scheduler_output=None)
|
||||
assert "img_001" in meta_load.loads
|
||||
_, load_blocks = meta_load.loads["img_001"]
|
||||
_, load_blocks, shape = meta_load.loads["img_001"]
|
||||
assert load_blocks == meta_save.saves["img_001"]
|
||||
assert shape == tuple(src.shape)
|
||||
|
||||
load_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(load_cache, meta_load)
|
||||
|
||||
Reference in New Issue
Block a user