[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-10-12 12:32:39 +00:00
committed by GitHub
parent aa1d44df16
commit 258b97ff6b
18 changed files with 416 additions and 121 deletions
@@ -22,12 +22,16 @@ class _Pos:
self.offset = offset
self.length = length
def get_num_embeds(self):
return self.length
class _Feature:
def __init__(self, mm_hash, length=1, identifier=None):
self.mm_hash = mm_hash
self.identifier = identifier if identifier is not None else mm_hash
self.mm_position = _Pos(0, length)
self.modality = "image"
class _Request:
@@ -80,7 +84,7 @@ def _make_scheduler(
def _load_ids(meta) -> list[int]:
"""Transfer ids the scheduler dispatched in this step's metadata."""
return [transfer_id for transfer_id, _ in meta.loads.values()]
return [transfer_id for transfer_id, _, _ in meta.loads.values()]
def _load_blocks(meta, mm_hash: str) -> list[int]:
@@ -90,7 +94,7 @@ def _load_blocks(meta, mm_hash: str) -> list[int]:
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
"""Pre-populate a ready cache entry backed by real blocks."""
s._cache.alloc(mm_hash, n_blocks)
s._cache.alloc(mm_hash, n_blocks, (n_blocks, _BS // s._dtype.itemsize))
s._cache.mark_ready(mm_hash)
@@ -97,10 +97,9 @@ def test_second_instance_opens_existing_file_and_shares_memory():
r1.cleanup()
def test_dense_dp_engines_get_distinct_regions(monkeypatch):
def test_dense_dp_engines_get_distinct_regions():
"""Dense DP engines reset data_parallel_rank to 0 but share instance_id,
so the region must be keyed by data_parallel_index."""
monkeypatch.setattr(common, "_get_encoder_cache_hidden_dim", lambda cfg: 8)
instance_id = str(uuid.uuid4())
regions = []
try:
@@ -113,6 +112,7 @@ def test_dense_dp_engines_get_distinct_regions(monkeypatch):
cfg.instance_id = instance_id
cfg.parallel_config = parallel_config
cfg.model_config.dtype = torch.float16
cfg.model_config.get_inputs_embeds_size.return_value = 8
cfg.ec_transfer_config.ec_connector_extra_config = {"ec_cpu_bytes": 64}
regions.append(common.create_ec_shared_region(cfg))
@@ -91,7 +91,9 @@ def _make_region() -> ECSharedRegion:
def _vllm_config(rank: int = 0) -> Mock:
return create_ec_vllm_config(rank=rank, dtype=_DTYPE)
config = create_ec_vllm_config(rank=rank, dtype=_DTYPE)
config.model_config.get_inputs_embeds_size.return_value = _HIDDEN_DIM
return config
def _meta(
@@ -105,7 +107,7 @@ def _meta(
return ECCPUConnectorMetadata(
saves=saves or {},
loads={
mm_hash: (idx, block_ids)
mm_hash: (idx, block_ids, (len(block_ids), _HIDDEN_DIM))
for idx, (mm_hash, block_ids) in enumerate((loads or {}).items())
},
)
@@ -904,21 +906,29 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
# Build scheduler sharing the same region.
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
scheduler = ECCPUScheduler(_vllm_config())
config = _vllm_config()
# DeepStack-wide output: each embedding spans four blocks.
width = 4 * _HIDDEN_DIM
config.ec_transfer_config.mm_encoder_output_widths = {"image": width}
scheduler = ECCPUScheduler(config)
# -- Step 1: scheduler allocates, worker saves --
n_blocks = 3
src = torch.arange(
n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE
).reshape(n_blocks, _HIDDEN_DIM)
n_tokens = 2
src = torch.arange(n_tokens * width, dtype=_DTYPE, device=DEVICE_TYPE).reshape(
n_tokens, width
)
class _Pos:
offset = 0
length = n_blocks
length = n_tokens
def get_num_embeds(self):
return self.length
class _Feature:
identifier = "img_001"
mm_position = _Pos()
modality = "image"
class _Request:
request_id = "req_e2e"
@@ -948,8 +958,9 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
scheduler.update_state_after_alloc(_Request(), 0)
meta_load = scheduler.build_connector_meta(scheduler_output=None)
assert "img_001" in meta_load.loads
_, load_blocks = meta_load.loads["img_001"]
_, load_blocks, shape = meta_load.loads["img_001"]
assert load_blocks == meta_save.saves["img_001"]
assert shape == tuple(src.shape)
load_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(load_cache, meta_load)
@@ -45,6 +45,34 @@ KEY = (
SPEC = TensorSpec((2, 2), "torch.float32", 16)
def test_push_shapes_follow_each_modality(monkeypatch):
"""Each push is sized at its modality's measured encoder output width."""
from vllm.distributed.ec_transfer.ec_connector.mooncake import scheduler
monkeypatch.setattr(scheduler, "ensure_mooncake_available", lambda: None)
widths = {"image": 8192, "audio": 2048}
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths=widths)
request = SimpleNamespace(
request_id="mixed",
mm_features=[SimpleNamespace(identifier=m, modality=m) for m in widths],
get_num_encoder_embeds=lambda index: 2,
ec_transfer_params={
"consumer_zmq": "tcp://consumer:1234",
"ec_items": [{"mm_hash": m, "transfer_id": m} for m in widths],
},
)
instance = scheduler.ECMooncakeScheduler(config)
try:
for index in range(len(widths)):
instance.update_state_after_alloc(request, index)
metadata = instance.build_connector_meta(
SimpleNamespace(free_encoder_mm_hashes=[], preempted_req_ids=set())
)
assert [spec.shape for spec in metadata.pushes] == [(2, 8192), (2, 2048)]
finally:
instance.close()
def test_embedding_key_combines_namespace_and_identifier():
config = create_ec_vllm_config(ec_role="ec_producer")
config.ec_transfer_config.ec_connector_extra_config.update(
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Startup measurement of the encoder output width EC producers size by."""
from types import SimpleNamespace
import pytest
import torch
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector import encoder_output_width as eow
pytestmark = pytest.mark.cpu_test
class _DeepStackModel:
"""Emits each modality at its own width, as Qwen3-Omni does."""
widths = {"image": 4 * 16, "audio": 16}
def embed_multimodal(self, *, modality):
return [torch.zeros(3, self.widths[modality])]
class _Budget:
def __init__(self, *args, **kwargs):
self.mm_max_toks_per_item = dict.fromkeys(_DeepStackModel.widths, 8)
def get_dummy_encoder_profile_inputs(self, modality, max_items_per_batch):
assert max_items_per_batch == 1
return [(modality, None)]
def _worker(first_pp_rank=True, supports_mm=True):
runner = SimpleNamespace(
supports_mm_inputs=supports_mm,
get_model=_DeepStackModel,
device=torch.device("cpu"),
)
pp_group = SimpleNamespace(is_first_rank=first_pp_rank)
return SimpleNamespace(model_runner=runner, vllm_config=None), pp_group
@pytest.fixture
def patched(monkeypatch):
import vllm.distributed.parallel_state as ps
import vllm.multimodal.encoder_budget as budget_mod
import vllm.multimodal.utils as mm_utils
monkeypatch.setattr(budget_mod, "MultiModalBudget", _Budget)
monkeypatch.setattr(
mm_utils,
"group_and_batch_mm_kwargs",
lambda items, **_: iter([(items[0][0], 1, {"modality": items[0][0]})]),
)
return lambda group: monkeypatch.setattr(ps, "get_pp_group", lambda: group)
def test_worker_reports_the_encoder_output_width_per_modality(patched):
"""The width comes from the tensor the encoder emits, not from config."""
worker, group = _worker()
patched(group)
assert eow._measure_on_worker(worker) == {"image": 64, "audio": 16}
@pytest.mark.parametrize("first_pp_rank, supports_mm", [(False, True), (True, False)])
def test_ranks_without_an_encoder_report_nothing(patched, first_pp_rank, supports_mm):
worker, group = _worker(first_pp_rank, supports_mm)
patched(group)
assert eow._measure_on_worker(worker) == {}
def test_producer_records_the_widths_its_encoder_ranks_agree_on():
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths={})
reports = [{"image": 64}, {}, {"image": 64}]
eow.measure_encoder_output_widths(config, lambda fn: reports)
assert config.ec_transfer_config.mm_encoder_output_widths == {"image": 64}
assert eow.get_encoder_output_width(config, "image") == 64
@pytest.mark.parametrize(
"reports, match",
[
([{}, {}], "could not measure"),
([{"image": 64}, {"image": 16}], "disagree"),
],
ids=["no-encoder", "disagree"],
)
def test_producer_refuses_to_start_without_a_trustworthy_width(reports, match):
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths={})
with pytest.raises(ValueError, match=match):
eow.measure_encoder_output_widths(config, lambda fn: reports)
@@ -25,6 +25,9 @@ class _Pos:
def __init__(self, offset, length):
self.offset, self.length = offset, length
def get_num_embeds(self):
return self.length
class _Feature:
def __init__(self, mm_hash, length=1, data=None):
@@ -85,6 +88,9 @@ class _FakeTransport:
pass
_MISSING = object()
def _consumer_sched(monkeypatch):
def _region(cfg):
return ECSharedRegion(
@@ -95,16 +101,15 @@ def _consumer_sched(monkeypatch):
# ec_enable_nixl defaults to False, so this builds gate-off; the fields
# below flip on the NIXL consumer state directly.
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_consumer"))
# No measured widths: a consumer takes each shape from the announcement.
s = ECCPUScheduler(
create_ec_vllm_config(ec_role="ec_consumer", encoder_output_widths={})
)
# Turn on NIXL consumer state without constructing real transports.
s._nixl_enabled = True
s._transport = _FakeTransport()
s._data = None
s._compat_hash = "c"
# _setup_nixl normally computes these from model_config; set them
# directly since this helper builds gate-off then flips fields on.
s._hidden_dim = _HID
s._element_size = _ES
s._metadata_resolver._cache["image"] = {"image_grid_thw"}
return s
@@ -115,24 +120,32 @@ def _params(mm_hash, length):
"peer_host": "h",
"peer_port": 1,
"size_bytes": length * _HID * _ES,
"shape": [length, _HID],
}
}
def test_new_remote_read_defers_then_completes(monkeypatch):
"""The read is sized from the announced shape, not a local width."""
width = 4 * _HID
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
# Route _start_xfer to our fake session instead of real ZMQ/NIXL, but
# still reserve a real not-ready cache entry so mark_ready works.
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
def _fake_start(mm_hash, info, size, shape):
assert size == width * _ES
assert shape == (1, width)
entry = s._cache.alloc(mm_hash, size // _BS, shape)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
params = _params("h1", 1)
params["h1"]["size_bytes"] = width * _ES
params["h1"]["shape"] = [1, width]
req = _Request([_Feature("h1", 1)], params=params)
# Step 1: unseen remote item -> read started, request deferred.
assert s.ensure_cache_available(req, 0) is False
@@ -155,6 +168,7 @@ def test_new_remote_read_defers_then_completes(monkeypatch):
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta.loads
assert meta.loads["h1"][2] == (1, width)
assert "h1" not in s._step_completed # cleared by promote
# Step 3: still cached -> admitted directly, no new transfer.
@@ -176,8 +190,8 @@ def test_failed_read_falls_back_only_with_local_input(monkeypatch, payload):
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
def _fake_start(mm_hash, info, size, shape):
entry = s._cache.alloc(mm_hash, 1, shape)
assert entry is not None
fake.started.append(mm_hash)
return True
@@ -230,8 +244,8 @@ def test_remote_wait_budget_survives_retries_and_long_steps(monkeypatch, retry):
fake = _FakeSession()
s._sessions[("h", 1)] = fake
def start(mm_hash, info, size):
assert s._cache.alloc(mm_hash, 1) is not None
def start(mm_hash, info, size, shape):
assert s._cache.alloc(mm_hash, 1, shape) is not None
fake.started.append(mm_hash)
return True
@@ -261,8 +275,8 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
def _fake_start(mm_hash, info, size, shape):
entry = s._cache.alloc(mm_hash, 1, shape)
assert entry is not None
fake.started.append(mm_hash)
return True
@@ -287,14 +301,29 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
@pytest.mark.parametrize(
"size_fields",
[{"size_bytes": 999}, {"size_bytes": None}, {"size_bytes": "big"}, {}],
ids=["mismatch", "null-size", "text-size", "no-size"],
"fields",
[
{"size_bytes": 999},
{"size_bytes": None},
{"size_bytes": "big"},
{"size_bytes": _MISSING},
{"shape": _MISSING},
{"shape": [2, _HID]},
],
ids=[
"size-mismatch",
"null-size",
"text-size",
"no-size",
"no-shape",
"wrong-rows",
],
)
def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
"""Invalid remote sizes fail the request without raising in the scheduler."""
def test_invalid_announcement_fails_the_request(monkeypatch, fields):
"""An unusable announced size or shape fails the request, not the engine."""
s = _consumer_sched(monkeypatch)
announced = {"peer_host": "h", "peer_port": 1, **size_fields}
announced = {**_params("h1", 1)["h1"], **fields}
announced = {k: v for k, v in announced.items() if v is not _MISSING}
req = _Request([_Feature("h1", 1)], params={"h1": announced})
assert s.ensure_cache_available(req, 0) is False
assert s.take_unavailable_requests() == {"r1"}
@@ -306,7 +335,7 @@ def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
def test_deferral_budget_is_per_request(monkeypatch):
"""Shared hashes have independent wait budgets and cancellation cleanup."""
s = _consumer_sched(monkeypatch)
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
monkeypatch.setattr(s._cache, "alloc", lambda key, n, shape=None: None)
params = _params("h1", 1)
old = _Request([_Feature("h1", 1)], params=params, req_id="old")
new = _Request([_Feature("h1", 1)], params=params, req_id="new")
@@ -419,7 +448,7 @@ def test_in_flight_hash_defers_without_second_transfer(monkeypatch):
s = _consumer_sched(monkeypatch)
started: list[str] = []
def _spy_start_xfer(mm_hash, info, size):
def _spy_start_xfer(mm_hash, info, size, shape):
started.append(mm_hash)
return True
@@ -16,6 +16,9 @@ class _Pos:
def __init__(self, offset, length):
self.offset, self.length = offset, length
def get_num_embeds(self):
return self.length
class _Feature:
def __init__(self, mm_hash, length=1):
@@ -53,11 +56,8 @@ def test_request_finished_producer_emits_params(monkeypatch):
# Simulate NIXL-enabled producer bookkeeping without building real NIXL.
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
# _setup_nixl normally computes these from model_config; set them
# directly since this test builds gate-off then flips fields on.
s._hidden_dim, s._element_size = 32, 2
# feature length=2, hidden_dim=32, element_size=2 -> 128 bytes -> 2 blocks.
entry = s._cache.alloc("h1", 2)
entry = s._cache.alloc("h1", 2, (2, 32))
assert entry is not None
s._cache.mark_ready("h1")
@@ -71,6 +71,7 @@ def test_request_finished_producer_emits_params(monkeypatch):
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
"shape": [2, 32],
}
}
s.shutdown()
@@ -84,8 +85,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
s._hidden_dim, s._element_size = 32, 2
s._cache.alloc("h1", 2) # allocated but not marked ready
s._cache.alloc("h1", 2, (2, 32)) # allocated but not marked ready
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
@@ -96,6 +96,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
"shape": [2, 32],
}
}
s.shutdown()
@@ -105,7 +106,6 @@ def test_request_finished_skips_unallocated_entry(monkeypatch):
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
s._hidden_dim, s._element_size = 32, 2
# No alloc() for "h1" at all — e.g. the cache was full at save time.
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
+14 -4
View File
@@ -18,12 +18,16 @@ _EXECUTOR_BACKEND = "mp"
_PCP_SIZE = 1
_EMBED_WIDTH = 32
def create_ec_vllm_config(
*,
ec_role: ECRole = "ec_both",
tensor_parallel_size: int = 1,
rank: int = 0,
dtype: torch.dtype = torch.float16,
encoder_output_widths: dict[str, int] | None = None,
) -> Mock:
"""Build a `VllmConfig` stand-in for EC connector unit tests.
@@ -31,10 +35,8 @@ def create_ec_vllm_config(
role derivation and the rank arithmetic the connector depends on behave as
they do in production.
`model_config` is a stub because nothing under test needs a real one: the
connector reads only `dtype` here, and the paths that inspect the model
itself are not exercised. Building a real `ModelConfig` would resolve an HF
model and inspect its architecture.
`model_config` supplies a dtype and embedding width without resolving an
HF model or inspecting its architecture.
Args:
ec_role: EC role this instance plays. `is_ec_producer` and
@@ -42,6 +44,8 @@ def create_ec_vllm_config(
tensor_parallel_size: TP degree.
rank: Global rank of this worker.
dtype: Encoder cache dtype.
encoder_output_widths: Per-modality encoder output width, as a
producer measures it at startup. Defaults to the embedding width.
Returns:
A `VllmConfig`-specced mock carrying the configs above.
@@ -59,9 +63,15 @@ def create_ec_vllm_config(
ec_role=ec_role,
engine_id=str(uuid.uuid4()),
)
ec_transfer_config.mm_encoder_output_widths = (
dict.fromkeys(("image", "video", "audio"), _EMBED_WIDTH)
if encoder_output_widths is None
else encoder_output_widths
)
model_config = Mock(spec=ModelConfig)
model_config.dtype = dtype
model_config.get_inputs_embeds_size.return_value = _EMBED_WIDTH
vllm_config = Mock(spec=VllmConfig)
vllm_config.ec_transfer_config = ec_transfer_config