[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -22,12 +22,16 @@ class _Pos:
|
||||
self.offset = offset
|
||||
self.length = length
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1, identifier=None):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = identifier if identifier is not None else mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
self.modality = "image"
|
||||
|
||||
|
||||
class _Request:
|
||||
@@ -80,7 +84,7 @@ def _make_scheduler(
|
||||
|
||||
def _load_ids(meta) -> list[int]:
|
||||
"""Transfer ids the scheduler dispatched in this step's metadata."""
|
||||
return [transfer_id for transfer_id, _ in meta.loads.values()]
|
||||
return [transfer_id for transfer_id, _, _ in meta.loads.values()]
|
||||
|
||||
|
||||
def _load_blocks(meta, mm_hash: str) -> list[int]:
|
||||
@@ -90,7 +94,7 @@ def _load_blocks(meta, mm_hash: str) -> list[int]:
|
||||
|
||||
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
|
||||
"""Pre-populate a ready cache entry backed by real blocks."""
|
||||
s._cache.alloc(mm_hash, n_blocks)
|
||||
s._cache.alloc(mm_hash, n_blocks, (n_blocks, _BS // s._dtype.itemsize))
|
||||
s._cache.mark_ready(mm_hash)
|
||||
|
||||
|
||||
|
||||
@@ -97,10 +97,9 @@ def test_second_instance_opens_existing_file_and_shares_memory():
|
||||
r1.cleanup()
|
||||
|
||||
|
||||
def test_dense_dp_engines_get_distinct_regions(monkeypatch):
|
||||
def test_dense_dp_engines_get_distinct_regions():
|
||||
"""Dense DP engines reset data_parallel_rank to 0 but share instance_id,
|
||||
so the region must be keyed by data_parallel_index."""
|
||||
monkeypatch.setattr(common, "_get_encoder_cache_hidden_dim", lambda cfg: 8)
|
||||
instance_id = str(uuid.uuid4())
|
||||
regions = []
|
||||
try:
|
||||
@@ -113,6 +112,7 @@ def test_dense_dp_engines_get_distinct_regions(monkeypatch):
|
||||
cfg.instance_id = instance_id
|
||||
cfg.parallel_config = parallel_config
|
||||
cfg.model_config.dtype = torch.float16
|
||||
cfg.model_config.get_inputs_embeds_size.return_value = 8
|
||||
cfg.ec_transfer_config.ec_connector_extra_config = {"ec_cpu_bytes": 64}
|
||||
regions.append(common.create_ec_shared_region(cfg))
|
||||
|
||||
|
||||
@@ -91,7 +91,9 @@ def _make_region() -> ECSharedRegion:
|
||||
|
||||
|
||||
def _vllm_config(rank: int = 0) -> Mock:
|
||||
return create_ec_vllm_config(rank=rank, dtype=_DTYPE)
|
||||
config = create_ec_vllm_config(rank=rank, dtype=_DTYPE)
|
||||
config.model_config.get_inputs_embeds_size.return_value = _HIDDEN_DIM
|
||||
return config
|
||||
|
||||
|
||||
def _meta(
|
||||
@@ -105,7 +107,7 @@ def _meta(
|
||||
return ECCPUConnectorMetadata(
|
||||
saves=saves or {},
|
||||
loads={
|
||||
mm_hash: (idx, block_ids)
|
||||
mm_hash: (idx, block_ids, (len(block_ids), _HIDDEN_DIM))
|
||||
for idx, (mm_hash, block_ids) in enumerate((loads or {}).items())
|
||||
},
|
||||
)
|
||||
@@ -904,21 +906,29 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
# Build scheduler sharing the same region.
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
|
||||
scheduler = ECCPUScheduler(_vllm_config())
|
||||
config = _vllm_config()
|
||||
# DeepStack-wide output: each embedding spans four blocks.
|
||||
width = 4 * _HIDDEN_DIM
|
||||
config.ec_transfer_config.mm_encoder_output_widths = {"image": width}
|
||||
scheduler = ECCPUScheduler(config)
|
||||
|
||||
# -- Step 1: scheduler allocates, worker saves --
|
||||
n_blocks = 3
|
||||
src = torch.arange(
|
||||
n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE
|
||||
).reshape(n_blocks, _HIDDEN_DIM)
|
||||
n_tokens = 2
|
||||
src = torch.arange(n_tokens * width, dtype=_DTYPE, device=DEVICE_TYPE).reshape(
|
||||
n_tokens, width
|
||||
)
|
||||
|
||||
class _Pos:
|
||||
offset = 0
|
||||
length = n_blocks
|
||||
length = n_tokens
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
class _Feature:
|
||||
identifier = "img_001"
|
||||
mm_position = _Pos()
|
||||
modality = "image"
|
||||
|
||||
class _Request:
|
||||
request_id = "req_e2e"
|
||||
@@ -948,8 +958,9 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
scheduler.update_state_after_alloc(_Request(), 0)
|
||||
meta_load = scheduler.build_connector_meta(scheduler_output=None)
|
||||
assert "img_001" in meta_load.loads
|
||||
_, load_blocks = meta_load.loads["img_001"]
|
||||
_, load_blocks, shape = meta_load.loads["img_001"]
|
||||
assert load_blocks == meta_save.saves["img_001"]
|
||||
assert shape == tuple(src.shape)
|
||||
|
||||
load_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(load_cache, meta_load)
|
||||
|
||||
@@ -45,6 +45,34 @@ KEY = (
|
||||
SPEC = TensorSpec((2, 2), "torch.float32", 16)
|
||||
|
||||
|
||||
def test_push_shapes_follow_each_modality(monkeypatch):
|
||||
"""Each push is sized at its modality's measured encoder output width."""
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake import scheduler
|
||||
|
||||
monkeypatch.setattr(scheduler, "ensure_mooncake_available", lambda: None)
|
||||
widths = {"image": 8192, "audio": 2048}
|
||||
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths=widths)
|
||||
request = SimpleNamespace(
|
||||
request_id="mixed",
|
||||
mm_features=[SimpleNamespace(identifier=m, modality=m) for m in widths],
|
||||
get_num_encoder_embeds=lambda index: 2,
|
||||
ec_transfer_params={
|
||||
"consumer_zmq": "tcp://consumer:1234",
|
||||
"ec_items": [{"mm_hash": m, "transfer_id": m} for m in widths],
|
||||
},
|
||||
)
|
||||
instance = scheduler.ECMooncakeScheduler(config)
|
||||
try:
|
||||
for index in range(len(widths)):
|
||||
instance.update_state_after_alloc(request, index)
|
||||
metadata = instance.build_connector_meta(
|
||||
SimpleNamespace(free_encoder_mm_hashes=[], preempted_req_ids=set())
|
||||
)
|
||||
assert [spec.shape for spec in metadata.pushes] == [(2, 8192), (2, 2048)]
|
||||
finally:
|
||||
instance.close()
|
||||
|
||||
|
||||
def test_embedding_key_combines_namespace_and_identifier():
|
||||
config = create_ec_vllm_config(ec_role="ec_producer")
|
||||
config.ec_transfer_config.ec_connector_extra_config.update(
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Startup measurement of the encoder output width EC producers size by."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.distributed.ec_transfer.ec_connector import encoder_output_width as eow
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _DeepStackModel:
|
||||
"""Emits each modality at its own width, as Qwen3-Omni does."""
|
||||
|
||||
widths = {"image": 4 * 16, "audio": 16}
|
||||
|
||||
def embed_multimodal(self, *, modality):
|
||||
return [torch.zeros(3, self.widths[modality])]
|
||||
|
||||
|
||||
class _Budget:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.mm_max_toks_per_item = dict.fromkeys(_DeepStackModel.widths, 8)
|
||||
|
||||
def get_dummy_encoder_profile_inputs(self, modality, max_items_per_batch):
|
||||
assert max_items_per_batch == 1
|
||||
return [(modality, None)]
|
||||
|
||||
|
||||
def _worker(first_pp_rank=True, supports_mm=True):
|
||||
runner = SimpleNamespace(
|
||||
supports_mm_inputs=supports_mm,
|
||||
get_model=_DeepStackModel,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
pp_group = SimpleNamespace(is_first_rank=first_pp_rank)
|
||||
return SimpleNamespace(model_runner=runner, vllm_config=None), pp_group
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patched(monkeypatch):
|
||||
import vllm.distributed.parallel_state as ps
|
||||
import vllm.multimodal.encoder_budget as budget_mod
|
||||
import vllm.multimodal.utils as mm_utils
|
||||
|
||||
monkeypatch.setattr(budget_mod, "MultiModalBudget", _Budget)
|
||||
monkeypatch.setattr(
|
||||
mm_utils,
|
||||
"group_and_batch_mm_kwargs",
|
||||
lambda items, **_: iter([(items[0][0], 1, {"modality": items[0][0]})]),
|
||||
)
|
||||
return lambda group: monkeypatch.setattr(ps, "get_pp_group", lambda: group)
|
||||
|
||||
|
||||
def test_worker_reports_the_encoder_output_width_per_modality(patched):
|
||||
"""The width comes from the tensor the encoder emits, not from config."""
|
||||
worker, group = _worker()
|
||||
patched(group)
|
||||
assert eow._measure_on_worker(worker) == {"image": 64, "audio": 16}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_pp_rank, supports_mm", [(False, True), (True, False)])
|
||||
def test_ranks_without_an_encoder_report_nothing(patched, first_pp_rank, supports_mm):
|
||||
worker, group = _worker(first_pp_rank, supports_mm)
|
||||
patched(group)
|
||||
assert eow._measure_on_worker(worker) == {}
|
||||
|
||||
|
||||
def test_producer_records_the_widths_its_encoder_ranks_agree_on():
|
||||
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths={})
|
||||
reports = [{"image": 64}, {}, {"image": 64}]
|
||||
eow.measure_encoder_output_widths(config, lambda fn: reports)
|
||||
assert config.ec_transfer_config.mm_encoder_output_widths == {"image": 64}
|
||||
assert eow.get_encoder_output_width(config, "image") == 64
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reports, match",
|
||||
[
|
||||
([{}, {}], "could not measure"),
|
||||
([{"image": 64}, {"image": 16}], "disagree"),
|
||||
],
|
||||
ids=["no-encoder", "disagree"],
|
||||
)
|
||||
def test_producer_refuses_to_start_without_a_trustworthy_width(reports, match):
|
||||
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths={})
|
||||
with pytest.raises(ValueError, match=match):
|
||||
eow.measure_encoder_output_widths(config, lambda fn: reports)
|
||||
@@ -25,6 +25,9 @@ class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset, self.length = offset, length
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1, data=None):
|
||||
@@ -85,6 +88,9 @@ class _FakeTransport:
|
||||
pass
|
||||
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
def _consumer_sched(monkeypatch):
|
||||
def _region(cfg):
|
||||
return ECSharedRegion(
|
||||
@@ -95,16 +101,15 @@ def _consumer_sched(monkeypatch):
|
||||
|
||||
# ec_enable_nixl defaults to False, so this builds gate-off; the fields
|
||||
# below flip on the NIXL consumer state directly.
|
||||
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_consumer"))
|
||||
# No measured widths: a consumer takes each shape from the announcement.
|
||||
s = ECCPUScheduler(
|
||||
create_ec_vllm_config(ec_role="ec_consumer", encoder_output_widths={})
|
||||
)
|
||||
# Turn on NIXL consumer state without constructing real transports.
|
||||
s._nixl_enabled = True
|
||||
s._transport = _FakeTransport()
|
||||
s._data = None
|
||||
s._compat_hash = "c"
|
||||
# _setup_nixl normally computes these from model_config; set them
|
||||
# directly since this helper builds gate-off then flips fields on.
|
||||
s._hidden_dim = _HID
|
||||
s._element_size = _ES
|
||||
s._metadata_resolver._cache["image"] = {"image_grid_thw"}
|
||||
return s
|
||||
|
||||
@@ -115,24 +120,32 @@ def _params(mm_hash, length):
|
||||
"peer_host": "h",
|
||||
"peer_port": 1,
|
||||
"size_bytes": length * _HID * _ES,
|
||||
"shape": [length, _HID],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_new_remote_read_defers_then_completes(monkeypatch):
|
||||
"""The read is sized from the announced shape, not a local width."""
|
||||
width = 4 * _HID
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
# Route _start_xfer to our fake session instead of real ZMQ/NIXL, but
|
||||
# still reserve a real not-ready cache entry so mark_ready works.
|
||||
def _fake_start(mm_hash, info, size):
|
||||
entry = s._cache.alloc(mm_hash, 1)
|
||||
def _fake_start(mm_hash, info, size, shape):
|
||||
assert size == width * _ES
|
||||
assert shape == (1, width)
|
||||
entry = s._cache.alloc(mm_hash, size // _BS, shape)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _fake_start)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
params = _params("h1", 1)
|
||||
params["h1"]["size_bytes"] = width * _ES
|
||||
params["h1"]["shape"] = [1, width]
|
||||
req = _Request([_Feature("h1", 1)], params=params)
|
||||
|
||||
# Step 1: unseen remote item -> read started, request deferred.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
@@ -155,6 +168,7 @@ def test_new_remote_read_defers_then_completes(monkeypatch):
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta.loads
|
||||
assert meta.loads["h1"][2] == (1, width)
|
||||
assert "h1" not in s._step_completed # cleared by promote
|
||||
|
||||
# Step 3: still cached -> admitted directly, no new transfer.
|
||||
@@ -176,8 +190,8 @@ def test_failed_read_falls_back_only_with_local_input(monkeypatch, payload):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
def _fake_start(mm_hash, info, size):
|
||||
entry = s._cache.alloc(mm_hash, 1)
|
||||
def _fake_start(mm_hash, info, size, shape):
|
||||
entry = s._cache.alloc(mm_hash, 1, shape)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
@@ -230,8 +244,8 @@ def test_remote_wait_budget_survives_retries_and_long_steps(monkeypatch, retry):
|
||||
fake = _FakeSession()
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
def start(mm_hash, info, size):
|
||||
assert s._cache.alloc(mm_hash, 1) is not None
|
||||
def start(mm_hash, info, size, shape):
|
||||
assert s._cache.alloc(mm_hash, 1, shape) is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
@@ -261,8 +275,8 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
def _fake_start(mm_hash, info, size):
|
||||
entry = s._cache.alloc(mm_hash, 1)
|
||||
def _fake_start(mm_hash, info, size, shape):
|
||||
entry = s._cache.alloc(mm_hash, 1, shape)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
@@ -287,14 +301,29 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"size_fields",
|
||||
[{"size_bytes": 999}, {"size_bytes": None}, {"size_bytes": "big"}, {}],
|
||||
ids=["mismatch", "null-size", "text-size", "no-size"],
|
||||
"fields",
|
||||
[
|
||||
{"size_bytes": 999},
|
||||
{"size_bytes": None},
|
||||
{"size_bytes": "big"},
|
||||
{"size_bytes": _MISSING},
|
||||
{"shape": _MISSING},
|
||||
{"shape": [2, _HID]},
|
||||
],
|
||||
ids=[
|
||||
"size-mismatch",
|
||||
"null-size",
|
||||
"text-size",
|
||||
"no-size",
|
||||
"no-shape",
|
||||
"wrong-rows",
|
||||
],
|
||||
)
|
||||
def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
|
||||
"""Invalid remote sizes fail the request without raising in the scheduler."""
|
||||
def test_invalid_announcement_fails_the_request(monkeypatch, fields):
|
||||
"""An unusable announced size or shape fails the request, not the engine."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
announced = {"peer_host": "h", "peer_port": 1, **size_fields}
|
||||
announced = {**_params("h1", 1)["h1"], **fields}
|
||||
announced = {k: v for k, v in announced.items() if v is not _MISSING}
|
||||
req = _Request([_Feature("h1", 1)], params={"h1": announced})
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert s.take_unavailable_requests() == {"r1"}
|
||||
@@ -306,7 +335,7 @@ def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
|
||||
def test_deferral_budget_is_per_request(monkeypatch):
|
||||
"""Shared hashes have independent wait budgets and cancellation cleanup."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
|
||||
monkeypatch.setattr(s._cache, "alloc", lambda key, n, shape=None: None)
|
||||
params = _params("h1", 1)
|
||||
old = _Request([_Feature("h1", 1)], params=params, req_id="old")
|
||||
new = _Request([_Feature("h1", 1)], params=params, req_id="new")
|
||||
@@ -419,7 +448,7 @@ def test_in_flight_hash_defers_without_second_transfer(monkeypatch):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
started: list[str] = []
|
||||
|
||||
def _spy_start_xfer(mm_hash, info, size):
|
||||
def _spy_start_xfer(mm_hash, info, size, shape):
|
||||
started.append(mm_hash)
|
||||
return True
|
||||
|
||||
|
||||
@@ -16,6 +16,9 @@ class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset, self.length = offset, length
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1):
|
||||
@@ -53,11 +56,8 @@ def test_request_finished_producer_emits_params(monkeypatch):
|
||||
# Simulate NIXL-enabled producer bookkeeping without building real NIXL.
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
# _setup_nixl normally computes these from model_config; set them
|
||||
# directly since this test builds gate-off then flips fields on.
|
||||
s._hidden_dim, s._element_size = 32, 2
|
||||
# feature length=2, hidden_dim=32, element_size=2 -> 128 bytes -> 2 blocks.
|
||||
entry = s._cache.alloc("h1", 2)
|
||||
entry = s._cache.alloc("h1", 2, (2, 32))
|
||||
assert entry is not None
|
||||
s._cache.mark_ready("h1")
|
||||
|
||||
@@ -71,6 +71,7 @@ def test_request_finished_producer_emits_params(monkeypatch):
|
||||
"peer_host": "1.2.3.4",
|
||||
"peer_port": 5601,
|
||||
"size_bytes": 2 * 32 * 2,
|
||||
"shape": [2, 32],
|
||||
}
|
||||
}
|
||||
s.shutdown()
|
||||
@@ -84,8 +85,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
s._hidden_dim, s._element_size = 32, 2
|
||||
s._cache.alloc("h1", 2) # allocated but not marked ready
|
||||
s._cache.alloc("h1", 2, (2, 32)) # allocated but not marked ready
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
assert delay is False
|
||||
@@ -96,6 +96,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
|
||||
"peer_host": "1.2.3.4",
|
||||
"peer_port": 5601,
|
||||
"size_bytes": 2 * 32 * 2,
|
||||
"shape": [2, 32],
|
||||
}
|
||||
}
|
||||
s.shutdown()
|
||||
@@ -105,7 +106,6 @@ def test_request_finished_skips_unallocated_entry(monkeypatch):
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
s._hidden_dim, s._element_size = 32, 2
|
||||
# No alloc() for "h1" at all — e.g. the cache was full at save time.
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
|
||||
@@ -18,12 +18,16 @@ _EXECUTOR_BACKEND = "mp"
|
||||
_PCP_SIZE = 1
|
||||
|
||||
|
||||
_EMBED_WIDTH = 32
|
||||
|
||||
|
||||
def create_ec_vllm_config(
|
||||
*,
|
||||
ec_role: ECRole = "ec_both",
|
||||
tensor_parallel_size: int = 1,
|
||||
rank: int = 0,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
encoder_output_widths: dict[str, int] | None = None,
|
||||
) -> Mock:
|
||||
"""Build a `VllmConfig` stand-in for EC connector unit tests.
|
||||
|
||||
@@ -31,10 +35,8 @@ def create_ec_vllm_config(
|
||||
role derivation and the rank arithmetic the connector depends on behave as
|
||||
they do in production.
|
||||
|
||||
`model_config` is a stub because nothing under test needs a real one: the
|
||||
connector reads only `dtype` here, and the paths that inspect the model
|
||||
itself are not exercised. Building a real `ModelConfig` would resolve an HF
|
||||
model and inspect its architecture.
|
||||
`model_config` supplies a dtype and embedding width without resolving an
|
||||
HF model or inspecting its architecture.
|
||||
|
||||
Args:
|
||||
ec_role: EC role this instance plays. `is_ec_producer` and
|
||||
@@ -42,6 +44,8 @@ def create_ec_vllm_config(
|
||||
tensor_parallel_size: TP degree.
|
||||
rank: Global rank of this worker.
|
||||
dtype: Encoder cache dtype.
|
||||
encoder_output_widths: Per-modality encoder output width, as a
|
||||
producer measures it at startup. Defaults to the embedding width.
|
||||
|
||||
Returns:
|
||||
A `VllmConfig`-specced mock carrying the configs above.
|
||||
@@ -59,9 +63,15 @@ def create_ec_vllm_config(
|
||||
ec_role=ec_role,
|
||||
engine_id=str(uuid.uuid4()),
|
||||
)
|
||||
ec_transfer_config.mm_encoder_output_widths = (
|
||||
dict.fromkeys(("image", "video", "audio"), _EMBED_WIDTH)
|
||||
if encoder_output_widths is None
|
||||
else encoder_output_widths
|
||||
)
|
||||
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.dtype = dtype
|
||||
model_config.get_inputs_embeds_size.return_value = _EMBED_WIDTH
|
||||
|
||||
vllm_config = Mock(spec=VllmConfig)
|
||||
vllm_config.ec_transfer_config = ec_transfer_config
|
||||
|
||||
Reference in New Issue
Block a user