[EC Connector] EC Offloading Connector use events instead of StepTracker (#49994)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
omerpaz95
2026-08-27 03:41:46 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 94d96e2446
commit f8d38fbc87
15 changed files with 1054 additions and 650 deletions
View File
@@ -21,13 +21,15 @@ def _cache(num_blocks: int = 8) -> EmbeddingCache:
# ── alloc ────────────────────────────────────────────────────────────────────
def test_alloc_returns_unique_block_ids():
def test_alloc_returns_unique_ascending_block_ids():
cache = _cache()
entry = cache.alloc("a", 3)
assert entry is not None
assert len(entry.block_ids) == 3
assert len(set(entry.block_ids)) == 3
assert all(0 <= i < 8 for i in entry.block_ids)
# Ascending order lets consumers detect which blocks are adjacent.
assert list(entry.block_ids) == sorted(entry.block_ids)
def test_alloc_returns_tuple():
@@ -161,6 +163,33 @@ def test_unpin_asserts_on_unpinned():
cache.unpin("a")
# ── has_held_entries ─────────────────────────────────────────────────────────
def test_has_held_entries_empty_cache():
cache = _cache()
assert cache.has_held_entries() is False
def test_has_held_entries_true_while_not_ready():
cache = _cache()
cache.alloc("a", 2) # not ready → held
assert cache.has_held_entries() is True
cache.mark_ready("a") # ready + unpinned → settled
assert cache.has_held_entries() is False
def test_has_held_entries_true_while_pinned():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
assert cache.has_held_entries() is False
cache.pin("a") # pinned → held
assert cache.has_held_entries() is True
cache.unpin("a")
assert cache.has_held_entries() is False
# ── eviction ─────────────────────────────────────────────────────────────────
@@ -3,6 +3,11 @@
import uuid
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.config.ec_transfer import ECRole
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
@@ -37,13 +42,26 @@ class _Request:
self.request_id = request_id
class _WorkerOutput:
"""Stand-in for ECConnectorOutput carrying a completion report.
`saves` are mm_hashes; `loads` are transfer ids. One entry per reporting
rank, so a list holding the same id twice is two ranks reporting it.
"""
def __init__(self, *, saves=None, loads=None):
self.ec_connector_worker_meta = ECCPUWorkerMetadata(
completed_saves=list(saves or []),
completed_loads=list(loads or []),
)
def _make_scheduler(
monkeypatch,
num_blocks=_N,
max_concurrent_batches=1,
*,
is_producer=True,
is_consumer=True,
ec_role: ECRole = "ec_both",
tensor_parallel_size=1,
) -> ECCPUScheduler:
region = ECSharedRegion(
engine_id=str(uuid.uuid4()),
@@ -52,27 +70,22 @@ def _make_scheduler(
)
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
_is_prod = is_producer
_is_cons = is_consumer
_mcb = max_concurrent_batches
class _EC:
is_ec_producer = _is_prod
is_ec_consumer = _is_cons
class _Cfg:
ec_transfer_config = _EC()
@property
def max_concurrent_batches(self):
return _mcb
return ECCPUScheduler(_Cfg())
return ECCPUScheduler(
create_ec_vllm_config(
ec_role=ec_role,
tensor_parallel_size=tensor_parallel_size,
)
)
class _SchedulerOutput:
def __init__(self, finished_req_ids=None):
self.finished_req_ids = finished_req_ids or set()
def _load_ids(meta) -> list[int]:
"""Transfer ids the scheduler dispatched in this step's metadata."""
return [transfer_id for transfer_id, _ in meta.loads.values()]
def _load_blocks(meta, mm_hash: str) -> list[int]:
"""Block ids the scheduler dispatched for one load."""
return meta.loads[mm_hash][1]
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
@@ -83,7 +96,9 @@ def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
def test_offload_reuse_cycle(monkeypatch):
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1)])
# Two blocks, so the reload also pins down that every allocated block id
# comes back in meta.loads, not just the first.
req = _Request([_Feature("h1", length=2)])
# Step A: first sight — allocate and save.
assert s.has_cache_item("h1") is False
@@ -92,24 +107,24 @@ def test_offload_reuse_cycle(monkeypatch):
meta_a = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_a.saves
assert meta_a.loads == {}
# Not ready yet (max_concurrent_batches=1 means ready next step).
# Not ready until the worker reports the save memcpy complete.
assert s.has_cache_item("h1") is False
# Step B: _step_readiness marks h1 ready (no new allocs).
s.build_connector_meta(scheduler_output=None)
# Worker reports the save copy finished → entry becomes ready.
s.update_connector_output(_WorkerOutput(saves=["h1"]))
assert s.has_cache_item("h1") is True
# Step C: reload now works — entry is ready at update_state_after_alloc time.
s.update_state_after_alloc(req, 0)
meta_c = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_c.loads
assert meta_c.loads["h1"] == meta_a.saves["h1"]
assert _load_blocks(meta_c, "h1") == meta_a.saves["h1"]
s.shutdown()
def test_has_cache_item_false_when_not_consumer(monkeypatch):
s = _make_scheduler(monkeypatch, is_consumer=False)
s = _make_scheduler(monkeypatch, ec_role="ec_producer")
assert s.has_cache_item("anything") is False
s.shutdown()
@@ -123,10 +138,10 @@ def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
meta = s.build_connector_meta(scheduler_output=None)
assert "ENC_KEY" in meta.saves
assert "PROC_KEY" not in meta.saves
# Not ready yet (1 step delay).
# Not ready until the completion report.
assert s.has_cache_item("ENC_KEY") is False
# After another step it becomes ready — keyed on identifier.
s.build_connector_meta(scheduler_output=None)
# After the report it becomes ready — keyed on identifier.
s.update_connector_output(_WorkerOutput(saves=["ENC_KEY"]))
assert s.has_cache_item("ENC_KEY") is True
assert s.has_cache_item("PROC_KEY") is False
s.shutdown()
@@ -136,13 +151,11 @@ def test_cpu_region_fifo_eviction(monkeypatch):
# Region holds exactly 2 one-block encodings.
s = _make_scheduler(monkeypatch, num_blocks=2)
# Fill with A and B, marking each ready.
# Fill with A and B, marking each ready via a completion report.
for h in ("A", "B"):
s.update_state_after_alloc(_Request([_Feature(h, length=1)]), 0)
s.build_connector_meta(scheduler_output=None)
# After the step above, A's step was called in B's build. A is ready.
# B was submitted in the last build, so one more step makes it ready.
s.build_connector_meta(scheduler_output=None) # marks B ready
s.update_connector_output(_WorkerOutput(saves=[h]))
assert s.has_cache_item("A") is True
assert s.has_cache_item("B") is True
@@ -161,8 +174,8 @@ def test_multiple_mm_items_per_request(monkeypatch):
meta = s.build_connector_meta(scheduler_output=None)
assert set(meta.saves) == {"h1", "h2"}
# Step delay: mark both ready.
s.build_connector_meta(scheduler_output=None)
# Worker reports both saves complete.
s.update_connector_output(_WorkerOutput(saves=["h1", "h2"]))
assert s.has_cache_item("h1") is True
assert s.has_cache_item("h2") is True
@@ -174,26 +187,6 @@ def test_multiple_mm_items_per_request(monkeypatch):
s.shutdown()
def test_load_returns_correct_block_ids(monkeypatch):
"""meta.loads must contain the same block IDs that meta.saves
allocated — verified through the public API only."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("a", length=2)])
s.update_state_after_alloc(req, 0)
meta_save = s.build_connector_meta(scheduler_output=None)
assert "a" in meta_save.saves
# Readiness delay.
s.build_connector_meta(scheduler_output=None)
# Reload.
s.update_state_after_alloc(req, 0)
meta_load = s.build_connector_meta(scheduler_output=None)
assert meta_load.loads["a"] == meta_save.saves["a"]
s.shutdown()
def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=2)
@@ -228,10 +221,10 @@ def test_load_not_emitted_for_uncached_entry(monkeypatch):
s.shutdown()
def test_delayed_unpin_protects_blocks_during_worker_read(monkeypatch):
"""Blocks being loaded must not be evictable until the delayed unpin
fires. Observable: a save that needs eviction cannot reclaim a
still-pinned loaded entry."""
def test_load_pin_protects_blocks_until_completion(monkeypatch):
"""Loaded blocks stay pinned until the worker reports the load memcpy
complete. Observable: a save that needs eviction cannot reclaim a
still-pinned loaded entry, but can once the completion report arrives."""
# 2 blocks, both occupied by "a" (ready).
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
@@ -240,94 +233,79 @@ def test_delayed_unpin_protects_blocks_during_worker_read(monkeypatch):
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "a" in meta.loads
(transfer_id,) = _load_ids(meta)
# Try to save "b" (needs 2 blocks, but "a" is pinned) — must fail.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta2.saves # can't evict pinned "a"
# After the delay, "a" unpins. Now "b" can evict it.
# The one participant reports the load complete → "a" unpins. Now "b" can
# evict it.
s.update_connector_output(_WorkerOutput(loads=[transfer_id]))
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta3 = s.build_connector_meta(scheduler_output=None)
assert "b" in meta3.saves
s.shutdown()
def test_delayed_unpin_depth_2(monkeypatch):
"""With max_concurrent_batches=2, loaded blocks stay pinned for 2 steps
after build. Observable via eviction failure."""
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2)
_seed_cached(s, "a", n_blocks=2)
# Load "a" → pinned.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
s.build_connector_meta(scheduler_output=None)
# Step 1 after load: still pinned — can't evict for "b".
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta1 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta1.saves
# Step 2 after load: still pinned.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta2.saves
# Step 3: unpin fires, now eviction works.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta3 = s.build_connector_meta(scheduler_output=None)
assert "b" in meta3.saves
s.shutdown()
def _can_evict(s: ECCPUScheduler, mm_hash: str, n_blocks: int) -> bool:
"""Whether a new save of `mm_hash` can claim `n_blocks`, i.e. whether the
blocks currently held by other entries are reclaimable."""
s.update_state_after_alloc(_Request([_Feature(mm_hash, length=n_blocks)]), 0)
meta = s.build_connector_meta(scheduler_output=None)
return mm_hash in meta.saves
def test_shutdown_unpins_pending_loads(monkeypatch):
"""Shutdown must unpin entries still in _pending_loads (not yet drained
to the unpin deque). Observable: after shutdown, the entry is evictable
(alloc can reclaim its blocks)."""
s = _make_scheduler(monkeypatch, num_blocks=2)
def test_load_needs_every_participant_before_unpin(monkeypatch):
"""With two participating ranks, one report must not release the pin.
This is the multi-rank case: each rank copies the same CPU blocks into its
own GPU memory, so the blocks stay in use until the last rank is done.
"""
s = _make_scheduler(monkeypatch, num_blocks=2, tensor_parallel_size=2)
_seed_cached(s, "a", n_blocks=2)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
# Don't call build_connector_meta — "a" stays in _pending_loads.
(transfer_id,) = _load_ids(s.build_connector_meta(scheduler_output=None))
# First rank reports: still in use by the second, so "a" must hold.
s.update_connector_output(_WorkerOutput(loads=[transfer_id]))
assert _can_evict(s, "b", 2) is False
assert s.has_pending_push_work() is True
# Second rank reports: now it is genuinely free.
s.update_connector_output(_WorkerOutput(loads=[transfer_id]))
assert s.has_pending_push_work() is False
assert _can_evict(s, "c", 2) is True
s.shutdown()
# After shutdown, scheduler reports no items (consumer role is off).
assert s.has_cache_item("a") is False
def test_late_report_does_not_release_a_later_load_of_same_hash(monkeypatch):
"""A straggling report from one dispatch must not release the pin taken by
a subsequent dispatch of the same mm_hash.
def test_shutdown_unpins_deferred_unpin_queue(monkeypatch):
"""Shutdown must unpin entries queued in the deferred unpin deque
(already drained from _pending_loads but not yet expired)."""
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2)
This is what the transfer id is for: mm_hash identifies a cache entry, not
a transfer, so it cannot tell the two dispatches apart.
"""
s = _make_scheduler(monkeypatch, num_blocks=2, tensor_parallel_size=2)
_seed_cached(s, "a", n_blocks=2)
# Load "a" and build — moves to unpin deque.
# First dispatch, fully reported by both ranks → pin released.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
s.build_connector_meta(scheduler_output=None)
(first_id,) = _load_ids(s.build_connector_meta(scheduler_output=None))
s.update_connector_output(_WorkerOutput(loads=[first_id, first_id]))
assert s.has_pending_push_work() is False
# "a" is in the deferred unpin deque (depth=2, not yet expired).
# Shutdown must unpin it.
s.shutdown()
assert s.has_cache_item("a") is False
# Second dispatch of the same hash takes a fresh pin.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
(second_id,) = _load_ids(s.build_connector_meta(scheduler_output=None))
assert second_id != first_id
def test_eviction_skips_entry_pinned_by_pending_load(monkeypatch):
"""A loaded entry still within the unpin delay window must survive
eviction attempts from new saves."""
# 2 blocks: A occupies both, ready.
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "A", n_blocks=2)
# Load A → pinned.
s.update_state_after_alloc(_Request([_Feature("A")]), 0)
meta_load = s.build_connector_meta(scheduler_output=None)
assert "A" in meta_load.loads
# Try to save C (needs 2 blocks) — can't evict pinned A.
s.update_state_after_alloc(_Request([_Feature("C", length=2)]), 0)
meta_save = s.build_connector_meta(scheduler_output=None)
assert "C" not in meta_save.saves
assert s.has_cache_item("A") is True # survived
# A replayed report for the first dispatch must be ignored entirely.
s.update_connector_output(_WorkerOutput(loads=[first_id, first_id]))
assert s.has_pending_push_work() is True
assert _can_evict(s, "b", 2) is False
s.shutdown()
@@ -350,30 +328,10 @@ def test_region_full_skips_save_and_never_blocks(monkeypatch):
s.shutdown()
def test_step_readiness_depth_2(monkeypatch):
"""With max_concurrent_batches=2, entries need 2 steps to become ready."""
s = _make_scheduler(monkeypatch, max_concurrent_batches=2)
req = _Request([_Feature("h1", length=1)])
s.update_state_after_alloc(req, 0)
meta_0 = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_0.saves
assert s.has_cache_item("h1") is False
# One step later: still not ready (depth=2).
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is False
# Two steps later: now ready.
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is True
s.shutdown()
def test_producer_only_never_emits_loads(monkeypatch):
"""A producer-only scheduler must never populate meta.loads, even when
entries are ready."""
s = _make_scheduler(monkeypatch, is_consumer=False)
s = _make_scheduler(monkeypatch, ec_role="ec_producer")
req = _Request([_Feature("h1", length=1)])
s.update_state_after_alloc(req, 0)
@@ -381,8 +339,7 @@ def test_producer_only_never_emits_loads(monkeypatch):
assert "h1" in meta.saves
assert meta.loads == {}
# Mark ready.
s.build_connector_meta(scheduler_output=None)
s.update_connector_output(_WorkerOutput(saves=["h1"]))
# Even if we re-encounter the same feature, no load.
s.update_state_after_alloc(req, 0)
@@ -393,7 +350,7 @@ def test_producer_only_never_emits_loads(monkeypatch):
def test_consumer_only_never_emits_saves(monkeypatch):
"""A consumer-only scheduler must never populate meta.saves."""
s = _make_scheduler(monkeypatch, is_producer=False)
s = _make_scheduler(monkeypatch, ec_role="ec_consumer")
_seed_cached(s, "a", n_blocks=1)
req = _Request([_Feature("a", length=1)])
@@ -406,7 +363,7 @@ def test_consumer_only_never_emits_saves(monkeypatch):
def test_consumer_only_has_cache_item(monkeypatch):
"""Consumer-only: has_cache_item works normally for ready entries."""
s = _make_scheduler(monkeypatch, is_producer=False)
s = _make_scheduler(monkeypatch, ec_role="ec_consumer")
assert s.has_cache_item("a") is False
_seed_cached(s, "a", n_blocks=1)
assert s.has_cache_item("a") is True
@@ -431,50 +388,75 @@ def test_save_not_emitted_for_already_cached_entry(monkeypatch):
s.shutdown()
# ── first-finish fast-path (idle engine) ────────────────────────────────────
# ── completion reporting (update_connector_output) ──────────────────────────
def test_first_finish_marks_ready_immediately(monkeypatch):
"""When the originating request finishes, the save entry becomes ready
without waiting for the step-count delay."""
s = _make_scheduler(monkeypatch, max_concurrent_batches=5)
req = _Request([_Feature("h1", length=1)], request_id="req_x")
def test_update_connector_output_ignores_foreign_meta(monkeypatch):
"""A worker output whose payload is not an ECCPUWorkerMetadata is a
no-op — no cache mutation, no crash."""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=1)
s.update_state_after_alloc(req, 0)
class _Foreign:
ec_connector_worker_meta = object()
s.update_connector_output(_Foreign()) # must not raise
assert s.has_cache_item("a") is True
s.shutdown()
def test_update_connector_output_ignores_unknown_report(monkeypatch):
"""Reports for a never-seen save hash or a transfer id this scheduler never
dispatched are dropped rather than mutating unrelated state."""
s = _make_scheduler(monkeypatch)
s.update_connector_output(_WorkerOutput(saves=["gone"], loads=[4242]))
assert s.has_cache_item("gone") is False
s.shutdown()
# ── has_pending_push_work ───────────────────────────────────────────────────
def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
"""A dispatched-but-unconfirmed save keeps push work pending until the
completion report marks it ready."""
s = _make_scheduler(monkeypatch)
assert s.has_pending_push_work() is False
s.update_state_after_alloc(_Request([_Feature("h1", length=1)]), 0)
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is False
assert s.has_pending_push_work() is True # save not yet confirmed
# Request finishes — readiness should fire immediately.
s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_x"}))
assert s.has_cache_item("h1") is True
s.update_connector_output(_WorkerOutput(saves=["h1"]))
assert s.has_pending_push_work() is False
s.shutdown()
def test_first_finish_unpins_immediately(monkeypatch):
"""When the originating request finishes, the load entry is unpinned
without waiting for the step-count delay."""
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=5)
_seed_cached(s, "a", n_blocks=2)
def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
"""A dispatched-but-unconfirmed load keeps push work pending until the
unpin completion report arrives."""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=1)
assert s.has_pending_push_work() is False # ready + unpinned
req = _Request([_Feature("a")], request_id="req_y")
s.update_state_after_alloc(req, 0)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "a" in meta.loads
assert s.has_pending_push_work() is True # pinned, awaiting unpin
# "a" is pinned. Try to evict — must fail.
s.update_state_after_alloc(
_Request([_Feature("b", length=2)], request_id="req_z"), 0
)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta2.saves
# req_y finishes — unpin fires in build_connector_meta (fast-path).
s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_y"}))
# Next step: "a" is now evictable, so "b" can be allocated.
s.update_state_after_alloc(
_Request([_Feature("b", length=2)], request_id="req_w"), 0
)
meta3 = s.build_connector_meta(scheduler_output=None)
assert "b" in meta3.saves
s.update_connector_output(_WorkerOutput(loads=_load_ids(meta)))
assert s.has_pending_push_work() is False
s.shutdown()
# ── shutdown ────────────────────────────────────────────────────────────────
def test_shutdown_disables_roles_and_cleans_region(monkeypatch):
"""After shutdown the scheduler serves no items and the region is
cleaned up."""
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
assert s.has_cache_item("a") is True
s.shutdown()
assert s.has_cache_item("a") is False # consumer role disabled
@@ -1,162 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for StepTracker."""
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
StepTracker,
)
def test_basic_step_count_expiry_depth_1():
"""With max_concurrent_batches=1, entry expires after 1 step."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
# Step 1: entry was added, committed to slot. Deque fills to maxlen=1,
# but no expiry yet (the entry is in the slot just committed).
result = t.step(set())
assert result == []
# Step 2: deque is full, oldest slot (containing h1) expires.
result = t.step(set())
assert result == ["h1"]
def test_basic_step_count_expiry_depth_2():
"""With max_concurrent_batches=2, entry expires after 2 steps."""
t = StepTracker(max_concurrent_batches=2)
t.add("h1", "req_a")
assert t.step(set()) == [] # step 1: committed
assert t.step(set()) == [] # step 2: deque fills but h1 in newer slot
assert t.step(set()) == ["h1"] # step 3: h1's slot is oldest, expires
def test_first_finish_fast_path():
"""Entry is returned immediately when request_id appears in finished."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
result = t.step({"req_a"})
assert result == ["h1"]
def test_no_double_processing_fast_path_then_expiry():
"""Entry fast-pathed should NOT fire again on deque expiry."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
# Fast-path fires h1.
result = t.step({"req_a"})
assert result == ["h1"]
# Normal expiry would fire now, but h1 is already processed.
result = t.step(set())
assert result == []
def test_no_double_processing_expiry_then_fast_path():
"""Entry expired via step-count should NOT fire again via fast-path."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
t.step(set()) # commit
result = t.step(set()) # expiry fires h1
assert result == ["h1"]
# req_a finishes later — must not re-fire.
result = t.step({"req_a"})
assert result == []
def test_multiplicity_same_hash_different_requests():
"""Two adds of the same mm_hash (different requests) → returned twice."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
t.add("h1", "req_b")
# Both requests finish in the same step.
result = t.step({"req_a", "req_b"})
assert sorted(result) == ["h1", "h1"]
def test_mixed_expiry_and_fast_path():
"""Some entries expire, others fast-path, in the same step."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
t.step(set()) # commit h1's slot
# Add h2 in the next step.
t.add("h2", "req_b")
# This step: h1's slot expires AND req_b finishes.
result = t.step({"req_b"})
assert set(result) == {"h1", "h2"}
def test_drain_all_returns_unprocessed():
"""drain_all returns everything pending and clears state."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
t.add("h2", "req_b")
t.step(set()) # commit both
t.add("h3", "req_c") # in _current, not yet committed
result = t.drain_all()
assert set(result) == {"h1", "h2", "h3"}
# After drain, everything is empty.
assert t.step(set()) == []
assert t.drain_all() == []
def test_drain_all_skips_already_processed():
"""drain_all should not re-return entries already fast-pathed."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
t.add("h2", "req_b")
# Fast-path h1.
result = t.step({"req_a"})
assert result == ["h1"]
# drain should only return h2.
result = t.drain_all()
assert result == ["h2"]
def test_empty_step_returns_nothing():
"""step() with nothing pending returns empty list."""
t = StepTracker(max_concurrent_batches=2)
assert t.step(set()) == []
assert t.step({"req_x"}) == []
def test_add_after_step_goes_to_next_slot():
"""Entries added after a step go into the next slot, not the current."""
t = StepTracker(max_concurrent_batches=1)
t.step(set()) # empty step
t.add("h1", "req_a")
# h1 is in _current. Next step commits it.
t.step(set())
# Now the deque is full (1 slot with h1). Next step expires it.
result = t.step(set())
assert result == ["h1"]
def test_fast_path_on_current_entries():
"""Entries still in _current (not yet committed) can be fast-pathed."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
# req_a finishes in the same step h1 was added.
result = t.step({"req_a"})
assert result == ["h1"]
# Entry is committed to slot but marked processed — won't re-fire.
for _ in range(5):
assert t.step(set()) == []
@@ -22,14 +22,16 @@ Mocking policy
import contextlib
import logging
import time
import uuid
from collections import deque
from unittest.mock import MagicMock, Mock, patch
import numpy as np
import pytest
import torch
from vllm.config import VllmConfig
from vllm.config.parallel import ParallelConfig
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
)
@@ -55,6 +57,18 @@ _requires_accelerator = pytest.mark.skipif(
reason="exercises real accelerator stream/event coordination in ECCPUWorker",
)
# The memory-lifetime tests hold a copy back with `torch.cuda._sleep`, which the
# CUDA-like platforms provide but XPU does not.
_requires_cuda_alike = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="stalling the compute stream requires a CUDA-like platform",
)
# Cycles to stall the compute stream for. Must comfortably outlast the host-side
# work the lifetime tests do while a copy is deliberately held back — ~0.5 s on
# a current datacenter GPU.
_STALL_CYCLES = 1_000_000_000
def _make_region() -> ECSharedRegion:
"""Fresh region backed by a real per-test mmap file."""
@@ -66,18 +80,28 @@ def _make_region() -> ECSharedRegion:
def _vllm_config(rank: int = 0) -> Mock:
cfg = Mock(spec=VllmConfig)
cfg.parallel_config = Mock(spec=ParallelConfig)
cfg.parallel_config.rank = rank
cfg.model_config = Mock()
cfg.model_config.dtype = _DTYPE
return cfg
return create_ec_vllm_config(rank=rank, dtype=_DTYPE)
def _meta(
*, saves: dict | None = None, loads: dict | None = None
) -> ECCPUConnectorMetadata:
return ECCPUConnectorMetadata(saves=saves or {}, loads=loads or {})
"""Build step metadata from plain `{mm_hash: block_ids}` dicts.
Load transfer ids are synthesized here so tests can keep naming loads by
mm_hash; `_load_id` recovers the id the worker will report.
"""
return ECCPUConnectorMetadata(
saves=saves or {},
loads={
mm_hash: (idx, block_ids)
for idx, (mm_hash, block_ids) in enumerate((loads or {}).items())
},
)
def _load_id(meta: ECCPUConnectorMetadata, mm_hash: str) -> int:
return meta.loads[mm_hash][0]
# ── fixtures ─────────────────────────────────────────────────────────────────
@@ -136,19 +160,52 @@ def make_worker():
region.cleanup()
def _wait_for_completion(
worker: ECCPUWorker, expected, direction: str, timeout_s: float = 30.0
) -> None:
"""Poll ``build_connector_worker_meta`` until ``expected`` is reported done.
Saves are reported by mm_hash, loads by transfer id.
"""
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
meta = worker.build_connector_worker_meta()
if meta is not None and expected in getattr(meta, f"completed_{direction}"):
return
time.sleep(0.001)
raise AssertionError(f"{direction} for {expected!r} never reported complete")
def _warm_up_stream_pools(worker: ECCPUWorker) -> None:
"""Run one throwaway save so the stream and event pools are populated.
The first CUDA stream creation in a process blocks the host for ~0.5 s on
driver initialization, which would swallow the stall the lifetime tests
below install. One flush puts a stream and its events in the worker's pools,
matching the steady state every step after the first sees.
"""
scratch = torch.zeros(_HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE)
worker.save_caches({"w": scratch}, "w", _meta(saves={"w": [0]}))
worker.flush_saves()
_wait_for_completion(worker, "w", "saves")
assert worker._stream_pool, "warm-up did not recycle a stream"
# ── save_caches ──────────────────────────────────────────────────────────────
@_requires_accelerator
@pytest.mark.parametrize(
"n_elements,n_blocks",
"n_elements,block_ids",
[
(3 * _HIDDEN_DIM, 3), # exact fit: 3 full blocks, no remainder
(3 * _HIDDEN_DIM + 4, 4), # partial last block: 4 fp16 trail in slot 3
(3 * _HIDDEN_DIM, [7, 2, 5]), # exact fit: 3 full blocks, no remainder
(3 * _HIDDEN_DIM + 4, [7, 2, 5, 0]), # partial last block: 4 fp16 trail
(3 * _HIDDEN_DIM, [2, 3, 4]), # adjacent: one coalesced copy
(3 * _HIDDEN_DIM + 4, [2, 3, 4, 6]), # coalesced run, then a gap
],
ids=["exact-fit", "partial-last-block"],
ids=["exact-fit", "partial-last-block", "adjacent-blocks", "run-then-gap"],
)
def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks):
def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, block_ids):
"""``save_caches`` + ``flush_saves`` copies the source GPU tensor's bytes
into the block IDs named by ``meta.saves``, in the order they appear,
via a single batched GPU→CPU copy.
@@ -156,6 +213,8 @@ def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks
When ``total_bytes`` is not a multiple of ``block_size_bytes`` the last
block is partially written; the unwritten tail must remain whatever
was there before. Blocks not named in ``meta.saves`` must be untouched.
Blocks adjacent in the region share one descriptor, which must land the
same bytes in the same blocks as one descriptor per block does.
"""
worker = make_worker()
sentinel = 0x5A
@@ -165,9 +224,9 @@ def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks
expected_bytes = src.cpu().reshape(-1).view(torch.uint8)
total_bytes = n_elements * _DTYPE.itemsize
block_ids = [7, 2, 5, 0][:n_blocks]
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
worker.flush_saves()
_wait_for_completion(worker, "h", "saves")
for slot, block_idx in enumerate(block_ids):
block_byte_start = slot * _BLOCK_SIZE_BYTES
@@ -189,6 +248,42 @@ def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks
)
@_requires_accelerator
@pytest.mark.parametrize(
"n_elements,block_ids,expected_sizes",
[
(3 * _HIDDEN_DIM, [2, 3, 4], [3 * _BLOCK_SIZE_BYTES]),
(3 * _HIDDEN_DIM, [7, 2, 5], [_BLOCK_SIZE_BYTES] * 3),
(3 * _HIDDEN_DIM, [2, 3, 6], [2 * _BLOCK_SIZE_BYTES, _BLOCK_SIZE_BYTES]),
(2 * _HIDDEN_DIM + 4, [2, 3, 4], [(2 * _HIDDEN_DIM + 4) * _DTYPE.itemsize]),
(_HIDDEN_DIM, [2, 3, 4], [_BLOCK_SIZE_BYTES]),
],
ids=[
"one-run",
"no-run",
"run-then-gap",
"run-stops-at-output",
"blocks-past-output-skipped",
],
)
def test_save_caches_coalesces_only_adjacent_blocks(
make_worker, n_elements, block_ids, expected_sizes
):
"""One descriptor per run of region-adjacent blocks, not one per block.
Per-descriptor overhead dominates these copies, so the byte tests above
would pass either way; this pins the descriptor layout that makes them
fast. A coalesced copy also stops at the encoder output, since entries are
sized by placeholder count, which can exceed the number of embeddings.
"""
worker = make_worker()
src = torch.arange(n_elements, dtype=_DTYPE, device=DEVICE_TYPE)
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
assert worker._save_bufs.sizes[: worker._save_count].tolist() == expected_sizes
def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
"""When the scheduler hasn't pre-allocated blocks for ``mm_hash``,
``save_caches`` + ``flush_saves`` is a pure no-op."""
@@ -245,6 +340,7 @@ def test_save_caches_batches_multiple_hashes(make_worker):
worker.save_caches(cache, "a", _meta(saves={"a": [1], "b": [3]}))
worker.save_caches(cache, "b", _meta(saves={"a": [1], "b": [3]}))
worker.flush_saves()
_wait_for_completion(worker, "a", "saves")
expected_a = src_a.cpu().view(torch.uint8)
expected_b = src_b.cpu().view(torch.uint8)
@@ -258,6 +354,50 @@ def test_save_caches_batches_multiple_hashes(make_worker):
assert torch.all(worker._region.blocks[idx] == sentinel)
class _HighAddress:
"""A device tensor reporting an address with the high bit set.
XPU USM allocations live up there and no host allocator will hand out such
an address, so naming one is the only way to reach the arithmetic. Every
attribute other than the address comes from the real tensor.
"""
def __init__(self, real: torch.Tensor, address: int):
self._real = real
self._address = address
def __getattr__(self, name):
return getattr(self._real, name)
def view(self, *args):
return self
def data_ptr(self) -> int:
return self._address
@_requires_accelerator
def test_save_descriptors_hold_high_bit_source_addresses(make_worker):
"""Source addresses >= 2**63 survive into the descriptor buffers."""
worker = make_worker()
real = torch.zeros(2 * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE)
address = 2**64 - 8 * _BLOCK_SIZE_BYTES
# Two runs, so the second descriptor's address is an offset off the base.
worker.save_caches(
{"a": _HighAddress(real, address)}, "a", _meta(saves={"a": [1, 5]})
)
assert worker._save_count == 2
stored = worker._save_bufs.src_np[:2].astype(np.uint64)
assert list(stored) == [address, address + _BLOCK_SIZE_BYTES]
# The batch describes an address nothing may read from; drop it unflushed.
worker._save_bufs = None
worker._save_count = 0
worker._save_mm_hashes = []
# ── start_load_caches ────────────────────────────────────────────────────────
@@ -288,19 +428,31 @@ def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker
@_requires_accelerator
def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker):
"""If ``encoder_cache`` already holds the ``mm_hash``, the worker must
not overwrite it."""
@pytest.mark.parametrize(
"loads,expected_sizes",
[
({"a": [2, 3], "b": [4, 5]}, [4 * _BLOCK_SIZE_BYTES]),
({"b": [4, 5], "a": [2, 3]}, [4 * _BLOCK_SIZE_BYTES]),
({"a": [2, 3], "b": [6, 7]}, [2 * _BLOCK_SIZE_BYTES] * 2),
],
ids=["adjacent-entries", "adjacent-entries-reversed", "gap-between-entries"],
)
def test_start_load_caches_coalesces_across_entries(make_worker, loads, expected_sizes):
"""Blocks adjacent in the region share a descriptor across entries too.
Entries are ordered by first block, so whether they coalesce follows from
the region layout rather than the order the scheduler dispatched them.
"""
worker = make_worker()
worker._region.blocks[0].fill_(0x42)
captured: list[list[int]] = []
sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device=DEVICE_TYPE)
encoder_cache = {"h": sentinel}
worker.start_load_caches(encoder_cache, _meta(loads={"h": [0]}))
with patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.swap_blocks_batch",
side_effect=lambda src, dst, sizes, **kw: captured.append(sizes.tolist()),
):
worker.start_load_caches({}, _meta(loads=loads))
assert encoder_cache["h"] is sentinel, (
"existing encoder_cache entry must not be replaced"
)
assert captured == [expected_sizes]
@_requires_accelerator
@@ -315,30 +467,34 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker):
@_requires_accelerator
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
"""Cached entries are preserved while new ones are loaded."""
"""Every hash in ``meta.loads`` is copied from mmap, including one whose
key is already resident in ``encoder_cache`` — there is no worker-side
skip."""
worker = make_worker()
n_blocks = 2
n_blocks = 3
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
n_blocks, _HIDDEN_DIM
)
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
new_block_ids = [4, 2]
for i, idx in enumerate(new_block_ids):
block_ids = [0, 4, 2]
for i, idx in enumerate(block_ids):
worker._region.blocks[idx].copy_(src_int8[i])
cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device=DEVICE_TYPE)
encoder_cache: dict[str, torch.Tensor] = {"cached_h": cached_tensor}
worker.start_load_caches(
encoder_cache,
_meta(loads={"cached_h": [0], "new_h": new_block_ids}),
_meta(loads={"resident_h": [block_ids[0]], "new_h": block_ids[1:]}),
)
assert encoder_cache["cached_h"] is cached_tensor
assert "new_h" in encoder_cache
out = encoder_cache["new_h"]
assert out.shape == (n_blocks, _HIDDEN_DIM)
assert out.dtype == _DTYPE
assert torch.equal(out.cpu(), src_orig)
resident = encoder_cache["resident_h"]
assert resident is not cached_tensor, "resident entry must be reloaded, not skipped"
assert torch.equal(resident.cpu(), src_orig[:1])
new = encoder_cache["new_h"]
assert new.shape == (2, _HIDDEN_DIM)
assert new.dtype == _DTYPE
assert torch.equal(new.cpu(), src_orig[1:])
@_requires_accelerator
@@ -382,6 +538,7 @@ def test_save_then_load_round_trips_bytes(make_worker):
).reshape(n_blocks, _HIDDEN_DIM)
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
worker.flush_saves()
_wait_for_completion(worker, "h", "saves")
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
@@ -392,18 +549,133 @@ def test_save_then_load_round_trips_bytes(make_worker):
assert torch.equal(out.cpu(), src.cpu())
# ── memory lifetime across in-flight copies ─────────────────────────────────
@_requires_cuda_alike
def test_save_survives_encoder_cache_free_before_copy_runs(make_worker):
"""The bytes handed to ``save_caches`` must reach the mmap intact even when
the caller drops its ``encoder_cache`` reference before the copy has run.
The model runner pops encoder cache entries on whatever step the scheduler
asks it to, with no dependency on the save copy having landed. Dropping the
last reference hands the GPU memory back to the caching allocator, which is
free to reuse it for the next same-sized allocation on the compute stream —
the copy reads that memory on a different stream, so the allocator has no
reason to hold it back.
The stall makes the hazard deterministic rather than timing-dependent: the
save copy is gated behind the compute stream, so it provably has not started
while the overwrite (issued on an ungated stream and waited on) completes.
The mmap must be pinned, as it is in production: a copy into pageable host
memory blocks the host until it lands, which would hide the hazard.
"""
worker = make_worker(pin_memory_available=True)
_warm_up_stream_pools(worker)
worker._region.blocks.fill_(0x5A)
block_ids = [0, 1, 2]
saved = torch.full(
(len(block_ids), _HIDDEN_DIM), 1.0, dtype=_DTYPE, device=DEVICE_TYPE
)
expected = saved.cpu().reshape(-1).view(torch.uint8)
encoder_cache = {"h": saved}
worker.save_caches(encoder_cache, "h", _meta(saves={"h": block_ids}))
torch.cuda._sleep(_STALL_CYCLES)
worker.flush_saves()
saved_ptr = saved.data_ptr()
del saved
encoder_cache.clear()
# Same size on the same stream, so the allocator hands back the block just
# freed. Overwriting it from an ungated stream — and waiting for that write
# — orders the overwrite strictly before the still-stalled save copy.
overwrite = torch.empty(
(len(block_ids), _HIDDEN_DIM), dtype=_DTYPE, device=DEVICE_TYPE
)
reused = overwrite.data_ptr() == saved_ptr
other_stream = torch.cuda.Stream()
with torch.cuda.stream(other_stream):
overwrite.fill_(-1.0)
other_stream.synchronize()
_wait_for_completion(worker, "h", "saves")
actual = worker._region.blocks[block_ids].reshape(-1).view(torch.uint8)
assert torch.equal(actual, expected), (
"mmap holds bytes written after save_caches was called; the copy read "
f"freed source memory (allocator reused the freed block: {reused})"
)
@_requires_cuda_alike
def test_load_buffer_survives_eviction_while_consumer_read_is_queued(make_worker):
"""A loaded encoder cache entry must keep its bytes for a consumer already
queued on the compute stream, even after the entry is dropped and another
load is dispatched.
``start_load_caches`` allocates its destination inside the load stream's
context, which ties that memory to the load stream in the caching allocator.
Dropping the encoder cache entry returns it to the load stream's pool, where
the next load can claim it — while the model's read of it is still queued on
the compute stream.
"""
worker = make_worker(pin_memory_available=True)
_warm_up_stream_pools(worker)
a_blocks, b_blocks = [4, 5], [6, 7]
for block_idx, fill in zip(a_blocks + b_blocks, (0x11, 0x12, 0x21, 0x22)):
worker._region.blocks[block_idx].fill_(fill)
expected_a = worker._region.blocks[a_blocks].reshape(-1).clone()
encoder_cache: dict[str, torch.Tensor] = {}
meta_a = _meta(loads={"a": a_blocks})
worker.start_load_caches(encoder_cache, meta_a)
loaded_a = encoder_cache["a"]
# Queue a consumer of the loaded entry on the compute stream behind a stall,
# the way the model reads the encoder cache later in the step.
torch.cuda._sleep(_STALL_CYCLES)
consumed = loaded_a.clone()
# Draining the completion report recycles the load stream into the pool, so
# the next load allocates from the same allocator pool the entry was freed to.
_wait_for_completion(worker, _load_id(meta_a, "a"), "loads")
loaded_a_ptr = loaded_a.data_ptr()
del loaded_a
encoder_cache.clear()
worker.start_load_caches(encoder_cache, _meta(loads={"b": b_blocks}))
reused = encoder_cache["b"].data_ptr() == loaded_a_ptr
torch.accelerator.synchronize()
assert torch.equal(consumed.cpu().reshape(-1).view(torch.int8), expected_a), (
"queued consumer read bytes from a later load; the load destination was "
f"recycled while still in use (allocator reused the buffer: {reused})"
)
# ── buffer recycling ────────────────────────────────────────────────────────
@_requires_accelerator
def test_buffer_pool_is_reused_across_save_steps(make_worker):
"""After flush_saves, descriptor buffers are returned to the pool and
reused on the next flush — no reallocation."""
"""Once a save copy completes its descriptor buffers return to the pool and
are reused by the next flush — no reallocation.
The buffers hold the addresses the copy reads, so they stay out of the pool
until the transfer's end event fires.
"""
worker = make_worker()
src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE)
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0]}))
worker.flush_saves()
_wait_for_completion(worker, "h", "saves")
assert len(worker._buf_pool._pool) == 1
buf_id = id(worker._buf_pool._pool[0].src_ptrs)
@@ -411,6 +683,7 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
# Second step reuses the same buffer.
worker.save_caches({"h": src}, "h", _meta(saves={"h": [1]}))
worker.flush_saves()
_wait_for_completion(worker, "h", "saves")
assert len(worker._buf_pool._pool) == 1
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
@@ -418,35 +691,29 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
@_requires_accelerator
def test_buffer_pool_is_reused_across_load_steps(make_worker):
"""After start_load_caches, descriptor buffers are returned to the pool
and reused on the next call."""
"""Once a load copy completes its descriptor buffers return to the pool and
are reused by the next call — no reallocation."""
worker = make_worker()
worker._region.blocks[0].fill_(0x01)
worker._region.blocks[1].fill_(0x02)
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={"a": [0]}))
meta = _meta(loads={"a": [0]})
worker.start_load_caches(encoder_cache, meta)
_wait_for_completion(worker, _load_id(meta, "a"), "loads")
assert len(worker._buf_pool._pool) == 1
buf_id = id(worker._buf_pool._pool[0].src_ptrs)
encoder_cache2: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache2, _meta(loads={"b": [1]}))
meta2 = _meta(loads={"b": [1]})
worker.start_load_caches(encoder_cache2, meta2)
_wait_for_completion(worker, _load_id(meta2, "b"), "loads")
assert len(worker._buf_pool._pool) == 1
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
# ── stream management ────────────────────────────────────────────────────────
@_requires_accelerator
def test_stream_initialized_at_construction(make_worker):
"""``_load_stream`` must be a fully initialized accelerator stream."""
worker = make_worker()
assert isinstance(worker._load_stream, current_platform.Stream)
# ── lifecycle ────────────────────────────────────────────────────────────────
@@ -500,12 +767,15 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
worker = object.__new__(ECCPUWorker)
mock_region = Mock(spec=ECSharedRegion)
worker._region = mock_region
worker._load_stream = MagicMock()
worker._save_bufs = None
worker._save_count = 0
worker._save_mm_hashes = []
worker._inflight_saves = deque()
worker._inflight_loads = deque()
worker._stream_pool = []
worker._event_pool = []
worker.shutdown()
worker._load_stream.synchronize.assert_called_once()
mock_region.cleanup.assert_called_once()
mock_region.cleanup.side_effect = RuntimeError("boom")
@@ -525,12 +795,13 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
@_requires_accelerator
def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
"""Full pipeline: scheduler allocates blocks, worker saves GPU tensor to
mmap via flush_saves, scheduler marks ready after step delay, worker
loads from mmap back to GPU, and the result matches the original.
"""Full pipeline: scheduler allocates blocks, worker saves a GPU tensor to
mmap via flush_saves, the worker's completion report marks the entry ready,
the worker loads from mmap back to GPU, and the result matches the original.
Exercises the real scheduler + worker cooperation through a shared
ECSharedRegion, with real accelerator transfers and stream coordination.
ECSharedRegion, with real accelerator transfers and stream coordination,
and the event-driven mark_ready path.
"""
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
@@ -544,15 +815,7 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
# Build scheduler sharing the same region.
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
class _EC:
is_ec_producer = True
is_ec_consumer = True
class _Cfg:
ec_transfer_config = _EC()
max_concurrent_batches = 1
scheduler = ECCPUScheduler(_Cfg())
scheduler = ECCPUScheduler(_vllm_config())
# -- Step 1: scheduler allocates, worker saves --
n_blocks = 3
@@ -580,16 +843,24 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
worker.save_caches(encoder_cache, "img_001", meta_save)
worker.flush_saves()
# -- Step 2: readiness delay (max_concurrent_batches=1) --
meta_step2 = scheduler.build_connector_meta(scheduler_output=None)
# -- Step 2: worker reports the save memcpy complete → scheduler marks ready --
torch.accelerator.synchronize()
worker_meta = worker.build_connector_worker_meta()
assert worker_meta is not None
assert "img_001" in worker_meta.completed_saves
class _Output:
ec_connector_worker_meta = worker_meta
scheduler.update_connector_output(_Output())
assert scheduler.has_cache_item("img_001") is True
assert meta_step2.loads == {} # no load requested yet
# -- Step 3: scheduler emits load, worker loads --
scheduler.update_state_after_alloc(_Request(), 0)
meta_load = scheduler.build_connector_meta(scheduler_output=None)
assert "img_001" in meta_load.loads
assert meta_load.loads["img_001"] == meta_save.saves["img_001"]
_, load_blocks = meta_load.loads["img_001"]
assert load_blocks == meta_save.saves["img_001"]
load_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(load_cache, meta_load)
@@ -48,9 +48,10 @@ def _make_unique_image(base_image, iteration: int):
def _wait_for_ec_ready(llm: LLM) -> None:
"""Send a dummy request to force a scheduler step.
After generate() returns, the EC entry's data is in the mmap but
readiness may not have fired yet (first-finish fires in the NEXT
scheduler step). This forces that step.
When generate() returns, the save copy has been issued to the GPU but its
completion event may fire on a later step, so the entry is not yet marked
ready. This extra step lets the scheduler poll the worker's completion
report and mark the entry ready.
"""
llm.generate("hi", SamplingParams(max_tokens=1), use_tqdm=False)
+13 -13
View File
@@ -38,19 +38,19 @@ def test_metadata_saves_dict_operations():
assert meta.saves["mm_hash_2"] == [10, 11]
def test_metadata_loads_dict_operations():
"""The loads dict supports standard dict operations."""
def test_metadata_loads_carry_a_transfer_id_and_blocks():
"""Each load entry is a (transfer_id, block_ids) pair, so the worker can
report the transfer rather than the mm_hash."""
meta = ECCPUConnectorMetadata()
# Add multiple entries
meta.loads["mm_hash_1"] = [5, 6, 7]
meta.loads["mm_hash_2"] = [100]
meta.loads["mm_hash_1"] = (0, [5, 6, 7])
meta.loads["mm_hash_2"] = (1, [100])
assert "mm_hash_1" in meta.loads
assert "mm_hash_2" in meta.loads
assert len(meta.loads) == 2
assert meta.loads["mm_hash_1"] == [5, 6, 7]
assert meta.loads["mm_hash_2"] == [100]
assert meta.loads["mm_hash_1"] == (0, [5, 6, 7])
transfer_id, block_ids = meta.loads["mm_hash_2"]
assert transfer_id == 1
assert block_ids == [100]
def test_metadata_saves_and_loads_are_independent():
@@ -58,11 +58,11 @@ def test_metadata_saves_and_loads_are_independent():
meta = ECCPUConnectorMetadata()
meta.saves["key"] = [1, 2]
meta.loads["key"] = [3, 4]
meta.loads["key"] = (0, [3, 4])
# Same key but different dicts should not interfere
assert meta.saves["key"] == [1, 2]
assert meta.loads["key"] == [3, 4]
assert meta.loads["key"] == (0, [3, 4])
# Modifying one should not affect the other
meta.saves["other"] = [99]
@@ -74,11 +74,11 @@ def test_metadata_block_indices_are_integers():
meta = ECCPUConnectorMetadata()
meta.saves["hash"] = [0, 100, 999, 65535]
meta.loads["hash"] = [1, 2, 3]
meta.loads["hash"] = (0, [1, 2, 3])
for idx in meta.saves["hash"]:
assert isinstance(idx, int)
for idx in meta.loads["hash"]:
for idx in meta.loads["hash"][1]:
assert isinstance(idx, int)
+70
View File
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared config builders for EC connector unit tests."""
import uuid
from unittest.mock import Mock
import torch
from vllm.config import ModelConfig, VllmConfig
from vllm.config.ec_transfer import ECRole, ECTransferConfig
from vllm.config.parallel import ParallelConfig
# The scheduler derives its per-load ack count from the PCP size only under the
# `mp` backend -- which also keeps the world size off the local GPU count, so
# TP > 1 stays expressible on a single-GPU host.
_EXECUTOR_BACKEND = "mp"
_PCP_SIZE = 1
def create_ec_vllm_config(
*,
ec_role: ECRole = "ec_both",
tensor_parallel_size: int = 1,
rank: int = 0,
dtype: torch.dtype = torch.float16,
) -> Mock:
"""Build a `VllmConfig` stand-in for EC connector unit tests.
`ec_transfer_config` and `parallel_config` are the real config objects, so
role derivation and the rank arithmetic the connector depends on behave as
they do in production.
`model_config` is a stub because nothing under test needs a real one: the
connector reads only `dtype` here, and the paths that inspect the model
itself are not exercised. Building a real `ModelConfig` would resolve an HF
model and inspect its architecture.
Args:
ec_role: EC role this instance plays. `is_ec_producer` and
`is_ec_consumer` are derived from it, as in production.
tensor_parallel_size: TP degree.
rank: Global rank of this worker.
dtype: Encoder cache dtype.
Returns:
A `VllmConfig`-specced mock carrying the configs above.
"""
parallel_config = ParallelConfig(
tensor_parallel_size=tensor_parallel_size,
prefill_context_parallel_size=_PCP_SIZE,
distributed_executor_backend=_EXECUTOR_BACKEND,
rank=rank,
)
ec_transfer_config = ECTransferConfig(
# The role properties report False unless a connector is named.
ec_connector="ECCPUConnector",
ec_role=ec_role,
engine_id=str(uuid.uuid4()),
)
model_config = Mock(spec=ModelConfig)
model_config.dtype = dtype
vllm_config = Mock(spec=VllmConfig)
vllm_config.ec_transfer_config = ec_transfer_config
vllm_config.parallel_config = parallel_config
vllm_config.model_config = model_config
vllm_config.instance_id = f"ec-test-{uuid.uuid4()}"
return vllm_config