[EC Connector] CPU Offloading EC Connector (#47423)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Signed-off-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
omerpaz95
2026-07-13 20:09:41 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 62286308c9
commit 43c8cbf79b
18 changed files with 3008 additions and 0 deletions
@@ -0,0 +1,306 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for EmbeddingCache.
Pure integer bookkeeping — no mmap, torch, or CUDA — so these run anywhere.
"""
import threading
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
def _cache(num_blocks: int = 8) -> EmbeddingCache:
return EmbeddingCache(num_blocks)
# ── alloc ────────────────────────────────────────────────────────────────────
def test_alloc_returns_unique_block_ids():
cache = _cache()
entry = cache.alloc("a", 3)
assert entry is not None
assert len(entry.block_ids) == 3
assert len(set(entry.block_ids)) == 3
assert all(0 <= i < 8 for i in entry.block_ids)
def test_alloc_returns_tuple():
cache = _cache()
entry = cache.alloc("a", 2)
assert isinstance(entry.block_ids, tuple)
def test_alloc_returns_none_when_full():
cache = _cache(num_blocks=4)
cache.alloc("a", 4)
assert cache.alloc("b", 1) is None
def test_alloc_asserts_duplicate_key():
cache = _cache()
cache.alloc("a", 2)
with pytest.raises(AssertionError, match="duplicate alloc"):
cache.alloc("a", 2)
def test_alloc_asserts_n_blocks_exceeds_capacity():
cache = _cache(num_blocks=4)
with pytest.raises(AssertionError, match="capacity"):
cache.alloc("a", 5)
def test_alloc_zero_blocks():
cache = _cache()
entry = cache.alloc("a", 0)
assert entry is not None
assert entry.block_ids == ()
def test_alloc_starts_not_ready():
cache = _cache()
entry = cache.alloc("a", 2)
assert not entry.ready
assert not entry.evictable
# ── get ──────────────────────────────────────────────────────────────────────
def test_get_missing_returns_none():
cache = _cache()
assert cache.get("a") is None
def test_get_returns_entry():
cache = _cache()
alloc_entry = cache.alloc("a", 2)
got = cache.get("a")
assert got is alloc_entry
def test_get_returns_not_ready_entry():
cache = _cache()
cache.alloc("a", 2)
entry = cache.get("a")
assert entry is not None
assert not entry.ready
# ── mark_ready ───────────────────────────────────────────────────────────────
def test_mark_ready_transitions():
cache = _cache()
cache.alloc("a", 2)
entry = cache.get("a")
assert not entry.ready
cache.mark_ready("a")
assert entry.ready
assert entry.evictable
def test_mark_ready_asserts_already_ready():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
with pytest.raises(AssertionError, match="already-ready"):
cache.mark_ready("a")
def test_mark_ready_asserts_missing_key():
cache = _cache()
with pytest.raises(KeyError):
cache.mark_ready("nonexistent")
# ── pin / unpin ──────────────────────────────────────────────────────────────
def test_pin_asserts_not_ready():
cache = _cache()
cache.alloc("a", 2)
with pytest.raises(AssertionError, match="not-ready"):
cache.pin("a")
def test_pin_unpin_basic():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
entry = cache.get("a")
assert not entry.evictable
cache.unpin("a")
assert entry.evictable
def test_nested_pin():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
cache.pin("a")
cache.unpin("a")
entry = cache.get("a")
assert not entry.evictable # still pinned (count=1)
cache.unpin("a")
assert entry.evictable
def test_unpin_asserts_on_unpinned():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
with pytest.raises(AssertionError, match="unpinned"):
cache.unpin("a")
# ── eviction ─────────────────────────────────────────────────────────────────
def test_eviction_frees_ready_unpinned_fifo():
cache = _cache(num_blocks=4)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.alloc("b", 2)
cache.mark_ready("b")
# Pool full. Alloc should evict "a" (oldest).
entry_c = cache.alloc("c", 2)
assert entry_c is not None
assert cache.get("a") is None # evicted
assert cache.get("b") is not None # still there
def test_eviction_skips_pinned():
cache = _cache(num_blocks=4)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
cache.alloc("b", 2)
cache.mark_ready("b")
# "a" is pinned, so "b" should be evicted instead.
entry_c = cache.alloc("c", 2)
assert entry_c is not None
assert cache.get("a") is not None # pinned, survived
assert cache.get("b") is None # evicted
cache.unpin("a")
def test_eviction_skips_not_ready():
cache = _cache(num_blocks=4)
cache.alloc("a", 2) # not ready
cache.alloc("b", 2)
cache.mark_ready("b")
# "a" is not ready (non-evictable), "b" is evictable.
entry_c = cache.alloc("c", 2)
assert entry_c is not None
assert cache.get("a") is not None # not ready, survived
assert cache.get("b") is None # evicted
cache.mark_ready("a") # clean up
def test_eviction_returns_none_if_all_non_evictable():
cache = _cache(num_blocks=4)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
cache.alloc("b", 2) # not ready
# Both entries are non-evictable (a=pinned, b=not-ready).
assert cache.alloc("c", 1) is None
# Fail-fast: nothing was evicted.
assert cache.get("a") is not None
assert cache.get("b") is not None
cache.unpin("a")
def test_fail_fast_no_wasted_eviction():
cache = _cache(num_blocks=6)
cache.alloc("a", 2)
cache.mark_ready("a") # evictable: 2 blocks
cache.alloc("b", 2) # not ready
cache.alloc("c", 2) # not ready
# free=0, evictable=2, need 3 -> cannot succeed
assert cache.alloc("d", 3) is None
# "a" was NOT evicted despite being evictable
assert cache.get("a") is not None
def test_eviction_multiple_entries():
cache = _cache(num_blocks=6)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.alloc("b", 2)
cache.mark_ready("b")
cache.alloc("c", 2)
cache.mark_ready("c")
# Need 4 blocks: must evict both "a" and "b".
entry_d = cache.alloc("d", 4)
assert entry_d is not None
assert cache.get("a") is None
assert cache.get("b") is None
assert cache.get("c") is not None
# ── thread safety ────────────────────────────────────────────────────────────
def test_concurrent_pin_unpin():
cache = _cache(num_blocks=8)
cache.alloc("a", 4)
cache.mark_ready("a")
errors: list[Exception] = []
barrier = threading.Barrier(8)
def worker():
barrier.wait()
try:
for _ in range(500):
cache.pin("a")
cache.unpin("a")
except Exception as exc:
errors.append(exc)
threads = [threading.Thread(target=worker) for _ in range(8)]
for t in threads:
t.start()
for t in threads:
t.join()
assert not errors, errors
entry = cache.get("a")
assert entry.evictable # all pins balanced
def test_concurrent_alloc_evict():
cache = _cache(num_blocks=4)
errors: list[Exception] = []
barrier = threading.Barrier(4)
counter = {"i": 0}
counter_lock = threading.Lock()
def worker():
barrier.wait()
try:
for _ in range(200):
with counter_lock:
key = f"k{counter['i']}"
counter["i"] += 1
entry = cache.alloc(key, 1)
if entry is not None:
cache.mark_ready(key)
except Exception as exc:
errors.append(exc)
threads = [threading.Thread(target=worker) for _ in range(4)]
for t in threads:
t.start()
for t in threads:
t.join()
assert not errors, errors
@@ -0,0 +1,480 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
_N = 16
_BS = 64
class _Pos:
def __init__(self, offset, length):
self.offset = offset
self.length = length
class _Feature:
def __init__(self, mm_hash, length=1, identifier=None):
self.mm_hash = mm_hash
self.identifier = identifier if identifier is not None else mm_hash
self.mm_position = _Pos(0, length)
class _Request:
_counter = 0
def __init__(self, features, request_id=None):
self.mm_features = features
if request_id is None:
_Request._counter += 1
self.request_id = f"req_{_Request._counter}"
else:
self.request_id = request_id
def _make_scheduler(
monkeypatch,
num_blocks=_N,
max_concurrent_batches=1,
*,
is_producer=True,
is_consumer=True,
) -> ECCPUScheduler:
region = ECSharedRegion(
engine_id=str(uuid.uuid4()),
num_blocks=num_blocks,
block_size_bytes=_BS,
)
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
_is_prod = is_producer
_is_cons = is_consumer
_mcb = max_concurrent_batches
class _EC:
is_ec_producer = _is_prod
is_ec_consumer = _is_cons
class _Cfg:
ec_transfer_config = _EC()
@property
def max_concurrent_batches(self):
return _mcb
return ECCPUScheduler(_Cfg())
class _SchedulerOutput:
def __init__(self, finished_req_ids=None):
self.finished_req_ids = finished_req_ids or set()
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
"""Pre-populate a ready cache entry backed by real blocks."""
s._cache.alloc(mm_hash, n_blocks)
s._cache.mark_ready(mm_hash)
def test_offload_reuse_cycle(monkeypatch):
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1)])
# Step A: first sight — allocate and save.
assert s.has_cache_item("h1") is False
assert s.ensure_cache_available(req, 0) is True
s.update_state_after_alloc(req, 0)
meta_a = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_a.saves
assert meta_a.loads == {}
# Not ready yet (max_concurrent_batches=1 means ready next step).
assert s.has_cache_item("h1") is False
# Step B: _step_readiness marks h1 ready (no new allocs).
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is True
# Step C: reload now works — entry is ready at update_state_after_alloc time.
s.update_state_after_alloc(req, 0)
meta_c = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_c.loads
assert meta_c.loads["h1"] == meta_a.saves["h1"]
s.shutdown()
def test_has_cache_item_false_when_not_consumer(monkeypatch):
s = _make_scheduler(monkeypatch, is_consumer=False)
assert s.has_cache_item("anything") is False
s.shutdown()
def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
"""The connector must key the encoder cache on feature.identifier (what
has_cache_item is called with), NOT feature.mm_hash."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")])
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "ENC_KEY" in meta.saves
assert "PROC_KEY" not in meta.saves
# Not ready yet (1 step delay).
assert s.has_cache_item("ENC_KEY") is False
# After another step it becomes ready — keyed on identifier.
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("ENC_KEY") is True
assert s.has_cache_item("PROC_KEY") is False
s.shutdown()
def test_cpu_region_fifo_eviction(monkeypatch):
# Region holds exactly 2 one-block encodings.
s = _make_scheduler(monkeypatch, num_blocks=2)
# Fill with A and B, marking each ready.
for h in ("A", "B"):
s.update_state_after_alloc(_Request([_Feature(h, length=1)]), 0)
s.build_connector_meta(scheduler_output=None)
# After the step above, A's step was called in B's build. A is ready.
# B was submitted in the last build, so one more step makes it ready.
s.build_connector_meta(scheduler_output=None) # marks B ready
assert s.has_cache_item("A") is True
assert s.has_cache_item("B") is True
# Allocate C: must evict A (oldest) to make room.
s.update_state_after_alloc(_Request([_Feature("C", length=1)]), 0)
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("A") is False # evicted
assert s.has_cache_item("B") is True
def test_multiple_mm_items_per_request(monkeypatch):
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1), _Feature("h2", length=1)])
s.update_state_after_alloc(req, 0)
s.update_state_after_alloc(req, 1)
meta = s.build_connector_meta(scheduler_output=None)
assert set(meta.saves) == {"h1", "h2"}
# Step delay: mark both ready.
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is True
assert s.has_cache_item("h2") is True
# Now reload both.
s.update_state_after_alloc(req, 0)
s.update_state_after_alloc(req, 1)
meta2 = s.build_connector_meta(scheduler_output=None)
assert set(meta2.loads) == {"h1", "h2"}
s.shutdown()
def test_load_returns_correct_block_ids(monkeypatch):
"""meta.loads must contain the same block IDs that meta.saves
allocated — verified through the public API only."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("a", length=2)])
s.update_state_after_alloc(req, 0)
meta_save = s.build_connector_meta(scheduler_output=None)
assert "a" in meta_save.saves
# Readiness delay.
s.build_connector_meta(scheduler_output=None)
# Reload.
s.update_state_after_alloc(req, 0)
meta_load = s.build_connector_meta(scheduler_output=None)
assert meta_load.loads["a"] == meta_save.saves["a"]
s.shutdown()
def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=2)
_seed_cached(s, "b", n_blocks=2)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "a" in meta.loads
assert "b" not in meta.loads
s.shutdown()
def test_repeated_reload_same_step_loads_once(monkeypatch):
"""The same mm_hash requested twice in one step must appear in
meta.loads exactly once."""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=2)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert list(meta.loads.keys()).count("a") == 1
s.shutdown()
def test_load_not_emitted_for_uncached_entry(monkeypatch):
"""A feature that was never saved must not appear in meta.loads."""
s = _make_scheduler(monkeypatch)
s.update_state_after_alloc(_Request([_Feature("missing")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert meta.loads == {}
s.shutdown()
def test_delayed_unpin_protects_blocks_during_worker_read(monkeypatch):
"""Blocks being loaded must not be evictable until the delayed unpin
fires. Observable: a save that needs eviction cannot reclaim a
still-pinned loaded entry."""
# 2 blocks, both occupied by "a" (ready).
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
# Load "a" — pins it.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "a" in meta.loads
# Try to save "b" (needs 2 blocks, but "a" is pinned) — must fail.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta2.saves # can't evict pinned "a"
# After the delay, "a" unpins. Now "b" can evict it.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta3 = s.build_connector_meta(scheduler_output=None)
assert "b" in meta3.saves
s.shutdown()
def test_delayed_unpin_depth_2(monkeypatch):
"""With max_concurrent_batches=2, loaded blocks stay pinned for 2 steps
after build. Observable via eviction failure."""
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2)
_seed_cached(s, "a", n_blocks=2)
# Load "a" → pinned.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
s.build_connector_meta(scheduler_output=None)
# Step 1 after load: still pinned — can't evict for "b".
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta1 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta1.saves
# Step 2 after load: still pinned.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta2.saves
# Step 3: unpin fires, now eviction works.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta3 = s.build_connector_meta(scheduler_output=None)
assert "b" in meta3.saves
s.shutdown()
def test_shutdown_unpins_pending_loads(monkeypatch):
"""Shutdown must unpin entries still in _pending_loads (not yet drained
to the unpin deque). Observable: after shutdown, the entry is evictable
(alloc can reclaim its blocks)."""
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
# Don't call build_connector_meta — "a" stays in _pending_loads.
s.shutdown()
# After shutdown, scheduler reports no items (consumer role is off).
assert s.has_cache_item("a") is False
def test_shutdown_unpins_deferred_unpin_queue(monkeypatch):
"""Shutdown must unpin entries queued in the deferred unpin deque
(already drained from _pending_loads but not yet expired)."""
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2)
_seed_cached(s, "a", n_blocks=2)
# Load "a" and build — moves to unpin deque.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
s.build_connector_meta(scheduler_output=None)
# "a" is in the deferred unpin deque (depth=2, not yet expired).
# Shutdown must unpin it.
s.shutdown()
assert s.has_cache_item("a") is False
def test_eviction_skips_entry_pinned_by_pending_load(monkeypatch):
"""A loaded entry still within the unpin delay window must survive
eviction attempts from new saves."""
# 2 blocks: A occupies both, ready.
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "A", n_blocks=2)
# Load A → pinned.
s.update_state_after_alloc(_Request([_Feature("A")]), 0)
meta_load = s.build_connector_meta(scheduler_output=None)
assert "A" in meta_load.loads
# Try to save C (needs 2 blocks) — can't evict pinned A.
s.update_state_after_alloc(_Request([_Feature("C", length=2)]), 0)
meta_save = s.build_connector_meta(scheduler_output=None)
assert "C" not in meta_save.saves
assert s.has_cache_item("A") is True # survived
s.shutdown()
def test_region_full_skips_save_and_never_blocks(monkeypatch):
"""When the region is fully occupied by pinned entries, new saves are
silently skipped and ensure_cache_available never blocks."""
s = _make_scheduler(monkeypatch, num_blocks=1)
_seed_cached(s, "pinned", n_blocks=1)
s._cache.pin("pinned")
req = _Request([_Feature("new", length=1)])
assert s.ensure_cache_available(req, 0) is True
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "new" not in meta.saves
assert s.has_cache_item("new") is False
s._cache.unpin("pinned")
s.shutdown()
def test_step_readiness_depth_2(monkeypatch):
"""With max_concurrent_batches=2, entries need 2 steps to become ready."""
s = _make_scheduler(monkeypatch, max_concurrent_batches=2)
req = _Request([_Feature("h1", length=1)])
s.update_state_after_alloc(req, 0)
meta_0 = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_0.saves
assert s.has_cache_item("h1") is False
# One step later: still not ready (depth=2).
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is False
# Two steps later: now ready.
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is True
s.shutdown()
def test_producer_only_never_emits_loads(monkeypatch):
"""A producer-only scheduler must never populate meta.loads, even when
entries are ready."""
s = _make_scheduler(monkeypatch, is_consumer=False)
req = _Request([_Feature("h1", length=1)])
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta.saves
assert meta.loads == {}
# Mark ready.
s.build_connector_meta(scheduler_output=None)
# Even if we re-encounter the same feature, no load.
s.update_state_after_alloc(req, 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert meta2.loads == {}
s.shutdown()
def test_consumer_only_never_emits_saves(monkeypatch):
"""A consumer-only scheduler must never populate meta.saves."""
s = _make_scheduler(monkeypatch, is_producer=False)
_seed_cached(s, "a", n_blocks=1)
req = _Request([_Feature("a", length=1)])
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert meta.saves == {}
assert "a" in meta.loads
s.shutdown()
def test_consumer_only_has_cache_item(monkeypatch):
"""Consumer-only: has_cache_item works normally for ready entries."""
s = _make_scheduler(monkeypatch, is_producer=False)
assert s.has_cache_item("a") is False
_seed_cached(s, "a", n_blocks=1)
assert s.has_cache_item("a") is True
s.shutdown()
def test_save_not_emitted_for_already_cached_entry(monkeypatch):
"""An entry already in the cache (from a prior save) must not trigger
a second allocation or appear in meta.saves again."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1)])
# First encounter → save.
s.update_state_after_alloc(req, 0)
meta1 = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta1.saves
# Second encounter (same step or later) → no save.
s.update_state_after_alloc(req, 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "h1" not in meta2.saves
s.shutdown()
# ── first-finish fast-path (idle engine) ────────────────────────────────────
def test_first_finish_marks_ready_immediately(monkeypatch):
"""When the originating request finishes, the save entry becomes ready
without waiting for the step-count delay."""
s = _make_scheduler(monkeypatch, max_concurrent_batches=5)
req = _Request([_Feature("h1", length=1)], request_id="req_x")
s.update_state_after_alloc(req, 0)
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is False
# Request finishes — readiness should fire immediately.
s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_x"}))
assert s.has_cache_item("h1") is True
s.shutdown()
def test_first_finish_unpins_immediately(monkeypatch):
"""When the originating request finishes, the load entry is unpinned
without waiting for the step-count delay."""
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=5)
_seed_cached(s, "a", n_blocks=2)
req = _Request([_Feature("a")], request_id="req_y")
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "a" in meta.loads
# "a" is pinned. Try to evict — must fail.
s.update_state_after_alloc(
_Request([_Feature("b", length=2)], request_id="req_z"), 0
)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta2.saves
# req_y finishes — unpin fires in build_connector_meta (fast-path).
s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_y"}))
# Next step: "a" is now evictable, so "b" can be allocated.
s.update_state_after_alloc(
_Request([_Feature("b", length=2)], request_id="req_w"), 0
)
meta3 = s.build_connector_meta(scheduler_output=None)
assert "b" in meta3.saves
s.shutdown()
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for StepTracker."""
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
StepTracker,
)
def test_basic_step_count_expiry_depth_1():
"""With max_concurrent_batches=1, entry expires after 1 step."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
# Step 1: entry was added, committed to slot. Deque fills to maxlen=1,
# but no expiry yet (the entry is in the slot just committed).
result = t.step(set())
assert result == []
# Step 2: deque is full, oldest slot (containing h1) expires.
result = t.step(set())
assert result == ["h1"]
def test_basic_step_count_expiry_depth_2():
"""With max_concurrent_batches=2, entry expires after 2 steps."""
t = StepTracker(max_concurrent_batches=2)
t.add("h1", "req_a")
assert t.step(set()) == [] # step 1: committed
assert t.step(set()) == [] # step 2: deque fills but h1 in newer slot
assert t.step(set()) == ["h1"] # step 3: h1's slot is oldest, expires
def test_first_finish_fast_path():
"""Entry is returned immediately when request_id appears in finished."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
result = t.step({"req_a"})
assert result == ["h1"]
def test_no_double_processing_fast_path_then_expiry():
"""Entry fast-pathed should NOT fire again on deque expiry."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
# Fast-path fires h1.
result = t.step({"req_a"})
assert result == ["h1"]
# Normal expiry would fire now, but h1 is already processed.
result = t.step(set())
assert result == []
def test_no_double_processing_expiry_then_fast_path():
"""Entry expired via step-count should NOT fire again via fast-path."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
t.step(set()) # commit
result = t.step(set()) # expiry fires h1
assert result == ["h1"]
# req_a finishes later — must not re-fire.
result = t.step({"req_a"})
assert result == []
def test_multiplicity_same_hash_different_requests():
"""Two adds of the same mm_hash (different requests) → returned twice."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
t.add("h1", "req_b")
# Both requests finish in the same step.
result = t.step({"req_a", "req_b"})
assert sorted(result) == ["h1", "h1"]
def test_mixed_expiry_and_fast_path():
"""Some entries expire, others fast-path, in the same step."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
t.step(set()) # commit h1's slot
# Add h2 in the next step.
t.add("h2", "req_b")
# This step: h1's slot expires AND req_b finishes.
result = t.step({"req_b"})
assert set(result) == {"h1", "h2"}
def test_drain_all_returns_unprocessed():
"""drain_all returns everything pending and clears state."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
t.add("h2", "req_b")
t.step(set()) # commit both
t.add("h3", "req_c") # in _current, not yet committed
result = t.drain_all()
assert set(result) == {"h1", "h2", "h3"}
# After drain, everything is empty.
assert t.step(set()) == []
assert t.drain_all() == []
def test_drain_all_skips_already_processed():
"""drain_all should not re-return entries already fast-pathed."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
t.add("h2", "req_b")
# Fast-path h1.
result = t.step({"req_a"})
assert result == ["h1"]
# drain should only return h2.
result = t.drain_all()
assert result == ["h2"]
def test_empty_step_returns_nothing():
"""step() with nothing pending returns empty list."""
t = StepTracker(max_concurrent_batches=2)
assert t.step(set()) == []
assert t.step({"req_x"}) == []
def test_add_after_step_goes_to_next_slot():
"""Entries added after a step go into the next slot, not the current."""
t = StepTracker(max_concurrent_batches=1)
t.step(set()) # empty step
t.add("h1", "req_a")
# h1 is in _current. Next step commits it.
t.step(set())
# Now the deque is full (1 slot with h1). Next step expires it.
result = t.step(set())
assert result == ["h1"]
def test_fast_path_on_current_entries():
"""Entries still in _current (not yet committed) can be fast-pathed."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
# req_a finishes in the same step h1 was added.
result = t.step({"req_a"})
assert result == ["h1"]
# Entry is committed to slot but marked processed — won't re-fire.
for _ in range(5):
assert t.step(set()) == []
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
def _cfg():
ec = MagicMock()
ec.is_ec_producer = True
ec.is_ec_consumer = True
cfg = MagicMock()
cfg.ec_transfer_config = ec
return cfg
def test_scheduler_role_builds_only_scheduler(monkeypatch):
fake_sched = MagicMock()
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
assert c.connector_scheduler is fake_sched
assert c.connector_worker is None
c.has_cache_item("x")
fake_sched.has_cache_item.assert_called_once_with("x")
def test_worker_role_builds_only_worker(monkeypatch):
fake_worker = MagicMock()
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: fake_worker)
c = ECCPUConnector(_cfg(), ECConnectorRole.WORKER)
assert c.connector_worker is fake_worker
assert c.connector_scheduler is None
def test_request_finished_inherited_noop(monkeypatch):
monkeypatch.setattr(
ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock()
)
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
assert c.request_finished(MagicMock()) == (False, None)
def test_factory_registered():
cls = ECConnectorFactory._registry["ECCPUConnector"]()
assert cls is ECCPUConnector
@@ -0,0 +1,170 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECSharedRegion (the mmap substrate)."""
import os
import uuid
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
_wait_for_file_size,
)
def _make_region(num_blocks: int = 8, block_size_bytes: int = 64) -> ECSharedRegion:
return ECSharedRegion(
engine_id=str(uuid.uuid4()),
num_blocks=num_blocks,
block_size_bytes=block_size_bytes,
)
@pytest.fixture
def region() -> ECSharedRegion:
r = _make_region()
yield r
r.cleanup()
# ── mmap sharing between two instances ───────────────────────────────────────
def test_second_instance_opens_existing_file_and_shares_memory():
instance_id = str(uuid.uuid4())
r1 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
try:
r2 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
try:
assert r1._is_creator
assert not r2._is_creator
# Both map the same physical pages; writes via r1 are visible via r2.
r1.blocks[0, :4] = torch.tensor([10, 20, 30, 40], dtype=torch.int8)
assert r2.blocks[0, :4].tolist() == [10, 20, 30, 40]
finally:
r2.cleanup()
finally:
r1.cleanup()
def test_only_creator_unlinks_file_on_cleanup():
"""Critical contract: if the non-creator unlinks, the creator's mmap path
becomes a dangling backing file and a third opener would create a new one
out from under the creator."""
instance_id = str(uuid.uuid4())
r1 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
path = r1._mmap_path
r2 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
# Non-creator goes away first — file must still be on disk for r1.
r2.cleanup()
assert os.path.exists(path), "non-creator cleanup must not unlink the file"
# Creator goes away — file is removed.
r1.cleanup()
assert not os.path.exists(path), "creator cleanup must unlink the file"
# ── _wait_for_file_size ──────────────────────────────────────────────────────
def test_wait_for_file_size_returns_when_already_big_enough(tmp_path):
"""The fast path: file already at expected size — return immediately."""
p = tmp_path / "f.bin"
p.write_bytes(b"\x00" * 128)
fd = os.open(str(p), os.O_RDONLY)
try:
_wait_for_file_size(fd, expected_size=128, timeout=1.0) # must not raise
finally:
os.close(fd)
def test_wait_for_file_size_times_out_when_file_stays_empty(tmp_path):
p = tmp_path / "f.bin"
p.write_bytes(b"")
fd = os.open(str(p), os.O_RDONLY)
try:
with pytest.raises(TimeoutError):
_wait_for_file_size(fd, expected_size=4096, timeout=0.05)
finally:
os.close(fd)
# ── pin_memory ───────────────────────────────────────────────────────────────
def test_pin_memory_success_sets_flag(region):
"""When cudaHostRegister returns 0, _is_pinned flips to True
and cleanup will correspondingly call cudaHostUnregister."""
fake_cudart = MagicMock()
success = MagicMock()
success.value = 0
fake_cudart.cudaHostRegister.return_value = success
fake_cudart.cudaHostUnregister.return_value = success
with (
patch("torch.cuda.is_available", return_value=True),
patch("torch.cuda.cudart", return_value=fake_cudart),
):
region.pin_memory()
assert region._is_pinned is True
# cleanup must pair with cudaHostUnregister exactly once.
region.cleanup()
fake_cudart.cudaHostUnregister.assert_called_once()
def test_pin_memory_failure_leaves_flag_false():
"""If cudaHostRegister fails (non-zero), don't pretend it succeeded —
cleanup must NOT call cudaHostUnregister on memory we never registered."""
r = _make_region()
try:
fake_cudart = MagicMock()
fail = MagicMock()
fail.value = 1 # non-zero == error
fake_cudart.cudaHostRegister.return_value = fail
with (
patch("torch.cuda.is_available", return_value=True),
patch("torch.cuda.cudart", return_value=fake_cudart),
):
r.pin_memory()
assert r._is_pinned is False
# Now run cleanup and verify cudaHostUnregister was NOT called.
r.cleanup()
fake_cudart.cudaHostUnregister.assert_not_called()
finally:
r.cleanup()
def test_pin_memory_noop_without_cuda(region):
"""pin_memory is a no-op when CUDA is not available."""
with patch("torch.cuda.is_available", return_value=False):
region.pin_memory()
assert region._is_pinned is False
# ── cleanup idempotency ───────────────────────────────────────────────────────
def test_cleanup_is_idempotent(region):
region.cleanup()
region.cleanup() # fixture calls a third time — must not raise
@@ -0,0 +1,598 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ECCPUWorker.
The byte-level tests exercise real CUDA stream/event coordination
against a real ``ECSharedRegion`` mmap and are skipped on hosts without
CUDA. The lifecycle tests don't fire any ``torch.cuda.*`` primitives and
run anywhere.
Mocking policy
--------------
- ``create_ec_shared_region`` is patched to inject a real ``ECSharedRegion``
(mmap backed) with deterministic small dimensions, so byte assertions
hit the actual mmap. The real responsibility of the worker — block
index → mmap byte mapping, stream coordination, dtype/shape — is
exercised end to end against a real GPU.
- ``is_pin_memory_available`` is patched per-test so the rank-gating
path can be probed deterministically without depending on host caps.
- The shutdown test bypasses ``__init__`` via ``object.__new__`` and
injects a ``MagicMock`` region.
"""
import contextlib
import logging
import uuid
from unittest.mock import MagicMock, Mock, patch
import pytest
import torch
from vllm.config import VllmConfig
from vllm.config.parallel import ParallelConfig
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker
# ── shape constants ──────────────────────────────────────────────────────────
# hidden_dim=8, dtype=fp16 (2 bytes) → block_size_bytes = 16, so one row of
# encoder output fits in exactly one block.
_HIDDEN_DIM = 8
_DTYPE = torch.float16
_BLOCK_SIZE_BYTES = _HIDDEN_DIM * _DTYPE.itemsize
_NUM_BLOCKS = 8
_requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(),
reason="exercises real CUDA stream/event coordination in ECCPUWorker",
)
def _make_region() -> ECSharedRegion:
"""Fresh region backed by a real per-test mmap file."""
return ECSharedRegion(
engine_id=str(uuid.uuid4()),
num_blocks=_NUM_BLOCKS,
block_size_bytes=_BLOCK_SIZE_BYTES,
)
def _vllm_config(rank: int = 0) -> Mock:
cfg = Mock(spec=VllmConfig)
cfg.parallel_config = Mock(spec=ParallelConfig)
cfg.parallel_config.rank = rank
cfg.model_config = Mock()
cfg.model_config.dtype = _DTYPE
return cfg
def _meta(
*, saves: dict | None = None, loads: dict | None = None
) -> ECCPUConnectorMetadata:
return ECCPUConnectorMetadata(saves=saves or {}, loads=loads or {})
# ── fixtures ─────────────────────────────────────────────────────────────────
@pytest.fixture
def make_worker():
"""Factory that builds an ``ECCPUWorker`` over a real ``ECSharedRegion``.
Tracks the workers and regions it creates and tears them down at the
end of the test, so each test starts from a fresh /dev/shm file.
"""
workers: list[ECCPUWorker] = []
regions: list[ECSharedRegion] = []
def factory(
*,
rank: int = 0,
tp_rank: int = 0,
pcp_rank: int = 0,
pin_memory_available: bool = False,
) -> ECCPUWorker:
region = _make_region()
regions.append(region)
pcp_group = Mock()
pcp_group.rank_in_group = pcp_rank
with (
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.create_ec_shared_region",
return_value=region,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.is_pin_memory_available",
return_value=pin_memory_available,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_tensor_model_parallel_rank",
return_value=tp_rank,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_pcp_group",
return_value=pcp_group,
),
):
worker = ECCPUWorker(_vllm_config(rank=rank))
workers.append(worker)
return worker
yield factory
for worker in workers:
with contextlib.suppress(Exception):
worker.shutdown()
for region in regions:
with contextlib.suppress(Exception):
region.cleanup()
# ── save_caches ──────────────────────────────────────────────────────────────
@_requires_cuda
@pytest.mark.parametrize(
"n_elements,n_blocks",
[
(3 * _HIDDEN_DIM, 3), # exact fit: 3 full blocks, no remainder
(3 * _HIDDEN_DIM + 4, 4), # partial last block: 4 fp16 trail in slot 3
],
ids=["exact-fit", "partial-last-block"],
)
def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks):
"""``save_caches`` + ``flush_saves`` copies the source GPU tensor's bytes
into the block IDs named by ``meta.saves``, in the order they appear,
via a single batched GPU→CPU copy.
When ``total_bytes`` is not a multiple of ``block_size_bytes`` the last
block is partially written; the unwritten tail must remain whatever
was there before. Blocks not named in ``meta.saves`` must be untouched.
"""
worker = make_worker()
sentinel = 0x5A
worker._region.blocks.fill_(sentinel)
src = torch.arange(n_elements, dtype=_DTYPE, device="cuda")
expected_bytes = src.cpu().reshape(-1).view(torch.uint8)
total_bytes = n_elements * _DTYPE.itemsize
block_ids = [7, 2, 5, 0][:n_blocks]
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
worker.flush_saves()
for slot, block_idx in enumerate(block_ids):
block_byte_start = slot * _BLOCK_SIZE_BYTES
block_byte_end = min(block_byte_start + _BLOCK_SIZE_BYTES, total_bytes)
n_written = block_byte_end - block_byte_start
actual_bytes = worker._region.blocks[block_idx, :n_written].view(torch.uint8)
assert torch.equal(
actual_bytes, expected_bytes[block_byte_start:block_byte_end]
), f"block {block_idx} (slot {slot}) bytes mismatch"
if n_written < _BLOCK_SIZE_BYTES:
tail = worker._region.blocks[block_idx, n_written:]
assert torch.all(tail == sentinel), (
f"block {block_idx} tail was overwritten"
)
for idx in set(range(_NUM_BLOCKS)) - set(block_ids):
assert torch.all(worker._region.blocks[idx] == sentinel), (
f"block {idx} (unassigned) was overwritten"
)
def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
"""When the scheduler hasn't pre-allocated blocks for ``mm_hash``,
``save_caches`` + ``flush_saves`` is a pure no-op."""
worker = make_worker()
sentinel = 0x42
worker._region.blocks.fill_(sentinel)
worker.save_caches({}, "h", _meta(saves={}))
worker.flush_saves()
assert torch.all(worker._region.blocks == sentinel)
@pytest.mark.parametrize(
"tp_rank,pcp_rank",
[(1, 0), (0, 1), (1, 1)],
ids=["tp1-pcp0", "tp0-pcp1", "tp1-pcp1"],
)
def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
"""Only TP rank 0 + PCP rank 0 writes to mmap. All other combinations
must skip the save entirely."""
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
sentinel = 0x42
worker._region.blocks.fill_(sentinel)
worker.save_caches(
{"h": torch.zeros(8, dtype=_DTYPE)}, "h", _meta(saves={"h": [0]})
)
worker.flush_saves()
assert torch.all(worker._region.blocks == sentinel)
def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
"""``save_caches`` must raise ``AssertionError`` when the encoder output
is larger than the allocated block space."""
worker = make_worker()
src = torch.zeros(3 * _HIDDEN_DIM, dtype=_DTYPE)
with pytest.raises(AssertionError, match="exceeds allocated blocks"):
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]}))
@_requires_cuda
def test_save_caches_batches_multiple_hashes(make_worker):
"""Multiple save_caches calls are batched into a single flush."""
worker = make_worker()
sentinel = 0x5A
worker._region.blocks.fill_(sentinel)
src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device="cuda")
cache = {"a": src_a, "b": src_b}
worker.save_caches(cache, "a", _meta(saves={"a": [1], "b": [3]}))
worker.save_caches(cache, "b", _meta(saves={"a": [1], "b": [3]}))
worker.flush_saves()
expected_a = src_a.cpu().view(torch.uint8)
expected_b = src_b.cpu().view(torch.uint8)
actual_a = worker._region.blocks[1].view(torch.uint8)
actual_b = worker._region.blocks[3].view(torch.uint8)
assert torch.equal(actual_a, expected_a)
assert torch.equal(actual_b, expected_b)
# Unassigned blocks untouched.
for idx in (0, 2, 4, 5, 6, 7):
assert torch.all(worker._region.blocks[idx] == sentinel)
# ── start_load_caches ────────────────────────────────────────────────────────
@_requires_cuda
def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker):
"""Single batched load across all hashes with correct byte→dtype→shape."""
worker = make_worker()
n_blocks = 3
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
n_blocks, _HIDDEN_DIM
)
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
block_ids = [3, 1, 6]
for i, idx in enumerate(block_ids):
worker._region.blocks[idx].copy_(src_int8[i])
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
out = encoder_cache["h"]
assert out.is_cuda, "consumer worker must place the tensor on the GPU"
assert out.shape == (n_blocks, _HIDDEN_DIM)
assert out.dtype == _DTYPE
assert torch.equal(out.cpu(), src_orig)
@_requires_cuda
def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker):
"""If ``encoder_cache`` already holds the ``mm_hash``, the worker must
not overwrite it."""
worker = make_worker()
worker._region.blocks[0].fill_(0x42)
sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device="cuda")
encoder_cache = {"h": sentinel}
worker.start_load_caches(encoder_cache, _meta(loads={"h": [0]}))
assert encoder_cache["h"] is sentinel, (
"existing encoder_cache entry must not be replaced"
)
@_requires_cuda
def test_start_load_caches_noop_when_loads_is_empty(make_worker):
"""When ``meta.loads`` is empty the early-return must fire."""
worker = make_worker()
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={}))
assert encoder_cache == {}
@_requires_cuda
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
"""Cached entries are preserved while new ones are loaded."""
worker = make_worker()
n_blocks = 2
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
n_blocks, _HIDDEN_DIM
)
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
new_block_ids = [4, 2]
for i, idx in enumerate(new_block_ids):
worker._region.blocks[idx].copy_(src_int8[i])
cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device="cuda")
encoder_cache: dict[str, torch.Tensor] = {"cached_h": cached_tensor}
worker.start_load_caches(
encoder_cache,
_meta(loads={"cached_h": [0], "new_h": new_block_ids}),
)
assert encoder_cache["cached_h"] is cached_tensor
assert "new_h" in encoder_cache
out = encoder_cache["new_h"]
assert out.shape == (n_blocks, _HIDDEN_DIM)
assert out.dtype == _DTYPE
assert torch.equal(out.cpu(), src_orig)
@_requires_cuda
@pytest.mark.parametrize(
"tp_rank,pcp_rank",
[(0, 0), (1, 0), (0, 1), (1, 1)],
ids=["tp0-pcp0", "tp1-pcp0", "tp0-pcp1", "tp1-pcp1"],
)
def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank):
"""All TP/PCP ranks must load from mmap — loads are NOT gated like saves."""
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
n_blocks = 2
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
n_blocks, _HIDDEN_DIM
)
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
block_ids = [1, 3]
for i, idx in enumerate(block_ids):
worker._region.blocks[idx].copy_(src_int8[i])
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
out = encoder_cache["h"]
assert out.is_cuda
assert torch.equal(out.cpu(), src_orig)
# ── round-trip ───────────────────────────────────────────────────────────────
@_requires_cuda
def test_save_then_load_round_trips_bytes(make_worker):
"""Full producer→mmap→consumer byte path in one shot."""
worker = make_worker()
n_blocks = 3
block_ids = [5, 1, 6]
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
n_blocks, _HIDDEN_DIM
)
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
worker.flush_saves()
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
out = encoder_cache["h"]
assert out.shape == src.shape
assert out.dtype == src.dtype
assert torch.equal(out.cpu(), src.cpu())
# ── buffer recycling ────────────────────────────────────────────────────────
@_requires_cuda
def test_buffer_pool_is_reused_across_save_steps(make_worker):
"""After flush_saves, descriptor buffers are returned to the pool and
reused on the next flush — no reallocation."""
worker = make_worker()
src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0]}))
worker.flush_saves()
assert len(worker._buf_pool._pool) == 1
buf_id = id(worker._buf_pool._pool[0].src_ptrs)
# Second step reuses the same buffer.
worker.save_caches({"h": src}, "h", _meta(saves={"h": [1]}))
worker.flush_saves()
assert len(worker._buf_pool._pool) == 1
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
@_requires_cuda
def test_buffer_pool_is_reused_across_load_steps(make_worker):
"""After start_load_caches, descriptor buffers are returned to the pool
and reused on the next call."""
worker = make_worker()
worker._region.blocks[0].fill_(0x01)
worker._region.blocks[1].fill_(0x02)
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={"a": [0]}))
assert len(worker._buf_pool._pool) == 1
buf_id = id(worker._buf_pool._pool[0].src_ptrs)
encoder_cache2: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache2, _meta(loads={"b": [1]}))
assert len(worker._buf_pool._pool) == 1
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
# ── stream management ────────────────────────────────────────────────────────
@_requires_cuda
def test_stream_initialized_at_construction(make_worker):
"""``_load_stream`` must be a fully initialized CUDA stream."""
worker = make_worker()
assert isinstance(worker._load_stream, torch.cuda.Stream)
# ── lifecycle ────────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"pin_available,expected_pinned",
[
(True, True),
(False, False),
],
ids=["available", "unavailable"],
)
def test_init_pins_memory_when_available(pin_available, expected_pinned):
"""Every worker must call ``pin_memory()`` when the platform allows it."""
region = _make_region()
region.pin_memory = MagicMock() # type: ignore[method-assign]
pcp_group = Mock()
pcp_group.rank_in_group = 0
with (
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.create_ec_shared_region",
return_value=region,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.is_pin_memory_available",
return_value=pin_available,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_tensor_model_parallel_rank",
return_value=0,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_pcp_group",
return_value=pcp_group,
),
):
ECCPUWorker(_vllm_config(rank=0))
try:
if expected_pinned:
region.pin_memory.assert_called_once()
else:
region.pin_memory.assert_not_called()
finally:
region.cleanup()
def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
"""``shutdown`` must always call ``region.cleanup`` — and must never
raise."""
worker = object.__new__(ECCPUWorker)
mock_region = Mock(spec=ECSharedRegion)
worker._region = mock_region
worker._load_stream = MagicMock()
worker._save_bufs = None
worker._save_count = 0
worker.shutdown()
worker._load_stream.synchronize.assert_called_once()
mock_region.cleanup.assert_called_once()
mock_region.cleanup.side_effect = RuntimeError("boom")
with caplog_vllm.at_level(logging.DEBUG, logger="vllm"):
worker.shutdown() # exception must be swallowed
assert mock_region.cleanup.call_count == 2
assert any(
"worker region cleanup failed" in r.message
for r in caplog_vllm.records
if r.levelno == logging.DEBUG
)
# ── e2e: scheduler + worker pipeline ────────────────────────────────────────
@_requires_cuda
def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
"""Full pipeline: scheduler allocates blocks, worker saves GPU tensor to
mmap via flush_saves, scheduler marks ready after step delay, worker
loads from mmap back to GPU, and the result matches the original.
Exercises the real scheduler + worker cooperation through a shared
ECSharedRegion, with real CUDA transfers and stream coordination.
"""
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
ECCPUScheduler,
)
# Build worker first (gets us a real region).
worker = make_worker()
region = worker._region
# Build scheduler sharing the same region.
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
class _EC:
is_ec_producer = True
is_ec_consumer = True
class _Cfg:
ec_transfer_config = _EC()
max_concurrent_batches = 1
scheduler = ECCPUScheduler(_Cfg())
# -- Step 1: scheduler allocates, worker saves --
n_blocks = 3
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
n_blocks, _HIDDEN_DIM
)
class _Pos:
offset = 0
length = n_blocks
class _Feature:
identifier = "img_001"
mm_position = _Pos()
class _Request:
request_id = "req_e2e"
mm_features = [_Feature()]
scheduler.update_state_after_alloc(_Request(), 0)
meta_save = scheduler.build_connector_meta(scheduler_output=None)
assert "img_001" in meta_save.saves
encoder_cache = {"img_001": src}
worker.save_caches(encoder_cache, "img_001", meta_save)
worker.flush_saves()
# -- Step 2: readiness delay (max_concurrent_batches=1) --
meta_step2 = scheduler.build_connector_meta(scheduler_output=None)
assert scheduler.has_cache_item("img_001") is True
assert meta_step2.loads == {} # no load requested yet
# -- Step 3: scheduler emits load, worker loads --
scheduler.update_state_after_alloc(_Request(), 0)
meta_load = scheduler.build_connector_meta(scheduler_output=None)
assert "img_001" in meta_load.loads
assert meta_load.loads["img_001"] == meta_save.saves["img_001"]
load_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(load_cache, meta_load)
out = load_cache["img_001"]
assert out.is_cuda
assert out.shape == src.shape
assert out.dtype == src.dtype
assert torch.equal(out.cpu(), src.cpu())
scheduler.shutdown()
@@ -0,0 +1,158 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E test for the ECCPUConnector.
Verifies:
- Accuracy: outputs from EC CPU cache match fresh encoder computation.
- Latency: loading from EC CPU cache is faster than a cold encoder run.
Requires a CUDA GPU and the Qwen2-VL-2B-Instruct model.
"""
import time
import pytest
from tqdm import tqdm
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.config import ECTransferConfig
from vllm.platforms import current_platform
MODEL = "Qwen/Qwen2-VL-2B-Instruct"
EC_CPU_BYTES = 500 << 20 # 500 MiB
def _build_image_prompt(image):
"""Build a Qwen2-VL prompt with a single image."""
return {
"prompt": (
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
"<|im_start|>user\n"
"<|vision_start|><|image_pad|><|vision_end|>"
"Describe this image briefly.<|im_end|>\n"
"<|im_start|>assistant\n"
),
"multi_modal_data": {"image": image},
}
def _make_unique_image(base_image, iteration: int):
"""Create a unique image by modifying a pixel to change the mm_hash."""
img = base_image.copy()
img.putpixel((0, 0), (iteration % 256, iteration // 256 % 256, 0))
return img
def _wait_for_ec_ready(llm: LLM) -> None:
"""Send a dummy request to force a scheduler step.
After generate() returns, the EC entry's data is in the mmap but
readiness may not have fired yet (first-finish fires in the NEXT
scheduler step). This forces that step.
"""
llm.generate("hi", SamplingParams(max_tokens=1), use_tqdm=False)
def _latency_test(llm: LLM) -> None:
"""Verify EC CPU cache hit is faster than cold encoder computation."""
if not current_platform.is_cuda():
pytest.skip("Latency test requires CUDA")
sampling_params = SamplingParams(max_tokens=1, temperature=0)
base_image = ImageAsset("stop_sign").pil_image
num_times_ec_better_than_cold = 0
num_tests = 10
total_cold_time = 0.0
total_ec_hit_time = 0.0
for i in tqdm(range(num_tests), desc="EC latency test"):
# Use a unique image each iteration so the cold run is truly cold
# (EC has never seen this mm_hash before).
image = _make_unique_image(base_image, i)
prompt = _build_image_prompt(image)
# Cold run: encoder computes from scratch, saves to EC CPU.
llm.llm_engine.reset_encoder_cache()
start_time = time.time()
llm.generate([prompt], sampling_params, use_tqdm=False)
cold_time = time.time() - start_time
total_cold_time += cold_time
# Ensure EC entry is marked ready.
_wait_for_ec_ready(llm)
# Clear GPU encoder cache but keep EC CPU cache intact.
llm.llm_engine.reset_encoder_cache()
# EC hit: scheduler loads from CPU mmap instead of re-encoding.
start_time = time.time()
llm.generate([prompt], sampling_params, use_tqdm=False)
ec_hit_time = time.time() - start_time
total_ec_hit_time += ec_hit_time
if ec_hit_time < cold_time:
num_times_ec_better_than_cold += 1
print("Average times:")
print(f" Cold: {total_cold_time * 1000 / num_tests:.2f}ms")
print(f" EC hit: {total_ec_hit_time * 1000 / num_tests:.2f}ms")
assert num_times_ec_better_than_cold >= 0.8 * num_tests, (
f"EC hit was faster only {num_times_ec_better_than_cold}/{num_tests} "
f"times (expected >= 80%)"
)
def _accuracy_test(llm: LLM) -> None:
"""Verify EC CPU cache produces correct outputs."""
sampling_params = SamplingParams(max_tokens=8, temperature=0)
image = ImageAsset("stop_sign").pil_image
prompt = _build_image_prompt(image)
# Seed: generate with fresh encoder to establish baseline output.
llm.llm_engine.reset_encoder_cache()
baseline = llm.generate([prompt], sampling_params, use_tqdm=False)
baseline_text = baseline[0].outputs[0].text
# Ensure EC entry is marked ready.
_wait_for_ec_ready(llm)
# Clear GPU encoder cache so subsequent runs go through EC CPU path.
llm.llm_engine.reset_encoder_cache()
# Generate multiple times from EC cache and check correctness.
test_count = 20
results = llm.generate([prompt] * test_count, sampling_params, use_tqdm=False)
success_count = sum(1 for r in results if r.outputs[0].text == baseline_text)
assert success_count >= 0.5 * test_count, (
f"Only {success_count}/{test_count} outputs matched baseline "
f"(expected >= 50%). Baseline: {baseline_text!r}"
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_ec_cpu_offloading() -> None:
"""Tests ECCPUConnector accuracy and latency with a VLM model."""
ec_transfer_config = ECTransferConfig(
ec_connector="ECCPUConnector",
ec_role="ec_both",
ec_connector_extra_config={"ec_cpu_bytes": EC_CPU_BYTES},
)
llm = LLM(
model=MODEL,
max_model_len=2048,
gpu_memory_utilization=0.5,
enforce_eager=True,
ec_transfer_config=ec_transfer_config,
)
try:
_latency_test(llm)
_accuracy_test(llm)
finally:
del llm
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECCPUConnectorMetadata."""
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
)
def test_metadata_is_ec_connector_metadata():
"""ECCPUConnectorMetadata must be a subclass of ECConnectorMetadata."""
meta = ECCPUConnectorMetadata()
assert isinstance(meta, ECConnectorMetadata)
def test_metadata_initialization():
"""ECCPUConnectorMetadata initializes with empty saves and loads dicts."""
meta = ECCPUConnectorMetadata()
assert hasattr(meta, "saves")
assert hasattr(meta, "loads")
assert meta.saves == {}
assert meta.loads == {}
def test_metadata_saves_dict_operations():
"""The saves dict supports standard dict operations."""
meta = ECCPUConnectorMetadata()
# Add multiple entries
meta.saves["mm_hash_1"] = [0, 1, 2]
meta.saves["mm_hash_2"] = [10, 11]
assert "mm_hash_1" in meta.saves
assert "mm_hash_2" in meta.saves
assert len(meta.saves) == 2
assert meta.saves["mm_hash_1"] == [0, 1, 2]
assert meta.saves["mm_hash_2"] == [10, 11]
def test_metadata_loads_dict_operations():
"""The loads dict supports standard dict operations."""
meta = ECCPUConnectorMetadata()
# Add multiple entries
meta.loads["mm_hash_1"] = [5, 6, 7]
meta.loads["mm_hash_2"] = [100]
assert "mm_hash_1" in meta.loads
assert "mm_hash_2" in meta.loads
assert len(meta.loads) == 2
assert meta.loads["mm_hash_1"] == [5, 6, 7]
assert meta.loads["mm_hash_2"] == [100]
def test_metadata_saves_and_loads_are_independent():
"""saves and loads dicts are independent."""
meta = ECCPUConnectorMetadata()
meta.saves["key"] = [1, 2]
meta.loads["key"] = [3, 4]
# Same key but different dicts should not interfere
assert meta.saves["key"] == [1, 2]
assert meta.loads["key"] == [3, 4]
# Modifying one should not affect the other
meta.saves["other"] = [99]
assert "other" not in meta.loads
def test_metadata_block_indices_are_integers():
"""Block indices in saves/loads lists should be integers."""
meta = ECCPUConnectorMetadata()
meta.saves["hash"] = [0, 100, 999, 65535]
meta.loads["hash"] = [1, 2, 3]
for idx in meta.saves["hash"]:
assert isinstance(idx, int)
for idx in meta.loads["hash"]:
assert isinstance(idx, int)
def test_metadata_multiple_instances_are_independent():
"""Different ECCPUConnectorMetadata instances should not share state."""
meta1 = ECCPUConnectorMetadata()
meta2 = ECCPUConnectorMetadata()
meta1.saves["x"] = [1]
meta2.saves["y"] = [2]
assert "x" in meta1.saves
assert "x" not in meta2.saves
assert "y" not in meta1.saves
assert "y" in meta2.saves