[EC Connector] CPU Offloading EC Connector (#47423)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Signed-off-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
omerpaz95
2026-07-13 20:09:41 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 62286308c9
commit 43c8cbf79b
18 changed files with 3008 additions and 0 deletions
@@ -0,0 +1,306 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for EmbeddingCache.
Pure integer bookkeeping — no mmap, torch, or CUDA — so these run anywhere.
"""
import threading
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
def _cache(num_blocks: int = 8) -> EmbeddingCache:
return EmbeddingCache(num_blocks)
# ── alloc ────────────────────────────────────────────────────────────────────
def test_alloc_returns_unique_block_ids():
cache = _cache()
entry = cache.alloc("a", 3)
assert entry is not None
assert len(entry.block_ids) == 3
assert len(set(entry.block_ids)) == 3
assert all(0 <= i < 8 for i in entry.block_ids)
def test_alloc_returns_tuple():
cache = _cache()
entry = cache.alloc("a", 2)
assert isinstance(entry.block_ids, tuple)
def test_alloc_returns_none_when_full():
cache = _cache(num_blocks=4)
cache.alloc("a", 4)
assert cache.alloc("b", 1) is None
def test_alloc_asserts_duplicate_key():
cache = _cache()
cache.alloc("a", 2)
with pytest.raises(AssertionError, match="duplicate alloc"):
cache.alloc("a", 2)
def test_alloc_asserts_n_blocks_exceeds_capacity():
cache = _cache(num_blocks=4)
with pytest.raises(AssertionError, match="capacity"):
cache.alloc("a", 5)
def test_alloc_zero_blocks():
cache = _cache()
entry = cache.alloc("a", 0)
assert entry is not None
assert entry.block_ids == ()
def test_alloc_starts_not_ready():
cache = _cache()
entry = cache.alloc("a", 2)
assert not entry.ready
assert not entry.evictable
# ── get ──────────────────────────────────────────────────────────────────────
def test_get_missing_returns_none():
cache = _cache()
assert cache.get("a") is None
def test_get_returns_entry():
cache = _cache()
alloc_entry = cache.alloc("a", 2)
got = cache.get("a")
assert got is alloc_entry
def test_get_returns_not_ready_entry():
cache = _cache()
cache.alloc("a", 2)
entry = cache.get("a")
assert entry is not None
assert not entry.ready
# ── mark_ready ───────────────────────────────────────────────────────────────
def test_mark_ready_transitions():
cache = _cache()
cache.alloc("a", 2)
entry = cache.get("a")
assert not entry.ready
cache.mark_ready("a")
assert entry.ready
assert entry.evictable
def test_mark_ready_asserts_already_ready():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
with pytest.raises(AssertionError, match="already-ready"):
cache.mark_ready("a")
def test_mark_ready_asserts_missing_key():
cache = _cache()
with pytest.raises(KeyError):
cache.mark_ready("nonexistent")
# ── pin / unpin ──────────────────────────────────────────────────────────────
def test_pin_asserts_not_ready():
cache = _cache()
cache.alloc("a", 2)
with pytest.raises(AssertionError, match="not-ready"):
cache.pin("a")
def test_pin_unpin_basic():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
entry = cache.get("a")
assert not entry.evictable
cache.unpin("a")
assert entry.evictable
def test_nested_pin():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
cache.pin("a")
cache.unpin("a")
entry = cache.get("a")
assert not entry.evictable # still pinned (count=1)
cache.unpin("a")
assert entry.evictable
def test_unpin_asserts_on_unpinned():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
with pytest.raises(AssertionError, match="unpinned"):
cache.unpin("a")
# ── eviction ─────────────────────────────────────────────────────────────────
def test_eviction_frees_ready_unpinned_fifo():
cache = _cache(num_blocks=4)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.alloc("b", 2)
cache.mark_ready("b")
# Pool full. Alloc should evict "a" (oldest).
entry_c = cache.alloc("c", 2)
assert entry_c is not None
assert cache.get("a") is None # evicted
assert cache.get("b") is not None # still there
def test_eviction_skips_pinned():
cache = _cache(num_blocks=4)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
cache.alloc("b", 2)
cache.mark_ready("b")
# "a" is pinned, so "b" should be evicted instead.
entry_c = cache.alloc("c", 2)
assert entry_c is not None
assert cache.get("a") is not None # pinned, survived
assert cache.get("b") is None # evicted
cache.unpin("a")
def test_eviction_skips_not_ready():
cache = _cache(num_blocks=4)
cache.alloc("a", 2) # not ready
cache.alloc("b", 2)
cache.mark_ready("b")
# "a" is not ready (non-evictable), "b" is evictable.
entry_c = cache.alloc("c", 2)
assert entry_c is not None
assert cache.get("a") is not None # not ready, survived
assert cache.get("b") is None # evicted
cache.mark_ready("a") # clean up
def test_eviction_returns_none_if_all_non_evictable():
cache = _cache(num_blocks=4)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
cache.alloc("b", 2) # not ready
# Both entries are non-evictable (a=pinned, b=not-ready).
assert cache.alloc("c", 1) is None
# Fail-fast: nothing was evicted.
assert cache.get("a") is not None
assert cache.get("b") is not None
cache.unpin("a")
def test_fail_fast_no_wasted_eviction():
cache = _cache(num_blocks=6)
cache.alloc("a", 2)
cache.mark_ready("a") # evictable: 2 blocks
cache.alloc("b", 2) # not ready
cache.alloc("c", 2) # not ready
# free=0, evictable=2, need 3 -> cannot succeed
assert cache.alloc("d", 3) is None
# "a" was NOT evicted despite being evictable
assert cache.get("a") is not None
def test_eviction_multiple_entries():
cache = _cache(num_blocks=6)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.alloc("b", 2)
cache.mark_ready("b")
cache.alloc("c", 2)
cache.mark_ready("c")
# Need 4 blocks: must evict both "a" and "b".
entry_d = cache.alloc("d", 4)
assert entry_d is not None
assert cache.get("a") is None
assert cache.get("b") is None
assert cache.get("c") is not None
# ── thread safety ────────────────────────────────────────────────────────────
def test_concurrent_pin_unpin():
cache = _cache(num_blocks=8)
cache.alloc("a", 4)
cache.mark_ready("a")
errors: list[Exception] = []
barrier = threading.Barrier(8)
def worker():
barrier.wait()
try:
for _ in range(500):
cache.pin("a")
cache.unpin("a")
except Exception as exc:
errors.append(exc)
threads = [threading.Thread(target=worker) for _ in range(8)]
for t in threads:
t.start()
for t in threads:
t.join()
assert not errors, errors
entry = cache.get("a")
assert entry.evictable # all pins balanced
def test_concurrent_alloc_evict():
cache = _cache(num_blocks=4)
errors: list[Exception] = []
barrier = threading.Barrier(4)
counter = {"i": 0}
counter_lock = threading.Lock()
def worker():
barrier.wait()
try:
for _ in range(200):
with counter_lock:
key = f"k{counter['i']}"
counter["i"] += 1
entry = cache.alloc(key, 1)
if entry is not None:
cache.mark_ready(key)
except Exception as exc:
errors.append(exc)
threads = [threading.Thread(target=worker) for _ in range(4)]
for t in threads:
t.start()
for t in threads:
t.join()
assert not errors, errors
@@ -0,0 +1,480 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
_N = 16
_BS = 64
class _Pos:
def __init__(self, offset, length):
self.offset = offset
self.length = length
class _Feature:
def __init__(self, mm_hash, length=1, identifier=None):
self.mm_hash = mm_hash
self.identifier = identifier if identifier is not None else mm_hash
self.mm_position = _Pos(0, length)
class _Request:
_counter = 0
def __init__(self, features, request_id=None):
self.mm_features = features
if request_id is None:
_Request._counter += 1
self.request_id = f"req_{_Request._counter}"
else:
self.request_id = request_id
def _make_scheduler(
monkeypatch,
num_blocks=_N,
max_concurrent_batches=1,
*,
is_producer=True,
is_consumer=True,
) -> ECCPUScheduler:
region = ECSharedRegion(
engine_id=str(uuid.uuid4()),
num_blocks=num_blocks,
block_size_bytes=_BS,
)
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
_is_prod = is_producer
_is_cons = is_consumer
_mcb = max_concurrent_batches
class _EC:
is_ec_producer = _is_prod
is_ec_consumer = _is_cons
class _Cfg:
ec_transfer_config = _EC()
@property
def max_concurrent_batches(self):
return _mcb
return ECCPUScheduler(_Cfg())
class _SchedulerOutput:
def __init__(self, finished_req_ids=None):
self.finished_req_ids = finished_req_ids or set()
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
"""Pre-populate a ready cache entry backed by real blocks."""
s._cache.alloc(mm_hash, n_blocks)
s._cache.mark_ready(mm_hash)
def test_offload_reuse_cycle(monkeypatch):
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1)])
# Step A: first sight — allocate and save.
assert s.has_cache_item("h1") is False
assert s.ensure_cache_available(req, 0) is True
s.update_state_after_alloc(req, 0)
meta_a = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_a.saves
assert meta_a.loads == {}
# Not ready yet (max_concurrent_batches=1 means ready next step).
assert s.has_cache_item("h1") is False
# Step B: _step_readiness marks h1 ready (no new allocs).
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is True
# Step C: reload now works — entry is ready at update_state_after_alloc time.
s.update_state_after_alloc(req, 0)
meta_c = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_c.loads
assert meta_c.loads["h1"] == meta_a.saves["h1"]
s.shutdown()
def test_has_cache_item_false_when_not_consumer(monkeypatch):
s = _make_scheduler(monkeypatch, is_consumer=False)
assert s.has_cache_item("anything") is False
s.shutdown()
def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
"""The connector must key the encoder cache on feature.identifier (what
has_cache_item is called with), NOT feature.mm_hash."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")])
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "ENC_KEY" in meta.saves
assert "PROC_KEY" not in meta.saves
# Not ready yet (1 step delay).
assert s.has_cache_item("ENC_KEY") is False
# After another step it becomes ready — keyed on identifier.
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("ENC_KEY") is True
assert s.has_cache_item("PROC_KEY") is False
s.shutdown()
def test_cpu_region_fifo_eviction(monkeypatch):
# Region holds exactly 2 one-block encodings.
s = _make_scheduler(monkeypatch, num_blocks=2)
# Fill with A and B, marking each ready.
for h in ("A", "B"):
s.update_state_after_alloc(_Request([_Feature(h, length=1)]), 0)
s.build_connector_meta(scheduler_output=None)
# After the step above, A's step was called in B's build. A is ready.
# B was submitted in the last build, so one more step makes it ready.
s.build_connector_meta(scheduler_output=None) # marks B ready
assert s.has_cache_item("A") is True
assert s.has_cache_item("B") is True
# Allocate C: must evict A (oldest) to make room.
s.update_state_after_alloc(_Request([_Feature("C", length=1)]), 0)
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("A") is False # evicted
assert s.has_cache_item("B") is True
def test_multiple_mm_items_per_request(monkeypatch):
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1), _Feature("h2", length=1)])
s.update_state_after_alloc(req, 0)
s.update_state_after_alloc(req, 1)
meta = s.build_connector_meta(scheduler_output=None)
assert set(meta.saves) == {"h1", "h2"}
# Step delay: mark both ready.
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is True
assert s.has_cache_item("h2") is True
# Now reload both.
s.update_state_after_alloc(req, 0)
s.update_state_after_alloc(req, 1)
meta2 = s.build_connector_meta(scheduler_output=None)
assert set(meta2.loads) == {"h1", "h2"}
s.shutdown()
def test_load_returns_correct_block_ids(monkeypatch):
"""meta.loads must contain the same block IDs that meta.saves
allocated — verified through the public API only."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("a", length=2)])
s.update_state_after_alloc(req, 0)
meta_save = s.build_connector_meta(scheduler_output=None)
assert "a" in meta_save.saves
# Readiness delay.
s.build_connector_meta(scheduler_output=None)
# Reload.
s.update_state_after_alloc(req, 0)
meta_load = s.build_connector_meta(scheduler_output=None)
assert meta_load.loads["a"] == meta_save.saves["a"]
s.shutdown()
def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=2)
_seed_cached(s, "b", n_blocks=2)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "a" in meta.loads
assert "b" not in meta.loads
s.shutdown()
def test_repeated_reload_same_step_loads_once(monkeypatch):
"""The same mm_hash requested twice in one step must appear in
meta.loads exactly once."""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=2)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert list(meta.loads.keys()).count("a") == 1
s.shutdown()
def test_load_not_emitted_for_uncached_entry(monkeypatch):
"""A feature that was never saved must not appear in meta.loads."""
s = _make_scheduler(monkeypatch)
s.update_state_after_alloc(_Request([_Feature("missing")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert meta.loads == {}
s.shutdown()
def test_delayed_unpin_protects_blocks_during_worker_read(monkeypatch):
"""Blocks being loaded must not be evictable until the delayed unpin
fires. Observable: a save that needs eviction cannot reclaim a
still-pinned loaded entry."""
# 2 blocks, both occupied by "a" (ready).
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
# Load "a" — pins it.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "a" in meta.loads
# Try to save "b" (needs 2 blocks, but "a" is pinned) — must fail.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta2.saves # can't evict pinned "a"
# After the delay, "a" unpins. Now "b" can evict it.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta3 = s.build_connector_meta(scheduler_output=None)
assert "b" in meta3.saves
s.shutdown()
def test_delayed_unpin_depth_2(monkeypatch):
"""With max_concurrent_batches=2, loaded blocks stay pinned for 2 steps
after build. Observable via eviction failure."""
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2)
_seed_cached(s, "a", n_blocks=2)
# Load "a" → pinned.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
s.build_connector_meta(scheduler_output=None)
# Step 1 after load: still pinned — can't evict for "b".
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta1 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta1.saves
# Step 2 after load: still pinned.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta2.saves
# Step 3: unpin fires, now eviction works.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta3 = s.build_connector_meta(scheduler_output=None)
assert "b" in meta3.saves
s.shutdown()
def test_shutdown_unpins_pending_loads(monkeypatch):
"""Shutdown must unpin entries still in _pending_loads (not yet drained
to the unpin deque). Observable: after shutdown, the entry is evictable
(alloc can reclaim its blocks)."""
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
# Don't call build_connector_meta — "a" stays in _pending_loads.
s.shutdown()
# After shutdown, scheduler reports no items (consumer role is off).
assert s.has_cache_item("a") is False
def test_shutdown_unpins_deferred_unpin_queue(monkeypatch):
"""Shutdown must unpin entries queued in the deferred unpin deque
(already drained from _pending_loads but not yet expired)."""
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2)
_seed_cached(s, "a", n_blocks=2)
# Load "a" and build — moves to unpin deque.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
s.build_connector_meta(scheduler_output=None)
# "a" is in the deferred unpin deque (depth=2, not yet expired).
# Shutdown must unpin it.
s.shutdown()
assert s.has_cache_item("a") is False
def test_eviction_skips_entry_pinned_by_pending_load(monkeypatch):
"""A loaded entry still within the unpin delay window must survive
eviction attempts from new saves."""
# 2 blocks: A occupies both, ready.
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "A", n_blocks=2)
# Load A → pinned.
s.update_state_after_alloc(_Request([_Feature("A")]), 0)
meta_load = s.build_connector_meta(scheduler_output=None)
assert "A" in meta_load.loads
# Try to save C (needs 2 blocks) — can't evict pinned A.
s.update_state_after_alloc(_Request([_Feature("C", length=2)]), 0)
meta_save = s.build_connector_meta(scheduler_output=None)
assert "C" not in meta_save.saves
assert s.has_cache_item("A") is True # survived
s.shutdown()
def test_region_full_skips_save_and_never_blocks(monkeypatch):
"""When the region is fully occupied by pinned entries, new saves are
silently skipped and ensure_cache_available never blocks."""
s = _make_scheduler(monkeypatch, num_blocks=1)
_seed_cached(s, "pinned", n_blocks=1)
s._cache.pin("pinned")
req = _Request([_Feature("new", length=1)])
assert s.ensure_cache_available(req, 0) is True
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "new" not in meta.saves
assert s.has_cache_item("new") is False
s._cache.unpin("pinned")
s.shutdown()
def test_step_readiness_depth_2(monkeypatch):
"""With max_concurrent_batches=2, entries need 2 steps to become ready."""
s = _make_scheduler(monkeypatch, max_concurrent_batches=2)
req = _Request([_Feature("h1", length=1)])
s.update_state_after_alloc(req, 0)
meta_0 = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_0.saves
assert s.has_cache_item("h1") is False
# One step later: still not ready (depth=2).
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is False
# Two steps later: now ready.
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is True
s.shutdown()
def test_producer_only_never_emits_loads(monkeypatch):
"""A producer-only scheduler must never populate meta.loads, even when
entries are ready."""
s = _make_scheduler(monkeypatch, is_consumer=False)
req = _Request([_Feature("h1", length=1)])
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta.saves
assert meta.loads == {}
# Mark ready.
s.build_connector_meta(scheduler_output=None)
# Even if we re-encounter the same feature, no load.
s.update_state_after_alloc(req, 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert meta2.loads == {}
s.shutdown()
def test_consumer_only_never_emits_saves(monkeypatch):
"""A consumer-only scheduler must never populate meta.saves."""
s = _make_scheduler(monkeypatch, is_producer=False)
_seed_cached(s, "a", n_blocks=1)
req = _Request([_Feature("a", length=1)])
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert meta.saves == {}
assert "a" in meta.loads
s.shutdown()
def test_consumer_only_has_cache_item(monkeypatch):
"""Consumer-only: has_cache_item works normally for ready entries."""
s = _make_scheduler(monkeypatch, is_producer=False)
assert s.has_cache_item("a") is False
_seed_cached(s, "a", n_blocks=1)
assert s.has_cache_item("a") is True
s.shutdown()
def test_save_not_emitted_for_already_cached_entry(monkeypatch):
"""An entry already in the cache (from a prior save) must not trigger
a second allocation or appear in meta.saves again."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1)])
# First encounter → save.
s.update_state_after_alloc(req, 0)
meta1 = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta1.saves
# Second encounter (same step or later) → no save.
s.update_state_after_alloc(req, 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "h1" not in meta2.saves
s.shutdown()
# ── first-finish fast-path (idle engine) ────────────────────────────────────
def test_first_finish_marks_ready_immediately(monkeypatch):
"""When the originating request finishes, the save entry becomes ready
without waiting for the step-count delay."""
s = _make_scheduler(monkeypatch, max_concurrent_batches=5)
req = _Request([_Feature("h1", length=1)], request_id="req_x")
s.update_state_after_alloc(req, 0)
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("h1") is False
# Request finishes — readiness should fire immediately.
s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_x"}))
assert s.has_cache_item("h1") is True
s.shutdown()
def test_first_finish_unpins_immediately(monkeypatch):
"""When the originating request finishes, the load entry is unpinned
without waiting for the step-count delay."""
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=5)
_seed_cached(s, "a", n_blocks=2)
req = _Request([_Feature("a")], request_id="req_y")
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "a" in meta.loads
# "a" is pinned. Try to evict — must fail.
s.update_state_after_alloc(
_Request([_Feature("b", length=2)], request_id="req_z"), 0
)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta2.saves
# req_y finishes — unpin fires in build_connector_meta (fast-path).
s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_y"}))
# Next step: "a" is now evictable, so "b" can be allocated.
s.update_state_after_alloc(
_Request([_Feature("b", length=2)], request_id="req_w"), 0
)
meta3 = s.build_connector_meta(scheduler_output=None)
assert "b" in meta3.saves
s.shutdown()
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for StepTracker."""
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
StepTracker,
)
def test_basic_step_count_expiry_depth_1():
"""With max_concurrent_batches=1, entry expires after 1 step."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
# Step 1: entry was added, committed to slot. Deque fills to maxlen=1,
# but no expiry yet (the entry is in the slot just committed).
result = t.step(set())
assert result == []
# Step 2: deque is full, oldest slot (containing h1) expires.
result = t.step(set())
assert result == ["h1"]
def test_basic_step_count_expiry_depth_2():
"""With max_concurrent_batches=2, entry expires after 2 steps."""
t = StepTracker(max_concurrent_batches=2)
t.add("h1", "req_a")
assert t.step(set()) == [] # step 1: committed
assert t.step(set()) == [] # step 2: deque fills but h1 in newer slot
assert t.step(set()) == ["h1"] # step 3: h1's slot is oldest, expires
def test_first_finish_fast_path():
"""Entry is returned immediately when request_id appears in finished."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
result = t.step({"req_a"})
assert result == ["h1"]
def test_no_double_processing_fast_path_then_expiry():
"""Entry fast-pathed should NOT fire again on deque expiry."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
# Fast-path fires h1.
result = t.step({"req_a"})
assert result == ["h1"]
# Normal expiry would fire now, but h1 is already processed.
result = t.step(set())
assert result == []
def test_no_double_processing_expiry_then_fast_path():
"""Entry expired via step-count should NOT fire again via fast-path."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
t.step(set()) # commit
result = t.step(set()) # expiry fires h1
assert result == ["h1"]
# req_a finishes later — must not re-fire.
result = t.step({"req_a"})
assert result == []
def test_multiplicity_same_hash_different_requests():
"""Two adds of the same mm_hash (different requests) → returned twice."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
t.add("h1", "req_b")
# Both requests finish in the same step.
result = t.step({"req_a", "req_b"})
assert sorted(result) == ["h1", "h1"]
def test_mixed_expiry_and_fast_path():
"""Some entries expire, others fast-path, in the same step."""
t = StepTracker(max_concurrent_batches=1)
t.add("h1", "req_a")
t.step(set()) # commit h1's slot
# Add h2 in the next step.
t.add("h2", "req_b")
# This step: h1's slot expires AND req_b finishes.
result = t.step({"req_b"})
assert set(result) == {"h1", "h2"}
def test_drain_all_returns_unprocessed():
"""drain_all returns everything pending and clears state."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
t.add("h2", "req_b")
t.step(set()) # commit both
t.add("h3", "req_c") # in _current, not yet committed
result = t.drain_all()
assert set(result) == {"h1", "h2", "h3"}
# After drain, everything is empty.
assert t.step(set()) == []
assert t.drain_all() == []
def test_drain_all_skips_already_processed():
"""drain_all should not re-return entries already fast-pathed."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
t.add("h2", "req_b")
# Fast-path h1.
result = t.step({"req_a"})
assert result == ["h1"]
# drain should only return h2.
result = t.drain_all()
assert result == ["h2"]
def test_empty_step_returns_nothing():
"""step() with nothing pending returns empty list."""
t = StepTracker(max_concurrent_batches=2)
assert t.step(set()) == []
assert t.step({"req_x"}) == []
def test_add_after_step_goes_to_next_slot():
"""Entries added after a step go into the next slot, not the current."""
t = StepTracker(max_concurrent_batches=1)
t.step(set()) # empty step
t.add("h1", "req_a")
# h1 is in _current. Next step commits it.
t.step(set())
# Now the deque is full (1 slot with h1). Next step expires it.
result = t.step(set())
assert result == ["h1"]
def test_fast_path_on_current_entries():
"""Entries still in _current (not yet committed) can be fast-pathed."""
t = StepTracker(max_concurrent_batches=3)
t.add("h1", "req_a")
# req_a finishes in the same step h1 was added.
result = t.step({"req_a"})
assert result == ["h1"]
# Entry is committed to slot but marked processed — won't re-fire.
for _ in range(5):
assert t.step(set()) == []
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
def _cfg():
ec = MagicMock()
ec.is_ec_producer = True
ec.is_ec_consumer = True
cfg = MagicMock()
cfg.ec_transfer_config = ec
return cfg
def test_scheduler_role_builds_only_scheduler(monkeypatch):
fake_sched = MagicMock()
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
assert c.connector_scheduler is fake_sched
assert c.connector_worker is None
c.has_cache_item("x")
fake_sched.has_cache_item.assert_called_once_with("x")
def test_worker_role_builds_only_worker(monkeypatch):
fake_worker = MagicMock()
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: fake_worker)
c = ECCPUConnector(_cfg(), ECConnectorRole.WORKER)
assert c.connector_worker is fake_worker
assert c.connector_scheduler is None
def test_request_finished_inherited_noop(monkeypatch):
monkeypatch.setattr(
ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock()
)
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
assert c.request_finished(MagicMock()) == (False, None)
def test_factory_registered():
cls = ECConnectorFactory._registry["ECCPUConnector"]()
assert cls is ECCPUConnector
@@ -0,0 +1,170 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECSharedRegion (the mmap substrate)."""
import os
import uuid
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
_wait_for_file_size,
)
def _make_region(num_blocks: int = 8, block_size_bytes: int = 64) -> ECSharedRegion:
return ECSharedRegion(
engine_id=str(uuid.uuid4()),
num_blocks=num_blocks,
block_size_bytes=block_size_bytes,
)
@pytest.fixture
def region() -> ECSharedRegion:
r = _make_region()
yield r
r.cleanup()
# ── mmap sharing between two instances ───────────────────────────────────────
def test_second_instance_opens_existing_file_and_shares_memory():
instance_id = str(uuid.uuid4())
r1 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
try:
r2 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
try:
assert r1._is_creator
assert not r2._is_creator
# Both map the same physical pages; writes via r1 are visible via r2.
r1.blocks[0, :4] = torch.tensor([10, 20, 30, 40], dtype=torch.int8)
assert r2.blocks[0, :4].tolist() == [10, 20, 30, 40]
finally:
r2.cleanup()
finally:
r1.cleanup()
def test_only_creator_unlinks_file_on_cleanup():
"""Critical contract: if the non-creator unlinks, the creator's mmap path
becomes a dangling backing file and a third opener would create a new one
out from under the creator."""
instance_id = str(uuid.uuid4())
r1 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
path = r1._mmap_path
r2 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
# Non-creator goes away first — file must still be on disk for r1.
r2.cleanup()
assert os.path.exists(path), "non-creator cleanup must not unlink the file"
# Creator goes away — file is removed.
r1.cleanup()
assert not os.path.exists(path), "creator cleanup must unlink the file"
# ── _wait_for_file_size ──────────────────────────────────────────────────────
def test_wait_for_file_size_returns_when_already_big_enough(tmp_path):
"""The fast path: file already at expected size — return immediately."""
p = tmp_path / "f.bin"
p.write_bytes(b"\x00" * 128)
fd = os.open(str(p), os.O_RDONLY)
try:
_wait_for_file_size(fd, expected_size=128, timeout=1.0) # must not raise
finally:
os.close(fd)
def test_wait_for_file_size_times_out_when_file_stays_empty(tmp_path):
p = tmp_path / "f.bin"
p.write_bytes(b"")
fd = os.open(str(p), os.O_RDONLY)
try:
with pytest.raises(TimeoutError):
_wait_for_file_size(fd, expected_size=4096, timeout=0.05)
finally:
os.close(fd)
# ── pin_memory ───────────────────────────────────────────────────────────────
def test_pin_memory_success_sets_flag(region):
"""When cudaHostRegister returns 0, _is_pinned flips to True
and cleanup will correspondingly call cudaHostUnregister."""
fake_cudart = MagicMock()
success = MagicMock()
success.value = 0
fake_cudart.cudaHostRegister.return_value = success
fake_cudart.cudaHostUnregister.return_value = success
with (
patch("torch.cuda.is_available", return_value=True),
patch("torch.cuda.cudart", return_value=fake_cudart),
):
region.pin_memory()
assert region._is_pinned is True
# cleanup must pair with cudaHostUnregister exactly once.
region.cleanup()
fake_cudart.cudaHostUnregister.assert_called_once()
def test_pin_memory_failure_leaves_flag_false():
"""If cudaHostRegister fails (non-zero), don't pretend it succeeded —
cleanup must NOT call cudaHostUnregister on memory we never registered."""
r = _make_region()
try:
fake_cudart = MagicMock()
fail = MagicMock()
fail.value = 1 # non-zero == error
fake_cudart.cudaHostRegister.return_value = fail
with (
patch("torch.cuda.is_available", return_value=True),
patch("torch.cuda.cudart", return_value=fake_cudart),
):
r.pin_memory()
assert r._is_pinned is False
# Now run cleanup and verify cudaHostUnregister was NOT called.
r.cleanup()
fake_cudart.cudaHostUnregister.assert_not_called()
finally:
r.cleanup()
def test_pin_memory_noop_without_cuda(region):
"""pin_memory is a no-op when CUDA is not available."""
with patch("torch.cuda.is_available", return_value=False):
region.pin_memory()
assert region._is_pinned is False
# ── cleanup idempotency ───────────────────────────────────────────────────────
def test_cleanup_is_idempotent(region):
region.cleanup()
region.cleanup() # fixture calls a third time — must not raise
@@ -0,0 +1,598 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ECCPUWorker.
The byte-level tests exercise real CUDA stream/event coordination
against a real ``ECSharedRegion`` mmap and are skipped on hosts without
CUDA. The lifecycle tests don't fire any ``torch.cuda.*`` primitives and
run anywhere.
Mocking policy
--------------
- ``create_ec_shared_region`` is patched to inject a real ``ECSharedRegion``
(mmap backed) with deterministic small dimensions, so byte assertions
hit the actual mmap. The real responsibility of the worker — block
index → mmap byte mapping, stream coordination, dtype/shape — is
exercised end to end against a real GPU.
- ``is_pin_memory_available`` is patched per-test so the rank-gating
path can be probed deterministically without depending on host caps.
- The shutdown test bypasses ``__init__`` via ``object.__new__`` and
injects a ``MagicMock`` region.
"""
import contextlib
import logging
import uuid
from unittest.mock import MagicMock, Mock, patch
import pytest
import torch
from vllm.config import VllmConfig
from vllm.config.parallel import ParallelConfig
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker
# ── shape constants ──────────────────────────────────────────────────────────
# hidden_dim=8, dtype=fp16 (2 bytes) → block_size_bytes = 16, so one row of
# encoder output fits in exactly one block.
_HIDDEN_DIM = 8
_DTYPE = torch.float16
_BLOCK_SIZE_BYTES = _HIDDEN_DIM * _DTYPE.itemsize
_NUM_BLOCKS = 8
_requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(),
reason="exercises real CUDA stream/event coordination in ECCPUWorker",
)
def _make_region() -> ECSharedRegion:
"""Fresh region backed by a real per-test mmap file."""
return ECSharedRegion(
engine_id=str(uuid.uuid4()),
num_blocks=_NUM_BLOCKS,
block_size_bytes=_BLOCK_SIZE_BYTES,
)
def _vllm_config(rank: int = 0) -> Mock:
cfg = Mock(spec=VllmConfig)
cfg.parallel_config = Mock(spec=ParallelConfig)
cfg.parallel_config.rank = rank
cfg.model_config = Mock()
cfg.model_config.dtype = _DTYPE
return cfg
def _meta(
*, saves: dict | None = None, loads: dict | None = None
) -> ECCPUConnectorMetadata:
return ECCPUConnectorMetadata(saves=saves or {}, loads=loads or {})
# ── fixtures ─────────────────────────────────────────────────────────────────
@pytest.fixture
def make_worker():
"""Factory that builds an ``ECCPUWorker`` over a real ``ECSharedRegion``.
Tracks the workers and regions it creates and tears them down at the
end of the test, so each test starts from a fresh /dev/shm file.
"""
workers: list[ECCPUWorker] = []
regions: list[ECSharedRegion] = []
def factory(
*,
rank: int = 0,
tp_rank: int = 0,
pcp_rank: int = 0,
pin_memory_available: bool = False,
) -> ECCPUWorker:
region = _make_region()
regions.append(region)
pcp_group = Mock()
pcp_group.rank_in_group = pcp_rank
with (
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.create_ec_shared_region",
return_value=region,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.is_pin_memory_available",
return_value=pin_memory_available,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_tensor_model_parallel_rank",
return_value=tp_rank,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_pcp_group",
return_value=pcp_group,
),
):
worker = ECCPUWorker(_vllm_config(rank=rank))
workers.append(worker)
return worker
yield factory
for worker in workers:
with contextlib.suppress(Exception):
worker.shutdown()
for region in regions:
with contextlib.suppress(Exception):
region.cleanup()
# ── save_caches ──────────────────────────────────────────────────────────────
@_requires_cuda
@pytest.mark.parametrize(
"n_elements,n_blocks",
[
(3 * _HIDDEN_DIM, 3), # exact fit: 3 full blocks, no remainder
(3 * _HIDDEN_DIM + 4, 4), # partial last block: 4 fp16 trail in slot 3
],
ids=["exact-fit", "partial-last-block"],
)
def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks):
"""``save_caches`` + ``flush_saves`` copies the source GPU tensor's bytes
into the block IDs named by ``meta.saves``, in the order they appear,
via a single batched GPU→CPU copy.
When ``total_bytes`` is not a multiple of ``block_size_bytes`` the last
block is partially written; the unwritten tail must remain whatever
was there before. Blocks not named in ``meta.saves`` must be untouched.
"""
worker = make_worker()
sentinel = 0x5A
worker._region.blocks.fill_(sentinel)
src = torch.arange(n_elements, dtype=_DTYPE, device="cuda")
expected_bytes = src.cpu().reshape(-1).view(torch.uint8)
total_bytes = n_elements * _DTYPE.itemsize
block_ids = [7, 2, 5, 0][:n_blocks]
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
worker.flush_saves()
for slot, block_idx in enumerate(block_ids):
block_byte_start = slot * _BLOCK_SIZE_BYTES
block_byte_end = min(block_byte_start + _BLOCK_SIZE_BYTES, total_bytes)
n_written = block_byte_end - block_byte_start
actual_bytes = worker._region.blocks[block_idx, :n_written].view(torch.uint8)
assert torch.equal(
actual_bytes, expected_bytes[block_byte_start:block_byte_end]
), f"block {block_idx} (slot {slot}) bytes mismatch"
if n_written < _BLOCK_SIZE_BYTES:
tail = worker._region.blocks[block_idx, n_written:]
assert torch.all(tail == sentinel), (
f"block {block_idx} tail was overwritten"
)
for idx in set(range(_NUM_BLOCKS)) - set(block_ids):
assert torch.all(worker._region.blocks[idx] == sentinel), (
f"block {idx} (unassigned) was overwritten"
)
def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
"""When the scheduler hasn't pre-allocated blocks for ``mm_hash``,
``save_caches`` + ``flush_saves`` is a pure no-op."""
worker = make_worker()
sentinel = 0x42
worker._region.blocks.fill_(sentinel)
worker.save_caches({}, "h", _meta(saves={}))
worker.flush_saves()
assert torch.all(worker._region.blocks == sentinel)
@pytest.mark.parametrize(
"tp_rank,pcp_rank",
[(1, 0), (0, 1), (1, 1)],
ids=["tp1-pcp0", "tp0-pcp1", "tp1-pcp1"],
)
def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
"""Only TP rank 0 + PCP rank 0 writes to mmap. All other combinations
must skip the save entirely."""
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
sentinel = 0x42
worker._region.blocks.fill_(sentinel)
worker.save_caches(
{"h": torch.zeros(8, dtype=_DTYPE)}, "h", _meta(saves={"h": [0]})
)
worker.flush_saves()
assert torch.all(worker._region.blocks == sentinel)
def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
"""``save_caches`` must raise ``AssertionError`` when the encoder output
is larger than the allocated block space."""
worker = make_worker()
src = torch.zeros(3 * _HIDDEN_DIM, dtype=_DTYPE)
with pytest.raises(AssertionError, match="exceeds allocated blocks"):
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]}))
@_requires_cuda
def test_save_caches_batches_multiple_hashes(make_worker):
"""Multiple save_caches calls are batched into a single flush."""
worker = make_worker()
sentinel = 0x5A
worker._region.blocks.fill_(sentinel)
src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device="cuda")
cache = {"a": src_a, "b": src_b}
worker.save_caches(cache, "a", _meta(saves={"a": [1], "b": [3]}))
worker.save_caches(cache, "b", _meta(saves={"a": [1], "b": [3]}))
worker.flush_saves()
expected_a = src_a.cpu().view(torch.uint8)
expected_b = src_b.cpu().view(torch.uint8)
actual_a = worker._region.blocks[1].view(torch.uint8)
actual_b = worker._region.blocks[3].view(torch.uint8)
assert torch.equal(actual_a, expected_a)
assert torch.equal(actual_b, expected_b)
# Unassigned blocks untouched.
for idx in (0, 2, 4, 5, 6, 7):
assert torch.all(worker._region.blocks[idx] == sentinel)
# ── start_load_caches ────────────────────────────────────────────────────────
@_requires_cuda
def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker):
"""Single batched load across all hashes with correct byte→dtype→shape."""
worker = make_worker()
n_blocks = 3
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
n_blocks, _HIDDEN_DIM
)
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
block_ids = [3, 1, 6]
for i, idx in enumerate(block_ids):
worker._region.blocks[idx].copy_(src_int8[i])
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
out = encoder_cache["h"]
assert out.is_cuda, "consumer worker must place the tensor on the GPU"
assert out.shape == (n_blocks, _HIDDEN_DIM)
assert out.dtype == _DTYPE
assert torch.equal(out.cpu(), src_orig)
@_requires_cuda
def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker):
"""If ``encoder_cache`` already holds the ``mm_hash``, the worker must
not overwrite it."""
worker = make_worker()
worker._region.blocks[0].fill_(0x42)
sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device="cuda")
encoder_cache = {"h": sentinel}
worker.start_load_caches(encoder_cache, _meta(loads={"h": [0]}))
assert encoder_cache["h"] is sentinel, (
"existing encoder_cache entry must not be replaced"
)
@_requires_cuda
def test_start_load_caches_noop_when_loads_is_empty(make_worker):
"""When ``meta.loads`` is empty the early-return must fire."""
worker = make_worker()
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={}))
assert encoder_cache == {}
@_requires_cuda
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
"""Cached entries are preserved while new ones are loaded."""
worker = make_worker()
n_blocks = 2
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
n_blocks, _HIDDEN_DIM
)
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
new_block_ids = [4, 2]
for i, idx in enumerate(new_block_ids):
worker._region.blocks[idx].copy_(src_int8[i])
cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device="cuda")
encoder_cache: dict[str, torch.Tensor] = {"cached_h": cached_tensor}
worker.start_load_caches(
encoder_cache,
_meta(loads={"cached_h": [0], "new_h": new_block_ids}),
)
assert encoder_cache["cached_h"] is cached_tensor
assert "new_h" in encoder_cache
out = encoder_cache["new_h"]
assert out.shape == (n_blocks, _HIDDEN_DIM)
assert out.dtype == _DTYPE
assert torch.equal(out.cpu(), src_orig)
@_requires_cuda
@pytest.mark.parametrize(
"tp_rank,pcp_rank",
[(0, 0), (1, 0), (0, 1), (1, 1)],
ids=["tp0-pcp0", "tp1-pcp0", "tp0-pcp1", "tp1-pcp1"],
)
def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank):
"""All TP/PCP ranks must load from mmap — loads are NOT gated like saves."""
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
n_blocks = 2
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
n_blocks, _HIDDEN_DIM
)
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
block_ids = [1, 3]
for i, idx in enumerate(block_ids):
worker._region.blocks[idx].copy_(src_int8[i])
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
out = encoder_cache["h"]
assert out.is_cuda
assert torch.equal(out.cpu(), src_orig)
# ── round-trip ───────────────────────────────────────────────────────────────
@_requires_cuda
def test_save_then_load_round_trips_bytes(make_worker):
"""Full producer→mmap→consumer byte path in one shot."""
worker = make_worker()
n_blocks = 3
block_ids = [5, 1, 6]
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
n_blocks, _HIDDEN_DIM
)
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
worker.flush_saves()
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
out = encoder_cache["h"]
assert out.shape == src.shape
assert out.dtype == src.dtype
assert torch.equal(out.cpu(), src.cpu())
# ── buffer recycling ────────────────────────────────────────────────────────
@_requires_cuda
def test_buffer_pool_is_reused_across_save_steps(make_worker):
"""After flush_saves, descriptor buffers are returned to the pool and
reused on the next flush — no reallocation."""
worker = make_worker()
src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0]}))
worker.flush_saves()
assert len(worker._buf_pool._pool) == 1
buf_id = id(worker._buf_pool._pool[0].src_ptrs)
# Second step reuses the same buffer.
worker.save_caches({"h": src}, "h", _meta(saves={"h": [1]}))
worker.flush_saves()
assert len(worker._buf_pool._pool) == 1
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
@_requires_cuda
def test_buffer_pool_is_reused_across_load_steps(make_worker):
"""After start_load_caches, descriptor buffers are returned to the pool
and reused on the next call."""
worker = make_worker()
worker._region.blocks[0].fill_(0x01)
worker._region.blocks[1].fill_(0x02)
encoder_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache, _meta(loads={"a": [0]}))
assert len(worker._buf_pool._pool) == 1
buf_id = id(worker._buf_pool._pool[0].src_ptrs)
encoder_cache2: dict[str, torch.Tensor] = {}
worker.start_load_caches(encoder_cache2, _meta(loads={"b": [1]}))
assert len(worker._buf_pool._pool) == 1
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
# ── stream management ────────────────────────────────────────────────────────
@_requires_cuda
def test_stream_initialized_at_construction(make_worker):
"""``_load_stream`` must be a fully initialized CUDA stream."""
worker = make_worker()
assert isinstance(worker._load_stream, torch.cuda.Stream)
# ── lifecycle ────────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"pin_available,expected_pinned",
[
(True, True),
(False, False),
],
ids=["available", "unavailable"],
)
def test_init_pins_memory_when_available(pin_available, expected_pinned):
"""Every worker must call ``pin_memory()`` when the platform allows it."""
region = _make_region()
region.pin_memory = MagicMock() # type: ignore[method-assign]
pcp_group = Mock()
pcp_group.rank_in_group = 0
with (
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.create_ec_shared_region",
return_value=region,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.is_pin_memory_available",
return_value=pin_available,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_tensor_model_parallel_rank",
return_value=0,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_pcp_group",
return_value=pcp_group,
),
):
ECCPUWorker(_vllm_config(rank=0))
try:
if expected_pinned:
region.pin_memory.assert_called_once()
else:
region.pin_memory.assert_not_called()
finally:
region.cleanup()
def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
"""``shutdown`` must always call ``region.cleanup`` — and must never
raise."""
worker = object.__new__(ECCPUWorker)
mock_region = Mock(spec=ECSharedRegion)
worker._region = mock_region
worker._load_stream = MagicMock()
worker._save_bufs = None
worker._save_count = 0
worker.shutdown()
worker._load_stream.synchronize.assert_called_once()
mock_region.cleanup.assert_called_once()
mock_region.cleanup.side_effect = RuntimeError("boom")
with caplog_vllm.at_level(logging.DEBUG, logger="vllm"):
worker.shutdown() # exception must be swallowed
assert mock_region.cleanup.call_count == 2
assert any(
"worker region cleanup failed" in r.message
for r in caplog_vllm.records
if r.levelno == logging.DEBUG
)
# ── e2e: scheduler + worker pipeline ────────────────────────────────────────
@_requires_cuda
def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
"""Full pipeline: scheduler allocates blocks, worker saves GPU tensor to
mmap via flush_saves, scheduler marks ready after step delay, worker
loads from mmap back to GPU, and the result matches the original.
Exercises the real scheduler + worker cooperation through a shared
ECSharedRegion, with real CUDA transfers and stream coordination.
"""
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
ECCPUScheduler,
)
# Build worker first (gets us a real region).
worker = make_worker()
region = worker._region
# Build scheduler sharing the same region.
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
class _EC:
is_ec_producer = True
is_ec_consumer = True
class _Cfg:
ec_transfer_config = _EC()
max_concurrent_batches = 1
scheduler = ECCPUScheduler(_Cfg())
# -- Step 1: scheduler allocates, worker saves --
n_blocks = 3
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
n_blocks, _HIDDEN_DIM
)
class _Pos:
offset = 0
length = n_blocks
class _Feature:
identifier = "img_001"
mm_position = _Pos()
class _Request:
request_id = "req_e2e"
mm_features = [_Feature()]
scheduler.update_state_after_alloc(_Request(), 0)
meta_save = scheduler.build_connector_meta(scheduler_output=None)
assert "img_001" in meta_save.saves
encoder_cache = {"img_001": src}
worker.save_caches(encoder_cache, "img_001", meta_save)
worker.flush_saves()
# -- Step 2: readiness delay (max_concurrent_batches=1) --
meta_step2 = scheduler.build_connector_meta(scheduler_output=None)
assert scheduler.has_cache_item("img_001") is True
assert meta_step2.loads == {} # no load requested yet
# -- Step 3: scheduler emits load, worker loads --
scheduler.update_state_after_alloc(_Request(), 0)
meta_load = scheduler.build_connector_meta(scheduler_output=None)
assert "img_001" in meta_load.loads
assert meta_load.loads["img_001"] == meta_save.saves["img_001"]
load_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(load_cache, meta_load)
out = load_cache["img_001"]
assert out.is_cuda
assert out.shape == src.shape
assert out.dtype == src.dtype
assert torch.equal(out.cpu(), src.cpu())
scheduler.shutdown()
@@ -0,0 +1,158 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E test for the ECCPUConnector.
Verifies:
- Accuracy: outputs from EC CPU cache match fresh encoder computation.
- Latency: loading from EC CPU cache is faster than a cold encoder run.
Requires a CUDA GPU and the Qwen2-VL-2B-Instruct model.
"""
import time
import pytest
from tqdm import tqdm
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.config import ECTransferConfig
from vllm.platforms import current_platform
MODEL = "Qwen/Qwen2-VL-2B-Instruct"
EC_CPU_BYTES = 500 << 20 # 500 MiB
def _build_image_prompt(image):
"""Build a Qwen2-VL prompt with a single image."""
return {
"prompt": (
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
"<|im_start|>user\n"
"<|vision_start|><|image_pad|><|vision_end|>"
"Describe this image briefly.<|im_end|>\n"
"<|im_start|>assistant\n"
),
"multi_modal_data": {"image": image},
}
def _make_unique_image(base_image, iteration: int):
"""Create a unique image by modifying a pixel to change the mm_hash."""
img = base_image.copy()
img.putpixel((0, 0), (iteration % 256, iteration // 256 % 256, 0))
return img
def _wait_for_ec_ready(llm: LLM) -> None:
"""Send a dummy request to force a scheduler step.
After generate() returns, the EC entry's data is in the mmap but
readiness may not have fired yet (first-finish fires in the NEXT
scheduler step). This forces that step.
"""
llm.generate("hi", SamplingParams(max_tokens=1), use_tqdm=False)
def _latency_test(llm: LLM) -> None:
"""Verify EC CPU cache hit is faster than cold encoder computation."""
if not current_platform.is_cuda():
pytest.skip("Latency test requires CUDA")
sampling_params = SamplingParams(max_tokens=1, temperature=0)
base_image = ImageAsset("stop_sign").pil_image
num_times_ec_better_than_cold = 0
num_tests = 10
total_cold_time = 0.0
total_ec_hit_time = 0.0
for i in tqdm(range(num_tests), desc="EC latency test"):
# Use a unique image each iteration so the cold run is truly cold
# (EC has never seen this mm_hash before).
image = _make_unique_image(base_image, i)
prompt = _build_image_prompt(image)
# Cold run: encoder computes from scratch, saves to EC CPU.
llm.llm_engine.reset_encoder_cache()
start_time = time.time()
llm.generate([prompt], sampling_params, use_tqdm=False)
cold_time = time.time() - start_time
total_cold_time += cold_time
# Ensure EC entry is marked ready.
_wait_for_ec_ready(llm)
# Clear GPU encoder cache but keep EC CPU cache intact.
llm.llm_engine.reset_encoder_cache()
# EC hit: scheduler loads from CPU mmap instead of re-encoding.
start_time = time.time()
llm.generate([prompt], sampling_params, use_tqdm=False)
ec_hit_time = time.time() - start_time
total_ec_hit_time += ec_hit_time
if ec_hit_time < cold_time:
num_times_ec_better_than_cold += 1
print("Average times:")
print(f" Cold: {total_cold_time * 1000 / num_tests:.2f}ms")
print(f" EC hit: {total_ec_hit_time * 1000 / num_tests:.2f}ms")
assert num_times_ec_better_than_cold >= 0.8 * num_tests, (
f"EC hit was faster only {num_times_ec_better_than_cold}/{num_tests} "
f"times (expected >= 80%)"
)
def _accuracy_test(llm: LLM) -> None:
"""Verify EC CPU cache produces correct outputs."""
sampling_params = SamplingParams(max_tokens=8, temperature=0)
image = ImageAsset("stop_sign").pil_image
prompt = _build_image_prompt(image)
# Seed: generate with fresh encoder to establish baseline output.
llm.llm_engine.reset_encoder_cache()
baseline = llm.generate([prompt], sampling_params, use_tqdm=False)
baseline_text = baseline[0].outputs[0].text
# Ensure EC entry is marked ready.
_wait_for_ec_ready(llm)
# Clear GPU encoder cache so subsequent runs go through EC CPU path.
llm.llm_engine.reset_encoder_cache()
# Generate multiple times from EC cache and check correctness.
test_count = 20
results = llm.generate([prompt] * test_count, sampling_params, use_tqdm=False)
success_count = sum(1 for r in results if r.outputs[0].text == baseline_text)
assert success_count >= 0.5 * test_count, (
f"Only {success_count}/{test_count} outputs matched baseline "
f"(expected >= 50%). Baseline: {baseline_text!r}"
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_ec_cpu_offloading() -> None:
"""Tests ECCPUConnector accuracy and latency with a VLM model."""
ec_transfer_config = ECTransferConfig(
ec_connector="ECCPUConnector",
ec_role="ec_both",
ec_connector_extra_config={"ec_cpu_bytes": EC_CPU_BYTES},
)
llm = LLM(
model=MODEL,
max_model_len=2048,
gpu_memory_utilization=0.5,
enforce_eager=True,
ec_transfer_config=ec_transfer_config,
)
try:
_latency_test(llm)
_accuracy_test(llm)
finally:
del llm
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECCPUConnectorMetadata."""
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
)
def test_metadata_is_ec_connector_metadata():
"""ECCPUConnectorMetadata must be a subclass of ECConnectorMetadata."""
meta = ECCPUConnectorMetadata()
assert isinstance(meta, ECConnectorMetadata)
def test_metadata_initialization():
"""ECCPUConnectorMetadata initializes with empty saves and loads dicts."""
meta = ECCPUConnectorMetadata()
assert hasattr(meta, "saves")
assert hasattr(meta, "loads")
assert meta.saves == {}
assert meta.loads == {}
def test_metadata_saves_dict_operations():
"""The saves dict supports standard dict operations."""
meta = ECCPUConnectorMetadata()
# Add multiple entries
meta.saves["mm_hash_1"] = [0, 1, 2]
meta.saves["mm_hash_2"] = [10, 11]
assert "mm_hash_1" in meta.saves
assert "mm_hash_2" in meta.saves
assert len(meta.saves) == 2
assert meta.saves["mm_hash_1"] == [0, 1, 2]
assert meta.saves["mm_hash_2"] == [10, 11]
def test_metadata_loads_dict_operations():
"""The loads dict supports standard dict operations."""
meta = ECCPUConnectorMetadata()
# Add multiple entries
meta.loads["mm_hash_1"] = [5, 6, 7]
meta.loads["mm_hash_2"] = [100]
assert "mm_hash_1" in meta.loads
assert "mm_hash_2" in meta.loads
assert len(meta.loads) == 2
assert meta.loads["mm_hash_1"] == [5, 6, 7]
assert meta.loads["mm_hash_2"] == [100]
def test_metadata_saves_and_loads_are_independent():
"""saves and loads dicts are independent."""
meta = ECCPUConnectorMetadata()
meta.saves["key"] = [1, 2]
meta.loads["key"] = [3, 4]
# Same key but different dicts should not interfere
assert meta.saves["key"] == [1, 2]
assert meta.loads["key"] == [3, 4]
# Modifying one should not affect the other
meta.saves["other"] = [99]
assert "other" not in meta.loads
def test_metadata_block_indices_are_integers():
"""Block indices in saves/loads lists should be integers."""
meta = ECCPUConnectorMetadata()
meta.saves["hash"] = [0, 100, 999, 65535]
meta.loads["hash"] = [1, 2, 3]
for idx in meta.saves["hash"]:
assert isinstance(idx, int)
for idx in meta.loads["hash"]:
assert isinstance(idx, int)
def test_metadata_multiple_instances_are_independent():
"""Different ECCPUConnectorMetadata instances should not share state."""
meta1 = ECCPUConnectorMetadata()
meta2 = ECCPUConnectorMetadata()
meta1.saves["x"] = [1]
meta2.saves["y"] = [2]
assert "x" in meta1.saves
assert "x" not in meta2.saves
assert "y" not in meta1.saves
assert "y" in meta2.saves
@@ -0,0 +1,89 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared types for the ECCPUConnector scheduler and worker delegates."""
from dataclasses import dataclass, field
from typing import TYPE_CHECKING
import torch
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
@dataclass
class ECCPUConnectorMetadata(ECConnectorMetadata):
"""Per-step scheduler → worker payload for the ECCPUConnector.
Populated by `ECCPUScheduler.build_connector_meta`; consumed by
`ECCPUWorker` via the mixin's `bind_connector_metadata`.
"""
# Producer role: mm_hashes the scheduler has just allocated CPU
# blocks for this step; the worker's save_caches copies
# encoder_cache[mm_hash] → mmap at these block IDs.
saves: dict[str, list[int]] = field(default_factory=dict)
# Consumer role: mm_hashes whose bytes are available in the local mmap;
# the worker's start_load_caches copies mmap[block_ids] → GPU
# encoder_cache.
loads: dict[str, list[int]] = field(default_factory=dict)
def _get_encoder_cache_hidden_dim(vllm_config: "VllmConfig") -> int:
"""Return the per-token hidden dimension for encoder cache entries.
For most models this equals the LLM's hidden size. Qwen3-VL (and any
future model with deepstack visual encoding) is an exception: the ViT
concatenates its own output with features from N decoder layers before
storing in encoder_cache, producing a tensor of width
``out_hidden_size * (1 + N)`` per visual token. Using the plain LLM
hidden size would under-allocate EC blocks and silently truncate the
transferred data, leading to a shape mismatch on the consumer.
"""
model_config = vllm_config.model_config
hf_config = getattr(model_config, "hf_config", None)
vision_config = (
getattr(hf_config, "vision_config", None) if hf_config is not None else None
)
if vision_config is not None:
out_hidden_size = getattr(vision_config, "out_hidden_size", None)
deepstack_indexes = getattr(vision_config, "deepstack_visual_indexes", None)
if out_hidden_size is not None and deepstack_indexes:
return out_hidden_size * (1 + len(deepstack_indexes))
return model_config.get_inputs_embeds_size()
def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
"""Build the EC mmap region from `vllm_config`.
Both `ECCPUScheduler` and `ECCPUWorker` call this to get the same
shared region (same engine_id, same block_size_bytes).
"""
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None, "ec_transfer_config required to build region"
dp_rank = vllm_config.parallel_config.data_parallel_rank
engine_id = f"{vllm_config.instance_id}_dp{dp_rank}"
dtype = vllm_config.model_config.dtype
hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
element_size = torch.empty(0, dtype=dtype).element_size()
block_size_bytes = hidden_dim * element_size
cpu_bytes = ec_config.ec_connector_extra_config.get("ec_cpu_bytes")
if not cpu_bytes:
raise ValueError("ec_cpu_bytes must be specified in ec_connector_extra_config")
cpu_bytes = int(cpu_bytes)
num_blocks = cpu_bytes // block_size_bytes
return ECSharedRegion(
engine_id=engine_id,
num_blocks=num_blocks,
block_size_bytes=block_size_bytes,
)
@@ -0,0 +1,121 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ECCPUConnector — CPU encoder-cache offloading.
A role-routed shell: one instance per process. The scheduler delegate owns the
mmap region and offload bookkeeping; the worker delegate owns the mmap view and
GPU copy plumbing. An ec_both instance reuses encoder outputs it has already
offloaded to CPU instead of recomputing them.
"""
from typing import TYPE_CHECKING
import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorBase,
ECConnectorRole,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
class ECCPUConnector(ECConnectorBase):
"""EC connector that offloads encoder cache to a shared CPU mmap region."""
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None:
super().__init__(vllm_config=vllm_config, role=role)
self.connector_worker = None
self.connector_scheduler = None
if role == ECConnectorRole.WORKER:
self.connector_worker = self._make_worker(vllm_config)
elif role == ECConnectorRole.SCHEDULER:
self.connector_scheduler = self._make_scheduler(vllm_config)
else:
raise ValueError(f"Unknown ECConnectorRole: {role}")
# Construction seams.
def _make_worker(self, vllm_config: "VllmConfig"):
# Deferred import: the worker module touches torch/CUDA at import time
# via the region, so keep that cost off the scheduler path.
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import (
ECCPUWorker,
)
return ECCPUWorker(vllm_config)
def _make_scheduler(self, vllm_config: "VllmConfig"):
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
ECCPUScheduler,
)
return ECCPUScheduler(vllm_config)
# Worker-side forwarders.
def start_load_caches(
self, encoder_cache: dict[str, torch.Tensor], **kwargs
) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, ECCPUConnectorMetadata)
self.connector_worker.start_load_caches(
encoder_cache, connector_metadata=metadata
)
def save_caches(
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, ECCPUConnectorMetadata)
self.connector_worker.save_caches(
encoder_cache, mm_hash, connector_metadata=metadata
)
# Scheduler-side forwarders.
def has_cache_item(self, identifier: str) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_cache_item(identifier)
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.ensure_cache_available(
request, num_computed_tokens
)
def update_state_after_alloc(self, request: "Request", index: int) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_state_after_alloc(request, index)
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> ECCPUConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
if self.connector_worker is not None:
self.connector_worker.flush_saves()
return None, None
# Shared.
def shutdown(self) -> None:
if self.connector_scheduler is not None:
self.connector_scheduler.shutdown()
if self.connector_worker is not None:
self.connector_worker.shutdown()
@@ -0,0 +1,159 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Lightweight mmap-backed shared memory region for encoder cache (EC) data.
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
"""
import mmap
import os
import time
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0):
"""Spin-wait until the file reaches expected_size (creator truncated it)."""
deadline = time.monotonic() + timeout
while True:
if os.fstat(fd).st_size >= expected_size:
return
if time.monotonic() > deadline:
raise TimeoutError(
f"Timed out waiting for EC mmap file to reach {expected_size} bytes"
)
time.sleep(0.005)
class ECSharedRegion:
"""Flat mmap-backed memory region shared across TP workers for
encoder cache blocks.
Layout: (num_blocks, block_size_bytes) — contiguous, no per-worker
interleaving. All workers map the same file and see identical data.
File path: /dev/shm/vllm_ec_{engine_id}.mmap
This class owns only the shared memory substrate (mmap lifecycle, the
`blocks` view, CUDA host registration). Block allocation and eviction
are tracked by `EmbeddingCache` in the scheduler process.
"""
def __init__(
self,
engine_id: str,
num_blocks: int,
block_size_bytes: int,
) -> None:
self.num_blocks = num_blocks
self.block_size_bytes = block_size_bytes
total_size_bytes = num_blocks * block_size_bytes
# Path in /dev/shm (tmpfs); unique per engine instance.
self._mmap_path = f"/dev/shm/vllm_ec_{engine_id}.mmap"
# True for the process that created the file (responsible for unlink).
self._is_creator = False
# True after successful cudaHostRegister (cleanup must unregister).
self._is_pinned = False
# File descriptor for the shared memory backing file.
try:
self._fd: int | None = os.open(
self._mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
)
os.ftruncate(self._fd, total_size_bytes)
self._is_creator = True
logger.info(
"Created EC mmap file %s (%.2f MiB)",
self._mmap_path,
total_size_bytes / (1 << 20),
)
except FileExistsError:
self._fd = os.open(self._mmap_path, os.O_RDWR)
try:
_wait_for_file_size(self._fd, total_size_bytes)
except Exception:
os.close(self._fd)
self._fd = None
raise
logger.info("Opened existing EC mmap file %s", self._mmap_path)
# MAP_SHARED mmap over _fd; all processes see the same pages.
self._mmap_obj: mmap.mmap | None = mmap.mmap(
self._fd,
total_size_bytes,
flags=mmap.MAP_SHARED,
prot=mmap.PROT_READ | mmap.PROT_WRITE,
)
if self._is_creator:
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
self._mmap_obj.madvise(_MADV_POPULATE_WRITE, 0, total_size_bytes)
# (num_blocks, block_size_bytes) int8 tensor over the mmap buffer.
self.blocks: torch.Tensor = torch.frombuffer(
memoryview(self._mmap_obj), dtype=torch.int8
).view(num_blocks, block_size_bytes)
# Cached for cudaHostRegister/Unregister and pointer math.
self._blocks_ptr: int = self.blocks.data_ptr()
self._blocks_nbytes: int = self.blocks.nbytes
def pin_memory(self) -> None:
"""Register the entire mmap as CUDA pinned memory for fast DMA.
Each TP worker process owns its own virtual address for the same
shared physical pages, so every process must register independently.
No-op when CUDA is not available or already pinned.
"""
if self._is_pinned or not torch.cuda.is_available():
return
result = torch.cuda.cudart().cudaHostRegister(
self._blocks_ptr, self._blocks_nbytes, 0
)
if result.value != 0:
logger.warning(
"cudaHostRegister failed (code=%d) — "
"transfers will still work but may be slower (unpinned DMA)",
result.value,
)
else:
logger.debug("cudaHostRegister %.2f MB", self._blocks_nbytes / 1e6)
self._is_pinned = True
def cleanup(self) -> None:
"""Tear down the region. Lifecycle method; no concurrent access."""
logger.info("Starting ECSharedRegion cleanup...")
if self._is_creator:
try:
os.unlink(self._mmap_path)
logger.info("Removed EC mmap file %s", self._mmap_path)
except Exception:
logger.warning(
"Failed to unlink path %s", self._mmap_path, exc_info=True
)
self._is_creator = False
if self._is_pinned:
result = torch.cuda.cudart().cudaHostUnregister(self._blocks_ptr)
if result.value != 0:
logger.warning("cudaHostUnregister failed (code=%d)", result)
self._is_pinned = False
if hasattr(self, "blocks"):
del self.blocks
if self._mmap_obj:
try:
self._mmap_obj.close()
except Exception:
logger.warning("Failed to close mmap_obj", exc_info=True)
self._mmap_obj = None
if self._fd is not None:
try:
os.close(self._fd)
except Exception:
logger.warning("Failed to close fd %s", self._fd, exc_info=True)
self._fd = None
@@ -0,0 +1,122 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ECCPUScheduler — CPU offload scheduler delegate.
Owns the mmap region and the embedding cache, and handles the producer
(GPU->CPU offload) and consumer (CPU->GPU reload) scheduler-side logic
for the ECCPUConnector.
"""
from typing import TYPE_CHECKING
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
StepTracker,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
class ECCPUScheduler:
"""Scheduler delegate for the ECCPUConnector."""
def __init__(self, vllm_config: "VllmConfig") -> None:
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
self._is_producer: bool = ec_config.is_ec_producer
self._is_consumer: bool = ec_config.is_ec_consumer
self._region = create_ec_shared_region(vllm_config)
# Block allocator + LRU eviction policy for the shared region.
self._cache = EmbeddingCache(self._region.num_blocks)
max_batches = vllm_config.max_concurrent_batches
# Delays mark_ready until the GPU→mmap DMA is guaranteed complete.
self._ready_tracker = StepTracker(max_batches)
# Delays unpin until the mmap→GPU DMA is guaranteed complete.
self._unpin_tracker = StepTracker(max_batches)
# mm_hash → block IDs allocated this step for GPU→mmap saves.
self._pending_saves: dict[str, list[int]] = {}
# mm_hash → block IDs to load from mmap→GPU this step.
self._pending_loads: dict[str, list[int]] = {}
def has_cache_item(self, identifier: str) -> bool:
if not self._is_consumer:
return False
entry = self._cache.get(identifier)
return entry is not None and entry.ready
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
return True
def update_state_after_alloc(self, request: "Request", index: int) -> None:
feature = request.mm_features[index]
mm_hash = feature.identifier
if self._is_producer and self._cache.get(mm_hash) is None:
entry = self._cache.alloc(mm_hash, feature.mm_position.length)
if entry is not None:
self._pending_saves[mm_hash] = list(entry.block_ids)
self._ready_tracker.add(mm_hash, request.request_id)
if self._is_consumer and mm_hash not in self._pending_loads:
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
self._cache.pin(mm_hash)
self._pending_loads[mm_hash] = list(entry.block_ids)
self._unpin_tracker.add(mm_hash, request.request_id)
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> ECCPUConnectorMetadata:
finished = (
scheduler_output.finished_req_ids if scheduler_output is not None else set()
)
for key in self._ready_tracker.step(finished):
entry = self._cache.get(key)
if entry is not None and not entry.ready:
self._cache.mark_ready(key)
for key in self._unpin_tracker.step(finished):
self._cache.unpin(key)
meta = ECCPUConnectorMetadata()
if self._is_producer:
meta.saves = self._pending_saves
self._pending_saves = {}
if self._is_consumer:
meta.loads = self._pending_loads
self._pending_loads = {}
return meta
def shutdown(self) -> None:
# drain_all() covers both entries still in _current (never
# consumed by build_connector_meta) and entries in slots.
self._pending_loads.clear()
for mm_hash in self._unpin_tracker.drain_all():
self._cache.unpin(mm_hash)
self._ready_tracker.drain_all()
self._is_producer = False
self._is_consumer = False
try:
self._region.cleanup()
except Exception:
logger.debug("ec: region cleanup failed", exc_info=True)
@@ -0,0 +1,131 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""EmbeddingCache — named-entry block cache with FIFO eviction.
Manages a fixed pool of block IDs keyed by content identity (mm_hash).
Entries transition through: not-ready → ready (evictable) → pinned.
Eviction targets ready + unpinned entries in FIFO order.
All public methods are thread-safe.
"""
import threading
from collections import OrderedDict
class CacheEntry:
"""A single cache entry. Read `.block_ids` and `.ready` freely;
mutations only through EmbeddingCache methods."""
__slots__ = ("block_ids", "_pin_count")
def __init__(self, block_ids: tuple[int, ...]) -> None:
self.block_ids = block_ids
self._pin_count = -1 # not ready
@property
def ready(self) -> bool:
return self._pin_count >= 0
@property
def evictable(self) -> bool:
return self._pin_count == 0
def mark_ready(self):
self._pin_count = 0
def pin(self):
self._pin_count += 1
class EmbeddingCache:
"""Fixed-size block cache with FIFO eviction.
Entries are keyed by content identity (e.g. mm_hash). Blocks are
allocated from a free-set; when space is needed, ready + unpinned
entries are evicted oldest-first.
The caller is responsible for deciding *when* to call ``mark_ready``
(e.g. after enough engine steps have elapsed for the worker to have
completed the write).
"""
def __init__(self, num_blocks: int) -> None:
self._num_blocks = num_blocks
self._lock = threading.Lock()
# Available block IDs, LIFO stack.
self._free_blocks: list[int] = list(range(num_blocks))
# Key → live entry (all states: not-ready, ready, pinned).
self._entries: dict[str, CacheEntry] = {}
# Ready + unpinned entries in insertion order (FIFO eviction).
self._entries_free_list: OrderedDict[str, None] = OrderedDict()
# Total blocks reclaimable via eviction (fail-fast check).
self._evictable_block_count: int = 0
def get(self, key: str) -> CacheEntry | None:
"""Return the entry for *key*, or None if not present."""
with self._lock:
return self._entries.get(key)
def alloc(self, key: str, n_blocks: int) -> CacheEntry | None:
"""Allocate *n_blocks* for *key*, evicting as needed.
The entry starts not-ready. Returns None if there is not enough
space even after evicting all evictable entries.
"""
with self._lock:
assert key not in self._entries, (
f"EmbeddingCache: duplicate alloc for {key!r}"
)
assert n_blocks <= self._num_blocks, (
f"EmbeddingCache: {n_blocks} blocks requested but capacity "
f"is {self._num_blocks}"
)
if len(self._free_blocks) + self._evictable_block_count < n_blocks:
return None
self._evict_until(n_blocks)
block_ids = tuple(self._free_blocks.pop() for _ in range(n_blocks))
entry = CacheEntry(block_ids)
self._entries[key] = entry
return entry
def mark_ready(self, key: str) -> None:
"""Mark an entry as ready (data is CPU-visible)."""
with self._lock:
entry = self._entries[key]
assert entry._pin_count == -1, (
f"EmbeddingCache: mark_ready on already-ready entry {key!r}"
)
entry.mark_ready()
self._entries_free_list[key] = None
self._evictable_block_count += len(entry.block_ids)
def pin(self, key: str) -> None:
"""Pin an entry (prevent eviction)."""
with self._lock:
entry = self._entries[key]
assert entry.ready, f"EmbeddingCache: pin of not-ready entry {key!r}"
if entry.evictable:
del self._entries_free_list[key]
self._evictable_block_count -= len(entry.block_ids)
entry.pin()
def unpin(self, key: str) -> None:
"""Unpin an entry. Asserts currently pinned."""
with self._lock:
entry = self._entries[key]
assert entry._pin_count > 0, (
f"EmbeddingCache: unpin of unpinned entry {key!r}"
)
entry._pin_count -= 1
if entry._pin_count == 0:
self._entries_free_list[key] = None
self._evictable_block_count += len(entry.block_ids)
def _evict_until(self, n_blocks: int) -> None:
"""Evict ready+unpinned entries FIFO until enough space. Lock held."""
while len(self._free_blocks) < n_blocks:
key, _ = self._entries_free_list.popitem(last=False)
entry = self._entries.pop(key)
self._evictable_block_count -= len(entry.block_ids)
self._free_blocks.extend(entry.block_ids)
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""StepTracker — step-counting delay with first-finish fast-path.
Manages a collection of (mm_hash, request_id) entries that need to be
processed after a configurable number of engine steps have elapsed.
Additionally, if the originating request finishes before the step-count
expires, the entry is processed immediately (the GPU transfer is
guaranteed complete by request finish time).
Used by ECCPUScheduler: one instance for readiness marking, one for
unpins.
"""
from collections import deque
from dataclasses import dataclass
@dataclass(slots=True)
class _PendingEntry:
mm_hash: str
request_id: str
processed: bool = False
class StepTracker:
"""Track pending operations with step-count delay and first-finish.
Each ``add(mm_hash, request_id)`` registers an operation that will be
returned from ``step()`` either when:
(a) ``max_concurrent_batches`` steps elapse (deque expiry), OR
(b) ``request_id`` appears in ``finished_req_ids`` (first-finish).
Each add is processed exactly once — never both (a) and (b).
"""
def __init__(self, max_concurrent_batches: int) -> None:
# One slot per past step; oldest at the right. When the deque is
# full the rightmost slot is popped and its entries expire.
self._slots: deque[list[_PendingEntry]] = deque(maxlen=max_concurrent_batches)
# Reverse index: request_id → entries across all slots + _current.
# Enables O(1) lookup for the first-finish fast-path.
self._req_index: dict[str, list[_PendingEntry]] = {}
# Entries added this step (via add()), committed to a new slot at
# the start of the next step() call.
self._current: list[_PendingEntry] = []
def add(self, mm_hash: str, request_id: str) -> None:
"""Register an mm_hash needing processing after a delay."""
entry = _PendingEntry(mm_hash=mm_hash, request_id=request_id)
self._current.append(entry)
self._req_index.setdefault(request_id, []).append(entry)
def step(self, finished_req_ids: set[str]) -> list[str]:
"""Advance one step. Return mm_hashes ready for processing.
Returns:
List of mm_hashes to process. May contain duplicates if the
same mm_hash was added multiple times (from different requests).
"""
result: list[str] = []
# Phase 1: first-finish fast-path.
for req_id in finished_req_ids:
entries = self._req_index.pop(req_id, None)
if entries is None:
continue
for entry in entries:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
# Phase 2: step-count expiry.
if len(self._slots) == self._slots.maxlen:
expired_slot = self._slots.pop()
for entry in expired_slot:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
self._cleanup_expired(expired_slot)
# Phase 3: commit current entries to new slot.
self._slots.appendleft(self._current)
self._current = []
return result
def drain_all(self) -> list[str]:
"""Return all unprocessed entries and clear internal state."""
result: list[str] = []
for entry in self._current:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
self._current = []
for slot in self._slots:
for entry in slot:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
self._slots.clear()
self._req_index.clear()
return result
def _cleanup_expired(self, slot: list[_PendingEntry]) -> None:
"""Prune reverse index for entries in an expired slot."""
for entry in slot:
entries_for_req = self._req_index.get(entry.request_id)
if entries_for_req is None:
# Already removed by fast-path (pop in step()).
continue
entries_for_req.remove(entry)
if not entries_for_req:
del self._req_index[entry.request_id]
@@ -0,0 +1,202 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worker-side of the ECCPUConnector.
Thin, stateless across steps: opens the shared mmap region and uses the
per-step connector metadata (`ECCPUConnectorMetadata`) to decide which
blocks to copy in each direction.
"""
from typing import TYPE_CHECKING
import torch
from vllm._custom_ops import swap_blocks_batch
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.worker.descriptor_buffers import (
DescriptorBufferPool,
DescriptorBuffers,
)
from vllm.distributed.parallel_state import (
get_pcp_group,
get_tensor_model_parallel_rank,
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.platform_utils import is_pin_memory_available
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
class ECCPUWorker:
"""Worker-side delegate for the ECCPUConnector.
- Producer role: copies `encoder_cache[mm_hash]` → `mmap[block_ids]`
for each entry in `metadata.saves`. Descriptor buffers are filled
directly in `save_caches`; the actual DMA is issued as a single
batched call in `flush_saves`.
- Consumer role: copies `mmap[block_ids]` → `encoder_cache[mm_hash]`
for all entries in `metadata.loads` via a single `swap_blocks_batch`
call on the load stream.
- On `ec_both` nodes both paths run back-to-back in a single step.
"""
def __init__(self, vllm_config: "VllmConfig") -> None:
self._region = create_ec_shared_region(vllm_config)
# Model dtype; used to reinterpret raw int8 blocks on load.
self._dtype = vllm_config.model_config.dtype
if is_pin_memory_available():
self._region.pin_memory()
# All TP/PCP ranks hold identical encoder output. Only one rank
# per mmap needs to write — saves host memory bandwidth.
# DCP is a subdivision of TP, so tp_rank==0 covers it.
self._is_save_rank = (
get_tensor_model_parallel_rank() == 0 and get_pcp_group().rank_in_group == 0
)
# Dedicated stream for async mmap→GPU loads (overlaps with compute).
self._load_stream = current_platform.Stream()
# Descriptor buffer pool (recycled across steps, shared by both paths).
self._buf_pool = DescriptorBufferPool()
# Active save buffer being filled during save_caches calls this step.
self._save_bufs: DescriptorBuffers | None = None
self._save_count: int = 0
def save_caches(
self,
encoder_cache: dict[str, torch.Tensor],
mm_hash: str,
connector_metadata: ECCPUConnectorMetadata,
) -> None:
"""Fill descriptor buffers directly for batched flush."""
if not self._is_save_rank:
return
block_ids = connector_metadata.saves.get(mm_hash)
if block_ids is None:
return
src = encoder_cache[mm_hash]
total_bytes = src.numel() * src.element_size()
block_size = self._region.block_size_bytes
allocated_bytes = len(block_ids) * block_size
assert total_bytes <= allocated_bytes, (
f"EC: encoder output exceeds allocated blocks for mm_hash={mm_hash}: "
f"{total_bytes} bytes but only {allocated_bytes} allocated "
f"({len(block_ids)} blocks × {block_size}). "
f"shape={list(src.shape)}"
)
if self._save_bufs is None:
total = sum(len(v) for v in connector_metadata.saves.values())
self._save_bufs = self._buf_pool.acquire(total)
assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel()
src_ptrs, dst_ptrs, sizes = self._save_bufs
src_base = src.view(-1).view(torch.uint8).data_ptr()
dst_base = self._region.blocks.data_ptr()
idx = self._save_count
for i, block_idx in enumerate(block_ids):
start = i * block_size
src_ptrs[idx] = src_base + start
dst_ptrs[idx] = dst_base + block_idx * block_size
sizes[idx] = min(block_size, total_bytes - start)
idx += 1
self._save_count = idx
def flush_saves(self) -> None:
"""Flush all accumulated saves in a single swap_blocks_batch call."""
if self._save_count == 0:
return
bufs = self._save_bufs
assert bufs is not None
src_ptrs, dst_ptrs, sizes = bufs
n = self._save_count
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
self._buf_pool.release(bufs)
self._save_bufs = None
self._save_count = 0
def start_load_caches(
self,
encoder_cache: dict[str, torch.Tensor],
connector_metadata: ECCPUConnectorMetadata,
) -> None:
"""Consumer path: single batched copy of all loads from mmap→GPU."""
if not connector_metadata.loads:
return
block_size = self._region.block_size_bytes
blocks = self._region.blocks
dtype = self._dtype
device_type = current_platform.device_type
src_base = blocks.data_ptr()
# Pre-filter: only hashes not already in encoder_cache.
load_items = {
h: idxs
for h, idxs in connector_metadata.loads.items()
if h not in encoder_cache
}
if not load_items:
return
total_blocks = sum(len(idxs) for idxs in load_items.values())
with current_platform.stream(self._load_stream):
# Single contiguous destination buffer for all loads.
dst_buf = torch.empty(
total_blocks, block_size, dtype=torch.int8, device=device_type
)
dst_buf_base = dst_buf.data_ptr()
bufs = self._buf_pool.acquire(total_blocks)
src_ptrs = bufs.src_ptrs[:total_blocks]
dst_ptrs = bufs.dst_ptrs[:total_blocks]
sizes = bufs.sizes[:total_blocks]
sizes[:] = block_size
op_idx = 0
for block_ids in load_items.values():
for block_idx in block_ids:
src_ptrs[op_idx] = src_base + block_idx * block_size
dst_ptrs[op_idx] = dst_buf_base + op_idx * block_size
op_idx += 1
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
self._buf_pool.release(bufs)
# Slice contiguous buffer into per-hash views.
offset = 0
for mm_hash, block_ids in load_items.items():
n = len(block_ids)
encoder_cache[mm_hash] = (
dst_buf[offset : offset + n].view(dtype).reshape(n, -1)
)
offset += n
current_platform.current_stream().wait_stream(self._load_stream)
def shutdown(self) -> None:
self._load_stream.synchronize()
self._save_bufs = None
self._save_count = 0
try:
self._region.cleanup()
except Exception:
logger.debug("EC: worker region cleanup failed", exc_info=True)
@@ -0,0 +1,46 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reusable pool of (src_ptrs, dst_ptrs, sizes) int64 tensor triples.
Used by ECCPUWorker to batch swap_blocks_batch descriptors without
per-step allocation overhead.
"""
from typing import NamedTuple
import torch
class DescriptorBuffers(NamedTuple):
src_ptrs: torch.Tensor
dst_ptrs: torch.Tensor
sizes: torch.Tensor
class DescriptorBufferPool:
"""Pool of descriptor buffer triples for swap_blocks_batch.
Each buffer is a `DescriptorBuffers` namedtuple of three 1-D int64
tensors of equal length. Buffers are recycled across steps; if a
returned buffer is too small it is discarded and a fresh one allocated.
"""
def __init__(self) -> None:
# LIFO stack of idle buffer triples.
self._pool: list[DescriptorBuffers] = []
def acquire(self, n: int) -> DescriptorBuffers:
"""Get a buffer triple with capacity >= *n*."""
if self._pool:
bufs = self._pool.pop()
if bufs.src_ptrs.numel() >= n:
return bufs
return DescriptorBuffers(
torch.empty(n, dtype=torch.int64),
torch.empty(n, dtype=torch.int64),
torch.empty(n, dtype=torch.int64),
)
def release(self, bufs: DescriptorBuffers) -> None:
"""Return a buffer triple to the pool for reuse."""
self._pool.append(bufs)
@@ -83,3 +83,9 @@ ECConnectorFactory.register_connector(
"vllm.distributed.ec_transfer.ec_connector.example_connector",
"ECExampleConnector",
)
ECConnectorFactory.register_connector(
"ECCPUConnector",
"vllm.distributed.ec_transfer.ec_connector.cpu.connector",
"ECCPUConnector",
)