diff --git a/tests/v1/ec_connector/unit/cpu/scheduler/test_embedding_cache.py b/tests/v1/ec_connector/unit/cpu/scheduler/test_embedding_cache.py new file mode 100644 index 0000000000..0fe465c3e8 --- /dev/null +++ b/tests/v1/ec_connector/unit/cpu/scheduler/test_embedding_cache.py @@ -0,0 +1,306 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for EmbeddingCache. + +Pure integer bookkeeping — no mmap, torch, or CUDA — so these run anywhere. +""" + +import threading + +import pytest + +from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import ( + EmbeddingCache, +) + + +def _cache(num_blocks: int = 8) -> EmbeddingCache: + return EmbeddingCache(num_blocks) + + +# ── alloc ──────────────────────────────────────────────────────────────────── + + +def test_alloc_returns_unique_block_ids(): + cache = _cache() + entry = cache.alloc("a", 3) + assert entry is not None + assert len(entry.block_ids) == 3 + assert len(set(entry.block_ids)) == 3 + assert all(0 <= i < 8 for i in entry.block_ids) + + +def test_alloc_returns_tuple(): + cache = _cache() + entry = cache.alloc("a", 2) + assert isinstance(entry.block_ids, tuple) + + +def test_alloc_returns_none_when_full(): + cache = _cache(num_blocks=4) + cache.alloc("a", 4) + assert cache.alloc("b", 1) is None + + +def test_alloc_asserts_duplicate_key(): + cache = _cache() + cache.alloc("a", 2) + with pytest.raises(AssertionError, match="duplicate alloc"): + cache.alloc("a", 2) + + +def test_alloc_asserts_n_blocks_exceeds_capacity(): + cache = _cache(num_blocks=4) + with pytest.raises(AssertionError, match="capacity"): + cache.alloc("a", 5) + + +def test_alloc_zero_blocks(): + cache = _cache() + entry = cache.alloc("a", 0) + assert entry is not None + assert entry.block_ids == () + + +def test_alloc_starts_not_ready(): + cache = _cache() + entry = cache.alloc("a", 2) + assert not entry.ready + assert not entry.evictable + + +# ── get ────────────────────────────────────────────────────────────────────── + + +def test_get_missing_returns_none(): + cache = _cache() + assert cache.get("a") is None + + +def test_get_returns_entry(): + cache = _cache() + alloc_entry = cache.alloc("a", 2) + got = cache.get("a") + assert got is alloc_entry + + +def test_get_returns_not_ready_entry(): + cache = _cache() + cache.alloc("a", 2) + entry = cache.get("a") + assert entry is not None + assert not entry.ready + + +# ── mark_ready ─────────────────────────────────────────────────────────────── + + +def test_mark_ready_transitions(): + cache = _cache() + cache.alloc("a", 2) + entry = cache.get("a") + assert not entry.ready + cache.mark_ready("a") + assert entry.ready + assert entry.evictable + + +def test_mark_ready_asserts_already_ready(): + cache = _cache() + cache.alloc("a", 2) + cache.mark_ready("a") + with pytest.raises(AssertionError, match="already-ready"): + cache.mark_ready("a") + + +def test_mark_ready_asserts_missing_key(): + cache = _cache() + with pytest.raises(KeyError): + cache.mark_ready("nonexistent") + + +# ── pin / unpin ────────────────────────────────────────────────────────────── + + +def test_pin_asserts_not_ready(): + cache = _cache() + cache.alloc("a", 2) + with pytest.raises(AssertionError, match="not-ready"): + cache.pin("a") + + +def test_pin_unpin_basic(): + cache = _cache() + cache.alloc("a", 2) + cache.mark_ready("a") + cache.pin("a") + entry = cache.get("a") + assert not entry.evictable + cache.unpin("a") + assert entry.evictable + + +def test_nested_pin(): + cache = _cache() + cache.alloc("a", 2) + cache.mark_ready("a") + cache.pin("a") + cache.pin("a") + cache.unpin("a") + entry = cache.get("a") + assert not entry.evictable # still pinned (count=1) + cache.unpin("a") + assert entry.evictable + + +def test_unpin_asserts_on_unpinned(): + cache = _cache() + cache.alloc("a", 2) + cache.mark_ready("a") + with pytest.raises(AssertionError, match="unpinned"): + cache.unpin("a") + + +# ── eviction ───────────────────────────────────────────────────────────────── + + +def test_eviction_frees_ready_unpinned_fifo(): + cache = _cache(num_blocks=4) + cache.alloc("a", 2) + cache.mark_ready("a") + cache.alloc("b", 2) + cache.mark_ready("b") + # Pool full. Alloc should evict "a" (oldest). + entry_c = cache.alloc("c", 2) + assert entry_c is not None + assert cache.get("a") is None # evicted + assert cache.get("b") is not None # still there + + +def test_eviction_skips_pinned(): + cache = _cache(num_blocks=4) + cache.alloc("a", 2) + cache.mark_ready("a") + cache.pin("a") + cache.alloc("b", 2) + cache.mark_ready("b") + # "a" is pinned, so "b" should be evicted instead. + entry_c = cache.alloc("c", 2) + assert entry_c is not None + assert cache.get("a") is not None # pinned, survived + assert cache.get("b") is None # evicted + cache.unpin("a") + + +def test_eviction_skips_not_ready(): + cache = _cache(num_blocks=4) + cache.alloc("a", 2) # not ready + cache.alloc("b", 2) + cache.mark_ready("b") + # "a" is not ready (non-evictable), "b" is evictable. + entry_c = cache.alloc("c", 2) + assert entry_c is not None + assert cache.get("a") is not None # not ready, survived + assert cache.get("b") is None # evicted + cache.mark_ready("a") # clean up + + +def test_eviction_returns_none_if_all_non_evictable(): + cache = _cache(num_blocks=4) + cache.alloc("a", 2) + cache.mark_ready("a") + cache.pin("a") + cache.alloc("b", 2) # not ready + # Both entries are non-evictable (a=pinned, b=not-ready). + assert cache.alloc("c", 1) is None + # Fail-fast: nothing was evicted. + assert cache.get("a") is not None + assert cache.get("b") is not None + cache.unpin("a") + + +def test_fail_fast_no_wasted_eviction(): + cache = _cache(num_blocks=6) + cache.alloc("a", 2) + cache.mark_ready("a") # evictable: 2 blocks + cache.alloc("b", 2) # not ready + cache.alloc("c", 2) # not ready + # free=0, evictable=2, need 3 -> cannot succeed + assert cache.alloc("d", 3) is None + # "a" was NOT evicted despite being evictable + assert cache.get("a") is not None + + +def test_eviction_multiple_entries(): + cache = _cache(num_blocks=6) + cache.alloc("a", 2) + cache.mark_ready("a") + cache.alloc("b", 2) + cache.mark_ready("b") + cache.alloc("c", 2) + cache.mark_ready("c") + # Need 4 blocks: must evict both "a" and "b". + entry_d = cache.alloc("d", 4) + assert entry_d is not None + assert cache.get("a") is None + assert cache.get("b") is None + assert cache.get("c") is not None + + +# ── thread safety ──────────────────────────────────────────────────────────── + + +def test_concurrent_pin_unpin(): + cache = _cache(num_blocks=8) + cache.alloc("a", 4) + cache.mark_ready("a") + errors: list[Exception] = [] + barrier = threading.Barrier(8) + + def worker(): + barrier.wait() + try: + for _ in range(500): + cache.pin("a") + cache.unpin("a") + except Exception as exc: + errors.append(exc) + + threads = [threading.Thread(target=worker) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, errors + entry = cache.get("a") + assert entry.evictable # all pins balanced + + +def test_concurrent_alloc_evict(): + cache = _cache(num_blocks=4) + errors: list[Exception] = [] + barrier = threading.Barrier(4) + counter = {"i": 0} + counter_lock = threading.Lock() + + def worker(): + barrier.wait() + try: + for _ in range(200): + with counter_lock: + key = f"k{counter['i']}" + counter["i"] += 1 + entry = cache.alloc(key, 1) + if entry is not None: + cache.mark_ready(key) + except Exception as exc: + errors.append(exc) + + threads = [threading.Thread(target=worker) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, errors diff --git a/tests/v1/ec_connector/unit/cpu/scheduler/test_scheduler.py b/tests/v1/ec_connector/unit/cpu/scheduler/test_scheduler.py new file mode 100644 index 0000000000..0e224c1af3 --- /dev/null +++ b/tests/v1/ec_connector/unit/cpu/scheduler/test_scheduler.py @@ -0,0 +1,480 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import uuid + +import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod +from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( + ECSharedRegion, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler + +_N = 16 +_BS = 64 + + +class _Pos: + def __init__(self, offset, length): + self.offset = offset + self.length = length + + +class _Feature: + def __init__(self, mm_hash, length=1, identifier=None): + self.mm_hash = mm_hash + self.identifier = identifier if identifier is not None else mm_hash + self.mm_position = _Pos(0, length) + + +class _Request: + _counter = 0 + + def __init__(self, features, request_id=None): + self.mm_features = features + if request_id is None: + _Request._counter += 1 + self.request_id = f"req_{_Request._counter}" + else: + self.request_id = request_id + + +def _make_scheduler( + monkeypatch, + num_blocks=_N, + max_concurrent_batches=1, + *, + is_producer=True, + is_consumer=True, +) -> ECCPUScheduler: + region = ECSharedRegion( + engine_id=str(uuid.uuid4()), + num_blocks=num_blocks, + block_size_bytes=_BS, + ) + monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region) + + _is_prod = is_producer + _is_cons = is_consumer + _mcb = max_concurrent_batches + + class _EC: + is_ec_producer = _is_prod + is_ec_consumer = _is_cons + + class _Cfg: + ec_transfer_config = _EC() + + @property + def max_concurrent_batches(self): + return _mcb + + return ECCPUScheduler(_Cfg()) + + +class _SchedulerOutput: + def __init__(self, finished_req_ids=None): + self.finished_req_ids = finished_req_ids or set() + + +def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int): + """Pre-populate a ready cache entry backed by real blocks.""" + s._cache.alloc(mm_hash, n_blocks) + s._cache.mark_ready(mm_hash) + + +def test_offload_reuse_cycle(monkeypatch): + s = _make_scheduler(monkeypatch) + req = _Request([_Feature("h1", length=1)]) + + # Step A: first sight — allocate and save. + assert s.has_cache_item("h1") is False + assert s.ensure_cache_available(req, 0) is True + s.update_state_after_alloc(req, 0) + meta_a = s.build_connector_meta(scheduler_output=None) + assert "h1" in meta_a.saves + assert meta_a.loads == {} + # Not ready yet (max_concurrent_batches=1 means ready next step). + assert s.has_cache_item("h1") is False + + # Step B: _step_readiness marks h1 ready (no new allocs). + s.build_connector_meta(scheduler_output=None) + assert s.has_cache_item("h1") is True + + # Step C: reload now works — entry is ready at update_state_after_alloc time. + s.update_state_after_alloc(req, 0) + meta_c = s.build_connector_meta(scheduler_output=None) + assert "h1" in meta_c.loads + assert meta_c.loads["h1"] == meta_a.saves["h1"] + + s.shutdown() + + +def test_has_cache_item_false_when_not_consumer(monkeypatch): + s = _make_scheduler(monkeypatch, is_consumer=False) + assert s.has_cache_item("anything") is False + s.shutdown() + + +def test_connector_keys_on_identifier_not_mm_hash(monkeypatch): + """The connector must key the encoder cache on feature.identifier (what + has_cache_item is called with), NOT feature.mm_hash.""" + s = _make_scheduler(monkeypatch) + req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")]) + s.update_state_after_alloc(req, 0) + meta = s.build_connector_meta(scheduler_output=None) + assert "ENC_KEY" in meta.saves + assert "PROC_KEY" not in meta.saves + # Not ready yet (1 step delay). + assert s.has_cache_item("ENC_KEY") is False + # After another step it becomes ready — keyed on identifier. + s.build_connector_meta(scheduler_output=None) + assert s.has_cache_item("ENC_KEY") is True + assert s.has_cache_item("PROC_KEY") is False + s.shutdown() + + +def test_cpu_region_fifo_eviction(monkeypatch): + # Region holds exactly 2 one-block encodings. + s = _make_scheduler(monkeypatch, num_blocks=2) + + # Fill with A and B, marking each ready. + for h in ("A", "B"): + s.update_state_after_alloc(_Request([_Feature(h, length=1)]), 0) + s.build_connector_meta(scheduler_output=None) + # After the step above, A's step was called in B's build. A is ready. + # B was submitted in the last build, so one more step makes it ready. + s.build_connector_meta(scheduler_output=None) # marks B ready + assert s.has_cache_item("A") is True + assert s.has_cache_item("B") is True + + # Allocate C: must evict A (oldest) to make room. + s.update_state_after_alloc(_Request([_Feature("C", length=1)]), 0) + s.build_connector_meta(scheduler_output=None) + assert s.has_cache_item("A") is False # evicted + assert s.has_cache_item("B") is True + + +def test_multiple_mm_items_per_request(monkeypatch): + s = _make_scheduler(monkeypatch) + req = _Request([_Feature("h1", length=1), _Feature("h2", length=1)]) + s.update_state_after_alloc(req, 0) + s.update_state_after_alloc(req, 1) + meta = s.build_connector_meta(scheduler_output=None) + assert set(meta.saves) == {"h1", "h2"} + + # Step delay: mark both ready. + s.build_connector_meta(scheduler_output=None) + assert s.has_cache_item("h1") is True + assert s.has_cache_item("h2") is True + + # Now reload both. + s.update_state_after_alloc(req, 0) + s.update_state_after_alloc(req, 1) + meta2 = s.build_connector_meta(scheduler_output=None) + assert set(meta2.loads) == {"h1", "h2"} + s.shutdown() + + +def test_load_returns_correct_block_ids(monkeypatch): + """meta.loads must contain the same block IDs that meta.saves + allocated — verified through the public API only.""" + s = _make_scheduler(monkeypatch) + req = _Request([_Feature("a", length=2)]) + + s.update_state_after_alloc(req, 0) + meta_save = s.build_connector_meta(scheduler_output=None) + assert "a" in meta_save.saves + + # Readiness delay. + s.build_connector_meta(scheduler_output=None) + + # Reload. + s.update_state_after_alloc(req, 0) + meta_load = s.build_connector_meta(scheduler_output=None) + assert meta_load.loads["a"] == meta_save.saves["a"] + s.shutdown() + + +def test_loads_only_serves_hashes_touched_this_step(monkeypatch): + s = _make_scheduler(monkeypatch) + _seed_cached(s, "a", n_blocks=2) + _seed_cached(s, "b", n_blocks=2) + + s.update_state_after_alloc(_Request([_Feature("a")]), 0) + meta = s.build_connector_meta(scheduler_output=None) + assert "a" in meta.loads + assert "b" not in meta.loads + s.shutdown() + + +def test_repeated_reload_same_step_loads_once(monkeypatch): + """The same mm_hash requested twice in one step must appear in + meta.loads exactly once.""" + s = _make_scheduler(monkeypatch) + _seed_cached(s, "a", n_blocks=2) + + s.update_state_after_alloc(_Request([_Feature("a")]), 0) + s.update_state_after_alloc(_Request([_Feature("a")]), 0) + meta = s.build_connector_meta(scheduler_output=None) + assert list(meta.loads.keys()).count("a") == 1 + s.shutdown() + + +def test_load_not_emitted_for_uncached_entry(monkeypatch): + """A feature that was never saved must not appear in meta.loads.""" + s = _make_scheduler(monkeypatch) + s.update_state_after_alloc(_Request([_Feature("missing")]), 0) + meta = s.build_connector_meta(scheduler_output=None) + assert meta.loads == {} + s.shutdown() + + +def test_delayed_unpin_protects_blocks_during_worker_read(monkeypatch): + """Blocks being loaded must not be evictable until the delayed unpin + fires. Observable: a save that needs eviction cannot reclaim a + still-pinned loaded entry.""" + # 2 blocks, both occupied by "a" (ready). + s = _make_scheduler(monkeypatch, num_blocks=2) + _seed_cached(s, "a", n_blocks=2) + + # Load "a" — pins it. + s.update_state_after_alloc(_Request([_Feature("a")]), 0) + meta = s.build_connector_meta(scheduler_output=None) + assert "a" in meta.loads + + # Try to save "b" (needs 2 blocks, but "a" is pinned) — must fail. + s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0) + meta2 = s.build_connector_meta(scheduler_output=None) + assert "b" not in meta2.saves # can't evict pinned "a" + + # After the delay, "a" unpins. Now "b" can evict it. + s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0) + meta3 = s.build_connector_meta(scheduler_output=None) + assert "b" in meta3.saves + s.shutdown() + + +def test_delayed_unpin_depth_2(monkeypatch): + """With max_concurrent_batches=2, loaded blocks stay pinned for 2 steps + after build. Observable via eviction failure.""" + s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2) + _seed_cached(s, "a", n_blocks=2) + + # Load "a" → pinned. + s.update_state_after_alloc(_Request([_Feature("a")]), 0) + s.build_connector_meta(scheduler_output=None) + + # Step 1 after load: still pinned — can't evict for "b". + s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0) + meta1 = s.build_connector_meta(scheduler_output=None) + assert "b" not in meta1.saves + + # Step 2 after load: still pinned. + s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0) + meta2 = s.build_connector_meta(scheduler_output=None) + assert "b" not in meta2.saves + + # Step 3: unpin fires, now eviction works. + s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0) + meta3 = s.build_connector_meta(scheduler_output=None) + assert "b" in meta3.saves + s.shutdown() + + +def test_shutdown_unpins_pending_loads(monkeypatch): + """Shutdown must unpin entries still in _pending_loads (not yet drained + to the unpin deque). Observable: after shutdown, the entry is evictable + (alloc can reclaim its blocks).""" + s = _make_scheduler(monkeypatch, num_blocks=2) + _seed_cached(s, "a", n_blocks=2) + + s.update_state_after_alloc(_Request([_Feature("a")]), 0) + # Don't call build_connector_meta — "a" stays in _pending_loads. + s.shutdown() + + # After shutdown, scheduler reports no items (consumer role is off). + assert s.has_cache_item("a") is False + + +def test_shutdown_unpins_deferred_unpin_queue(monkeypatch): + """Shutdown must unpin entries queued in the deferred unpin deque + (already drained from _pending_loads but not yet expired).""" + s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2) + _seed_cached(s, "a", n_blocks=2) + + # Load "a" and build — moves to unpin deque. + s.update_state_after_alloc(_Request([_Feature("a")]), 0) + s.build_connector_meta(scheduler_output=None) + + # "a" is in the deferred unpin deque (depth=2, not yet expired). + # Shutdown must unpin it. + s.shutdown() + assert s.has_cache_item("a") is False + + +def test_eviction_skips_entry_pinned_by_pending_load(monkeypatch): + """A loaded entry still within the unpin delay window must survive + eviction attempts from new saves.""" + # 2 blocks: A occupies both, ready. + s = _make_scheduler(monkeypatch, num_blocks=2) + _seed_cached(s, "A", n_blocks=2) + + # Load A → pinned. + s.update_state_after_alloc(_Request([_Feature("A")]), 0) + meta_load = s.build_connector_meta(scheduler_output=None) + assert "A" in meta_load.loads + + # Try to save C (needs 2 blocks) — can't evict pinned A. + s.update_state_after_alloc(_Request([_Feature("C", length=2)]), 0) + meta_save = s.build_connector_meta(scheduler_output=None) + assert "C" not in meta_save.saves + assert s.has_cache_item("A") is True # survived + s.shutdown() + + +def test_region_full_skips_save_and_never_blocks(monkeypatch): + """When the region is fully occupied by pinned entries, new saves are + silently skipped and ensure_cache_available never blocks.""" + s = _make_scheduler(monkeypatch, num_blocks=1) + _seed_cached(s, "pinned", n_blocks=1) + s._cache.pin("pinned") + + req = _Request([_Feature("new", length=1)]) + assert s.ensure_cache_available(req, 0) is True + + s.update_state_after_alloc(req, 0) + meta = s.build_connector_meta(scheduler_output=None) + assert "new" not in meta.saves + assert s.has_cache_item("new") is False + + s._cache.unpin("pinned") + s.shutdown() + + +def test_step_readiness_depth_2(monkeypatch): + """With max_concurrent_batches=2, entries need 2 steps to become ready.""" + s = _make_scheduler(monkeypatch, max_concurrent_batches=2) + req = _Request([_Feature("h1", length=1)]) + + s.update_state_after_alloc(req, 0) + meta_0 = s.build_connector_meta(scheduler_output=None) + assert "h1" in meta_0.saves + assert s.has_cache_item("h1") is False + + # One step later: still not ready (depth=2). + s.build_connector_meta(scheduler_output=None) + assert s.has_cache_item("h1") is False + + # Two steps later: now ready. + s.build_connector_meta(scheduler_output=None) + assert s.has_cache_item("h1") is True + s.shutdown() + + +def test_producer_only_never_emits_loads(monkeypatch): + """A producer-only scheduler must never populate meta.loads, even when + entries are ready.""" + s = _make_scheduler(monkeypatch, is_consumer=False) + req = _Request([_Feature("h1", length=1)]) + + s.update_state_after_alloc(req, 0) + meta = s.build_connector_meta(scheduler_output=None) + assert "h1" in meta.saves + assert meta.loads == {} + + # Mark ready. + s.build_connector_meta(scheduler_output=None) + + # Even if we re-encounter the same feature, no load. + s.update_state_after_alloc(req, 0) + meta2 = s.build_connector_meta(scheduler_output=None) + assert meta2.loads == {} + s.shutdown() + + +def test_consumer_only_never_emits_saves(monkeypatch): + """A consumer-only scheduler must never populate meta.saves.""" + s = _make_scheduler(monkeypatch, is_producer=False) + _seed_cached(s, "a", n_blocks=1) + + req = _Request([_Feature("a", length=1)]) + s.update_state_after_alloc(req, 0) + meta = s.build_connector_meta(scheduler_output=None) + assert meta.saves == {} + assert "a" in meta.loads + s.shutdown() + + +def test_consumer_only_has_cache_item(monkeypatch): + """Consumer-only: has_cache_item works normally for ready entries.""" + s = _make_scheduler(monkeypatch, is_producer=False) + assert s.has_cache_item("a") is False + _seed_cached(s, "a", n_blocks=1) + assert s.has_cache_item("a") is True + s.shutdown() + + +def test_save_not_emitted_for_already_cached_entry(monkeypatch): + """An entry already in the cache (from a prior save) must not trigger + a second allocation or appear in meta.saves again.""" + s = _make_scheduler(monkeypatch) + req = _Request([_Feature("h1", length=1)]) + + # First encounter → save. + s.update_state_after_alloc(req, 0) + meta1 = s.build_connector_meta(scheduler_output=None) + assert "h1" in meta1.saves + + # Second encounter (same step or later) → no save. + s.update_state_after_alloc(req, 0) + meta2 = s.build_connector_meta(scheduler_output=None) + assert "h1" not in meta2.saves + s.shutdown() + + +# ── first-finish fast-path (idle engine) ──────────────────────────────────── + + +def test_first_finish_marks_ready_immediately(monkeypatch): + """When the originating request finishes, the save entry becomes ready + without waiting for the step-count delay.""" + s = _make_scheduler(monkeypatch, max_concurrent_batches=5) + req = _Request([_Feature("h1", length=1)], request_id="req_x") + + s.update_state_after_alloc(req, 0) + s.build_connector_meta(scheduler_output=None) + assert s.has_cache_item("h1") is False + + # Request finishes — readiness should fire immediately. + s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_x"})) + assert s.has_cache_item("h1") is True + s.shutdown() + + +def test_first_finish_unpins_immediately(monkeypatch): + """When the originating request finishes, the load entry is unpinned + without waiting for the step-count delay.""" + s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=5) + _seed_cached(s, "a", n_blocks=2) + + req = _Request([_Feature("a")], request_id="req_y") + s.update_state_after_alloc(req, 0) + meta = s.build_connector_meta(scheduler_output=None) + assert "a" in meta.loads + + # "a" is pinned. Try to evict — must fail. + s.update_state_after_alloc( + _Request([_Feature("b", length=2)], request_id="req_z"), 0 + ) + meta2 = s.build_connector_meta(scheduler_output=None) + assert "b" not in meta2.saves + + # req_y finishes — unpin fires in build_connector_meta (fast-path). + s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_y"})) + + # Next step: "a" is now evictable, so "b" can be allocated. + s.update_state_after_alloc( + _Request([_Feature("b", length=2)], request_id="req_w"), 0 + ) + meta3 = s.build_connector_meta(scheduler_output=None) + assert "b" in meta3.saves + s.shutdown() diff --git a/tests/v1/ec_connector/unit/cpu/scheduler/test_step_tracker.py b/tests/v1/ec_connector/unit/cpu/scheduler/test_step_tracker.py new file mode 100644 index 0000000000..370b46412b --- /dev/null +++ b/tests/v1/ec_connector/unit/cpu/scheduler/test_step_tracker.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for StepTracker.""" + +from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import ( + StepTracker, +) + + +def test_basic_step_count_expiry_depth_1(): + """With max_concurrent_batches=1, entry expires after 1 step.""" + t = StepTracker(max_concurrent_batches=1) + t.add("h1", "req_a") + + # Step 1: entry was added, committed to slot. Deque fills to maxlen=1, + # but no expiry yet (the entry is in the slot just committed). + result = t.step(set()) + assert result == [] + + # Step 2: deque is full, oldest slot (containing h1) expires. + result = t.step(set()) + assert result == ["h1"] + + +def test_basic_step_count_expiry_depth_2(): + """With max_concurrent_batches=2, entry expires after 2 steps.""" + t = StepTracker(max_concurrent_batches=2) + t.add("h1", "req_a") + + assert t.step(set()) == [] # step 1: committed + assert t.step(set()) == [] # step 2: deque fills but h1 in newer slot + assert t.step(set()) == ["h1"] # step 3: h1's slot is oldest, expires + + +def test_first_finish_fast_path(): + """Entry is returned immediately when request_id appears in finished.""" + t = StepTracker(max_concurrent_batches=3) + t.add("h1", "req_a") + + result = t.step({"req_a"}) + assert result == ["h1"] + + +def test_no_double_processing_fast_path_then_expiry(): + """Entry fast-pathed should NOT fire again on deque expiry.""" + t = StepTracker(max_concurrent_batches=1) + t.add("h1", "req_a") + + # Fast-path fires h1. + result = t.step({"req_a"}) + assert result == ["h1"] + + # Normal expiry would fire now, but h1 is already processed. + result = t.step(set()) + assert result == [] + + +def test_no_double_processing_expiry_then_fast_path(): + """Entry expired via step-count should NOT fire again via fast-path.""" + t = StepTracker(max_concurrent_batches=1) + t.add("h1", "req_a") + + t.step(set()) # commit + result = t.step(set()) # expiry fires h1 + assert result == ["h1"] + + # req_a finishes later — must not re-fire. + result = t.step({"req_a"}) + assert result == [] + + +def test_multiplicity_same_hash_different_requests(): + """Two adds of the same mm_hash (different requests) → returned twice.""" + t = StepTracker(max_concurrent_batches=3) + t.add("h1", "req_a") + t.add("h1", "req_b") + + # Both requests finish in the same step. + result = t.step({"req_a", "req_b"}) + assert sorted(result) == ["h1", "h1"] + + +def test_mixed_expiry_and_fast_path(): + """Some entries expire, others fast-path, in the same step.""" + t = StepTracker(max_concurrent_batches=1) + t.add("h1", "req_a") + + t.step(set()) # commit h1's slot + + # Add h2 in the next step. + t.add("h2", "req_b") + + # This step: h1's slot expires AND req_b finishes. + result = t.step({"req_b"}) + assert set(result) == {"h1", "h2"} + + +def test_drain_all_returns_unprocessed(): + """drain_all returns everything pending and clears state.""" + t = StepTracker(max_concurrent_batches=3) + t.add("h1", "req_a") + t.add("h2", "req_b") + t.step(set()) # commit both + t.add("h3", "req_c") # in _current, not yet committed + + result = t.drain_all() + assert set(result) == {"h1", "h2", "h3"} + + # After drain, everything is empty. + assert t.step(set()) == [] + assert t.drain_all() == [] + + +def test_drain_all_skips_already_processed(): + """drain_all should not re-return entries already fast-pathed.""" + t = StepTracker(max_concurrent_batches=3) + t.add("h1", "req_a") + t.add("h2", "req_b") + + # Fast-path h1. + result = t.step({"req_a"}) + assert result == ["h1"] + + # drain should only return h2. + result = t.drain_all() + assert result == ["h2"] + + +def test_empty_step_returns_nothing(): + """step() with nothing pending returns empty list.""" + t = StepTracker(max_concurrent_batches=2) + assert t.step(set()) == [] + assert t.step({"req_x"}) == [] + + +def test_add_after_step_goes_to_next_slot(): + """Entries added after a step go into the next slot, not the current.""" + t = StepTracker(max_concurrent_batches=1) + + t.step(set()) # empty step + t.add("h1", "req_a") + + # h1 is in _current. Next step commits it. + t.step(set()) + + # Now the deque is full (1 slot with h1). Next step expires it. + result = t.step(set()) + assert result == ["h1"] + + +def test_fast_path_on_current_entries(): + """Entries still in _current (not yet committed) can be fast-pathed.""" + t = StepTracker(max_concurrent_batches=3) + t.add("h1", "req_a") + + # req_a finishes in the same step h1 was added. + result = t.step({"req_a"}) + assert result == ["h1"] + + # Entry is committed to slot but marked processed — won't re-fire. + for _ in range(5): + assert t.step(set()) == [] diff --git a/tests/v1/ec_connector/unit/cpu/test_connector.py b/tests/v1/ec_connector/unit/cpu/test_connector.py new file mode 100644 index 0000000000..a90c2f72bf --- /dev/null +++ b/tests/v1/ec_connector/unit/cpu/test_connector.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from unittest.mock import MagicMock + +from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole +from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector +from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory + + +def _cfg(): + ec = MagicMock() + ec.is_ec_producer = True + ec.is_ec_consumer = True + cfg = MagicMock() + cfg.ec_transfer_config = ec + return cfg + + +def test_scheduler_role_builds_only_scheduler(monkeypatch): + fake_sched = MagicMock() + monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched) + c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER) + assert c.connector_scheduler is fake_sched + assert c.connector_worker is None + c.has_cache_item("x") + fake_sched.has_cache_item.assert_called_once_with("x") + + +def test_worker_role_builds_only_worker(monkeypatch): + fake_worker = MagicMock() + monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: fake_worker) + c = ECCPUConnector(_cfg(), ECConnectorRole.WORKER) + assert c.connector_worker is fake_worker + assert c.connector_scheduler is None + + +def test_request_finished_inherited_noop(monkeypatch): + monkeypatch.setattr( + ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock() + ) + c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER) + assert c.request_finished(MagicMock()) == (False, None) + + +def test_factory_registered(): + cls = ECConnectorFactory._registry["ECCPUConnector"]() + assert cls is ECCPUConnector diff --git a/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py b/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py new file mode 100644 index 0000000000..8224db31bf --- /dev/null +++ b/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for ECSharedRegion (the mmap substrate).""" + +import os +import uuid +from unittest.mock import MagicMock, patch + +import pytest +import torch + +from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( + ECSharedRegion, + _wait_for_file_size, +) + + +def _make_region(num_blocks: int = 8, block_size_bytes: int = 64) -> ECSharedRegion: + return ECSharedRegion( + engine_id=str(uuid.uuid4()), + num_blocks=num_blocks, + block_size_bytes=block_size_bytes, + ) + + +@pytest.fixture +def region() -> ECSharedRegion: + r = _make_region() + yield r + r.cleanup() + + +# ── mmap sharing between two instances ─────────────────────────────────────── + + +def test_second_instance_opens_existing_file_and_shares_memory(): + instance_id = str(uuid.uuid4()) + r1 = ECSharedRegion( + engine_id=instance_id, + num_blocks=4, + block_size_bytes=64, + ) + try: + r2 = ECSharedRegion( + engine_id=instance_id, + num_blocks=4, + block_size_bytes=64, + ) + try: + assert r1._is_creator + assert not r2._is_creator + # Both map the same physical pages; writes via r1 are visible via r2. + r1.blocks[0, :4] = torch.tensor([10, 20, 30, 40], dtype=torch.int8) + assert r2.blocks[0, :4].tolist() == [10, 20, 30, 40] + finally: + r2.cleanup() + finally: + r1.cleanup() + + +def test_only_creator_unlinks_file_on_cleanup(): + """Critical contract: if the non-creator unlinks, the creator's mmap path + becomes a dangling backing file and a third opener would create a new one + out from under the creator.""" + instance_id = str(uuid.uuid4()) + r1 = ECSharedRegion( + engine_id=instance_id, + num_blocks=4, + block_size_bytes=64, + ) + path = r1._mmap_path + r2 = ECSharedRegion( + engine_id=instance_id, + num_blocks=4, + block_size_bytes=64, + ) + + # Non-creator goes away first — file must still be on disk for r1. + r2.cleanup() + assert os.path.exists(path), "non-creator cleanup must not unlink the file" + + # Creator goes away — file is removed. + r1.cleanup() + assert not os.path.exists(path), "creator cleanup must unlink the file" + + +# ── _wait_for_file_size ────────────────────────────────────────────────────── + + +def test_wait_for_file_size_returns_when_already_big_enough(tmp_path): + """The fast path: file already at expected size — return immediately.""" + p = tmp_path / "f.bin" + p.write_bytes(b"\x00" * 128) + fd = os.open(str(p), os.O_RDONLY) + try: + _wait_for_file_size(fd, expected_size=128, timeout=1.0) # must not raise + finally: + os.close(fd) + + +def test_wait_for_file_size_times_out_when_file_stays_empty(tmp_path): + p = tmp_path / "f.bin" + p.write_bytes(b"") + fd = os.open(str(p), os.O_RDONLY) + try: + with pytest.raises(TimeoutError): + _wait_for_file_size(fd, expected_size=4096, timeout=0.05) + finally: + os.close(fd) + + +# ── pin_memory ─────────────────────────────────────────────────────────────── + + +def test_pin_memory_success_sets_flag(region): + """When cudaHostRegister returns 0, _is_pinned flips to True + and cleanup will correspondingly call cudaHostUnregister.""" + fake_cudart = MagicMock() + success = MagicMock() + success.value = 0 + fake_cudart.cudaHostRegister.return_value = success + fake_cudart.cudaHostUnregister.return_value = success + + with ( + patch("torch.cuda.is_available", return_value=True), + patch("torch.cuda.cudart", return_value=fake_cudart), + ): + region.pin_memory() + assert region._is_pinned is True + # cleanup must pair with cudaHostUnregister exactly once. + region.cleanup() + fake_cudart.cudaHostUnregister.assert_called_once() + + +def test_pin_memory_failure_leaves_flag_false(): + """If cudaHostRegister fails (non-zero), don't pretend it succeeded — + cleanup must NOT call cudaHostUnregister on memory we never registered.""" + r = _make_region() + try: + fake_cudart = MagicMock() + fail = MagicMock() + fail.value = 1 # non-zero == error + fake_cudart.cudaHostRegister.return_value = fail + + with ( + patch("torch.cuda.is_available", return_value=True), + patch("torch.cuda.cudart", return_value=fake_cudart), + ): + r.pin_memory() + assert r._is_pinned is False + # Now run cleanup and verify cudaHostUnregister was NOT called. + r.cleanup() + fake_cudart.cudaHostUnregister.assert_not_called() + finally: + r.cleanup() + + +def test_pin_memory_noop_without_cuda(region): + """pin_memory is a no-op when CUDA is not available.""" + with patch("torch.cuda.is_available", return_value=False): + region.pin_memory() + assert region._is_pinned is False + + +# ── cleanup idempotency ─────────────────────────────────────────────────────── + + +def test_cleanup_is_idempotent(region): + region.cleanup() + region.cleanup() # fixture calls a third time — must not raise diff --git a/tests/v1/ec_connector/unit/cpu/worker/test_worker.py b/tests/v1/ec_connector/unit/cpu/worker/test_worker.py new file mode 100644 index 0000000000..4668abf251 --- /dev/null +++ b/tests/v1/ec_connector/unit/cpu/worker/test_worker.py @@ -0,0 +1,598 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for ECCPUWorker. + +The byte-level tests exercise real CUDA stream/event coordination +against a real ``ECSharedRegion`` mmap and are skipped on hosts without +CUDA. The lifecycle tests don't fire any ``torch.cuda.*`` primitives and +run anywhere. + +Mocking policy +-------------- +- ``create_ec_shared_region`` is patched to inject a real ``ECSharedRegion`` + (mmap backed) with deterministic small dimensions, so byte assertions + hit the actual mmap. The real responsibility of the worker — block + index → mmap byte mapping, stream coordination, dtype/shape — is + exercised end to end against a real GPU. +- ``is_pin_memory_available`` is patched per-test so the rank-gating + path can be probed deterministically without depending on host caps. +- The shutdown test bypasses ``__init__`` via ``object.__new__`` and + injects a ``MagicMock`` region. +""" + +import contextlib +import logging +import uuid +from unittest.mock import MagicMock, Mock, patch + +import pytest +import torch + +from vllm.config import VllmConfig +from vllm.config.parallel import ParallelConfig +from vllm.distributed.ec_transfer.ec_connector.cpu.common import ( + ECCPUConnectorMetadata, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( + ECSharedRegion, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker + +# ── shape constants ────────────────────────────────────────────────────────── + +# hidden_dim=8, dtype=fp16 (2 bytes) → block_size_bytes = 16, so one row of +# encoder output fits in exactly one block. +_HIDDEN_DIM = 8 +_DTYPE = torch.float16 +_BLOCK_SIZE_BYTES = _HIDDEN_DIM * _DTYPE.itemsize +_NUM_BLOCKS = 8 + +_requires_cuda = pytest.mark.skipif( + not torch.cuda.is_available(), + reason="exercises real CUDA stream/event coordination in ECCPUWorker", +) + + +def _make_region() -> ECSharedRegion: + """Fresh region backed by a real per-test mmap file.""" + return ECSharedRegion( + engine_id=str(uuid.uuid4()), + num_blocks=_NUM_BLOCKS, + block_size_bytes=_BLOCK_SIZE_BYTES, + ) + + +def _vllm_config(rank: int = 0) -> Mock: + cfg = Mock(spec=VllmConfig) + cfg.parallel_config = Mock(spec=ParallelConfig) + cfg.parallel_config.rank = rank + cfg.model_config = Mock() + cfg.model_config.dtype = _DTYPE + return cfg + + +def _meta( + *, saves: dict | None = None, loads: dict | None = None +) -> ECCPUConnectorMetadata: + return ECCPUConnectorMetadata(saves=saves or {}, loads=loads or {}) + + +# ── fixtures ───────────────────────────────────────────────────────────────── + + +@pytest.fixture +def make_worker(): + """Factory that builds an ``ECCPUWorker`` over a real ``ECSharedRegion``. + + Tracks the workers and regions it creates and tears them down at the + end of the test, so each test starts from a fresh /dev/shm file. + """ + workers: list[ECCPUWorker] = [] + regions: list[ECSharedRegion] = [] + + def factory( + *, + rank: int = 0, + tp_rank: int = 0, + pcp_rank: int = 0, + pin_memory_available: bool = False, + ) -> ECCPUWorker: + region = _make_region() + regions.append(region) + pcp_group = Mock() + pcp_group.rank_in_group = pcp_rank + with ( + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.create_ec_shared_region", + return_value=region, + ), + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.is_pin_memory_available", + return_value=pin_memory_available, + ), + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_tensor_model_parallel_rank", + return_value=tp_rank, + ), + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_pcp_group", + return_value=pcp_group, + ), + ): + worker = ECCPUWorker(_vllm_config(rank=rank)) + workers.append(worker) + return worker + + yield factory + + for worker in workers: + with contextlib.suppress(Exception): + worker.shutdown() + for region in regions: + with contextlib.suppress(Exception): + region.cleanup() + + +# ── save_caches ────────────────────────────────────────────────────────────── + + +@_requires_cuda +@pytest.mark.parametrize( + "n_elements,n_blocks", + [ + (3 * _HIDDEN_DIM, 3), # exact fit: 3 full blocks, no remainder + (3 * _HIDDEN_DIM + 4, 4), # partial last block: 4 fp16 trail in slot 3 + ], + ids=["exact-fit", "partial-last-block"], +) +def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks): + """``save_caches`` + ``flush_saves`` copies the source GPU tensor's bytes + into the block IDs named by ``meta.saves``, in the order they appear, + via a single batched GPU→CPU copy. + + When ``total_bytes`` is not a multiple of ``block_size_bytes`` the last + block is partially written; the unwritten tail must remain whatever + was there before. Blocks not named in ``meta.saves`` must be untouched. + """ + worker = make_worker() + sentinel = 0x5A + worker._region.blocks.fill_(sentinel) + + src = torch.arange(n_elements, dtype=_DTYPE, device="cuda") + expected_bytes = src.cpu().reshape(-1).view(torch.uint8) + total_bytes = n_elements * _DTYPE.itemsize + + block_ids = [7, 2, 5, 0][:n_blocks] + worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids})) + worker.flush_saves() + + for slot, block_idx in enumerate(block_ids): + block_byte_start = slot * _BLOCK_SIZE_BYTES + block_byte_end = min(block_byte_start + _BLOCK_SIZE_BYTES, total_bytes) + n_written = block_byte_end - block_byte_start + actual_bytes = worker._region.blocks[block_idx, :n_written].view(torch.uint8) + assert torch.equal( + actual_bytes, expected_bytes[block_byte_start:block_byte_end] + ), f"block {block_idx} (slot {slot}) bytes mismatch" + if n_written < _BLOCK_SIZE_BYTES: + tail = worker._region.blocks[block_idx, n_written:] + assert torch.all(tail == sentinel), ( + f"block {block_idx} tail was overwritten" + ) + + for idx in set(range(_NUM_BLOCKS)) - set(block_ids): + assert torch.all(worker._region.blocks[idx] == sentinel), ( + f"block {idx} (unassigned) was overwritten" + ) + + +def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker): + """When the scheduler hasn't pre-allocated blocks for ``mm_hash``, + ``save_caches`` + ``flush_saves`` is a pure no-op.""" + worker = make_worker() + sentinel = 0x42 + worker._region.blocks.fill_(sentinel) + + worker.save_caches({}, "h", _meta(saves={})) + worker.flush_saves() + + assert torch.all(worker._region.blocks == sentinel) + + +@pytest.mark.parametrize( + "tp_rank,pcp_rank", + [(1, 0), (0, 1), (1, 1)], + ids=["tp1-pcp0", "tp0-pcp1", "tp1-pcp1"], +) +def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank): + """Only TP rank 0 + PCP rank 0 writes to mmap. All other combinations + must skip the save entirely.""" + worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank) + sentinel = 0x42 + worker._region.blocks.fill_(sentinel) + + worker.save_caches( + {"h": torch.zeros(8, dtype=_DTYPE)}, "h", _meta(saves={"h": [0]}) + ) + worker.flush_saves() + + assert torch.all(worker._region.blocks == sentinel) + + +def test_save_caches_raises_when_allocated_blocks_too_small(make_worker): + """``save_caches`` must raise ``AssertionError`` when the encoder output + is larger than the allocated block space.""" + worker = make_worker() + src = torch.zeros(3 * _HIDDEN_DIM, dtype=_DTYPE) + with pytest.raises(AssertionError, match="exceeds allocated blocks"): + worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]})) + + +@_requires_cuda +def test_save_caches_batches_multiple_hashes(make_worker): + """Multiple save_caches calls are batched into a single flush.""" + worker = make_worker() + sentinel = 0x5A + worker._region.blocks.fill_(sentinel) + + src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda") + src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device="cuda") + + cache = {"a": src_a, "b": src_b} + worker.save_caches(cache, "a", _meta(saves={"a": [1], "b": [3]})) + worker.save_caches(cache, "b", _meta(saves={"a": [1], "b": [3]})) + worker.flush_saves() + + expected_a = src_a.cpu().view(torch.uint8) + expected_b = src_b.cpu().view(torch.uint8) + actual_a = worker._region.blocks[1].view(torch.uint8) + actual_b = worker._region.blocks[3].view(torch.uint8) + assert torch.equal(actual_a, expected_a) + assert torch.equal(actual_b, expected_b) + + # Unassigned blocks untouched. + for idx in (0, 2, 4, 5, 6, 7): + assert torch.all(worker._region.blocks[idx] == sentinel) + + +# ── start_load_caches ──────────────────────────────────────────────────────── + + +@_requires_cuda +def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker): + """Single batched load across all hashes with correct byte→dtype→shape.""" + worker = make_worker() + n_blocks = 3 + src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape( + n_blocks, _HIDDEN_DIM + ) + src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES) + + block_ids = [3, 1, 6] + for i, idx in enumerate(block_ids): + worker._region.blocks[idx].copy_(src_int8[i]) + + encoder_cache: dict[str, torch.Tensor] = {} + worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids})) + + out = encoder_cache["h"] + assert out.is_cuda, "consumer worker must place the tensor on the GPU" + assert out.shape == (n_blocks, _HIDDEN_DIM) + assert out.dtype == _DTYPE + assert torch.equal(out.cpu(), src_orig) + + +@_requires_cuda +def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker): + """If ``encoder_cache`` already holds the ``mm_hash``, the worker must + not overwrite it.""" + worker = make_worker() + worker._region.blocks[0].fill_(0x42) + + sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device="cuda") + encoder_cache = {"h": sentinel} + worker.start_load_caches(encoder_cache, _meta(loads={"h": [0]})) + + assert encoder_cache["h"] is sentinel, ( + "existing encoder_cache entry must not be replaced" + ) + + +@_requires_cuda +def test_start_load_caches_noop_when_loads_is_empty(make_worker): + """When ``meta.loads`` is empty the early-return must fire.""" + worker = make_worker() + encoder_cache: dict[str, torch.Tensor] = {} + worker.start_load_caches(encoder_cache, _meta(loads={})) + + assert encoder_cache == {} + + +@_requires_cuda +def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker): + """Cached entries are preserved while new ones are loaded.""" + worker = make_worker() + n_blocks = 2 + src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape( + n_blocks, _HIDDEN_DIM + ) + src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES) + new_block_ids = [4, 2] + for i, idx in enumerate(new_block_ids): + worker._region.blocks[idx].copy_(src_int8[i]) + + cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device="cuda") + encoder_cache: dict[str, torch.Tensor] = {"cached_h": cached_tensor} + worker.start_load_caches( + encoder_cache, + _meta(loads={"cached_h": [0], "new_h": new_block_ids}), + ) + + assert encoder_cache["cached_h"] is cached_tensor + assert "new_h" in encoder_cache + out = encoder_cache["new_h"] + assert out.shape == (n_blocks, _HIDDEN_DIM) + assert out.dtype == _DTYPE + assert torch.equal(out.cpu(), src_orig) + + +@_requires_cuda +@pytest.mark.parametrize( + "tp_rank,pcp_rank", + [(0, 0), (1, 0), (0, 1), (1, 1)], + ids=["tp0-pcp0", "tp1-pcp0", "tp0-pcp1", "tp1-pcp1"], +) +def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank): + """All TP/PCP ranks must load from mmap — loads are NOT gated like saves.""" + worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank) + n_blocks = 2 + src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape( + n_blocks, _HIDDEN_DIM + ) + src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES) + block_ids = [1, 3] + for i, idx in enumerate(block_ids): + worker._region.blocks[idx].copy_(src_int8[i]) + + encoder_cache: dict[str, torch.Tensor] = {} + worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids})) + + out = encoder_cache["h"] + assert out.is_cuda + assert torch.equal(out.cpu(), src_orig) + + +# ── round-trip ─────────────────────────────────────────────────────────────── + + +@_requires_cuda +def test_save_then_load_round_trips_bytes(make_worker): + """Full producer→mmap→consumer byte path in one shot.""" + worker = make_worker() + n_blocks = 3 + block_ids = [5, 1, 6] + + src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape( + n_blocks, _HIDDEN_DIM + ) + worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids})) + worker.flush_saves() + + encoder_cache: dict[str, torch.Tensor] = {} + worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids})) + + out = encoder_cache["h"] + assert out.shape == src.shape + assert out.dtype == src.dtype + assert torch.equal(out.cpu(), src.cpu()) + + +# ── buffer recycling ──────────────────────────────────────────────────────── + + +@_requires_cuda +def test_buffer_pool_is_reused_across_save_steps(make_worker): + """After flush_saves, descriptor buffers are returned to the pool and + reused on the next flush — no reallocation.""" + worker = make_worker() + src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda") + + worker.save_caches({"h": src}, "h", _meta(saves={"h": [0]})) + worker.flush_saves() + + assert len(worker._buf_pool._pool) == 1 + buf_id = id(worker._buf_pool._pool[0].src_ptrs) + + # Second step reuses the same buffer. + worker.save_caches({"h": src}, "h", _meta(saves={"h": [1]})) + worker.flush_saves() + + assert len(worker._buf_pool._pool) == 1 + assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id + + +@_requires_cuda +def test_buffer_pool_is_reused_across_load_steps(make_worker): + """After start_load_caches, descriptor buffers are returned to the pool + and reused on the next call.""" + worker = make_worker() + worker._region.blocks[0].fill_(0x01) + worker._region.blocks[1].fill_(0x02) + + encoder_cache: dict[str, torch.Tensor] = {} + worker.start_load_caches(encoder_cache, _meta(loads={"a": [0]})) + + assert len(worker._buf_pool._pool) == 1 + buf_id = id(worker._buf_pool._pool[0].src_ptrs) + + encoder_cache2: dict[str, torch.Tensor] = {} + worker.start_load_caches(encoder_cache2, _meta(loads={"b": [1]})) + + assert len(worker._buf_pool._pool) == 1 + assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id + + +# ── stream management ──────────────────────────────────────────────────────── + + +@_requires_cuda +def test_stream_initialized_at_construction(make_worker): + """``_load_stream`` must be a fully initialized CUDA stream.""" + worker = make_worker() + assert isinstance(worker._load_stream, torch.cuda.Stream) + + +# ── lifecycle ──────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "pin_available,expected_pinned", + [ + (True, True), + (False, False), + ], + ids=["available", "unavailable"], +) +def test_init_pins_memory_when_available(pin_available, expected_pinned): + """Every worker must call ``pin_memory()`` when the platform allows it.""" + region = _make_region() + region.pin_memory = MagicMock() # type: ignore[method-assign] + + pcp_group = Mock() + pcp_group.rank_in_group = 0 + with ( + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.create_ec_shared_region", + return_value=region, + ), + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.is_pin_memory_available", + return_value=pin_available, + ), + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_tensor_model_parallel_rank", + return_value=0, + ), + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_pcp_group", + return_value=pcp_group, + ), + ): + ECCPUWorker(_vllm_config(rank=0)) + + try: + if expected_pinned: + region.pin_memory.assert_called_once() + else: + region.pin_memory.assert_not_called() + finally: + region.cleanup() + + +def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm): + """``shutdown`` must always call ``region.cleanup`` — and must never + raise.""" + worker = object.__new__(ECCPUWorker) + mock_region = Mock(spec=ECSharedRegion) + worker._region = mock_region + worker._load_stream = MagicMock() + worker._save_bufs = None + worker._save_count = 0 + + worker.shutdown() + worker._load_stream.synchronize.assert_called_once() + mock_region.cleanup.assert_called_once() + + mock_region.cleanup.side_effect = RuntimeError("boom") + with caplog_vllm.at_level(logging.DEBUG, logger="vllm"): + worker.shutdown() # exception must be swallowed + + assert mock_region.cleanup.call_count == 2 + assert any( + "worker region cleanup failed" in r.message + for r in caplog_vllm.records + if r.levelno == logging.DEBUG + ) + + +# ── e2e: scheduler + worker pipeline ──────────────────────────────────────── + + +@_requires_cuda +def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch): + """Full pipeline: scheduler allocates blocks, worker saves GPU tensor to + mmap via flush_saves, scheduler marks ready after step delay, worker + loads from mmap back to GPU, and the result matches the original. + + Exercises the real scheduler + worker cooperation through a shared + ECSharedRegion, with real CUDA transfers and stream coordination. + """ + import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod + from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ( + ECCPUScheduler, + ) + + # Build worker first (gets us a real region). + worker = make_worker() + region = worker._region + + # Build scheduler sharing the same region. + monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region) + + class _EC: + is_ec_producer = True + is_ec_consumer = True + + class _Cfg: + ec_transfer_config = _EC() + max_concurrent_batches = 1 + + scheduler = ECCPUScheduler(_Cfg()) + + # -- Step 1: scheduler allocates, worker saves -- + n_blocks = 3 + src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape( + n_blocks, _HIDDEN_DIM + ) + + class _Pos: + offset = 0 + length = n_blocks + + class _Feature: + identifier = "img_001" + mm_position = _Pos() + + class _Request: + request_id = "req_e2e" + mm_features = [_Feature()] + + scheduler.update_state_after_alloc(_Request(), 0) + meta_save = scheduler.build_connector_meta(scheduler_output=None) + assert "img_001" in meta_save.saves + + encoder_cache = {"img_001": src} + worker.save_caches(encoder_cache, "img_001", meta_save) + worker.flush_saves() + + # -- Step 2: readiness delay (max_concurrent_batches=1) -- + meta_step2 = scheduler.build_connector_meta(scheduler_output=None) + assert scheduler.has_cache_item("img_001") is True + assert meta_step2.loads == {} # no load requested yet + + # -- Step 3: scheduler emits load, worker loads -- + scheduler.update_state_after_alloc(_Request(), 0) + meta_load = scheduler.build_connector_meta(scheduler_output=None) + assert "img_001" in meta_load.loads + assert meta_load.loads["img_001"] == meta_save.saves["img_001"] + + load_cache: dict[str, torch.Tensor] = {} + worker.start_load_caches(load_cache, meta_load) + + out = load_cache["img_001"] + assert out.is_cuda + assert out.shape == src.shape + assert out.dtype == src.dtype + assert torch.equal(out.cpu(), src.cpu()) + + scheduler.shutdown() diff --git a/tests/v1/ec_connector/unit/test_ec_cpu_connector.py b/tests/v1/ec_connector/unit/test_ec_cpu_connector.py new file mode 100644 index 0000000000..3a2600860a --- /dev/null +++ b/tests/v1/ec_connector/unit/test_ec_cpu_connector.py @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E test for the ECCPUConnector. + +Verifies: +- Accuracy: outputs from EC CPU cache match fresh encoder computation. +- Latency: loading from EC CPU cache is faster than a cold encoder run. + +Requires a CUDA GPU and the Qwen2-VL-2B-Instruct model. +""" + +import time + +import pytest +from tqdm import tqdm + +from vllm import LLM, SamplingParams +from vllm.assets.image import ImageAsset +from vllm.config import ECTransferConfig +from vllm.platforms import current_platform + +MODEL = "Qwen/Qwen2-VL-2B-Instruct" +EC_CPU_BYTES = 500 << 20 # 500 MiB + + +def _build_image_prompt(image): + """Build a Qwen2-VL prompt with a single image.""" + return { + "prompt": ( + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n" + "<|vision_start|><|image_pad|><|vision_end|>" + "Describe this image briefly.<|im_end|>\n" + "<|im_start|>assistant\n" + ), + "multi_modal_data": {"image": image}, + } + + +def _make_unique_image(base_image, iteration: int): + """Create a unique image by modifying a pixel to change the mm_hash.""" + img = base_image.copy() + img.putpixel((0, 0), (iteration % 256, iteration // 256 % 256, 0)) + return img + + +def _wait_for_ec_ready(llm: LLM) -> None: + """Send a dummy request to force a scheduler step. + + After generate() returns, the EC entry's data is in the mmap but + readiness may not have fired yet (first-finish fires in the NEXT + scheduler step). This forces that step. + """ + llm.generate("hi", SamplingParams(max_tokens=1), use_tqdm=False) + + +def _latency_test(llm: LLM) -> None: + """Verify EC CPU cache hit is faster than cold encoder computation.""" + if not current_platform.is_cuda(): + pytest.skip("Latency test requires CUDA") + + sampling_params = SamplingParams(max_tokens=1, temperature=0) + base_image = ImageAsset("stop_sign").pil_image + + num_times_ec_better_than_cold = 0 + num_tests = 10 + total_cold_time = 0.0 + total_ec_hit_time = 0.0 + + for i in tqdm(range(num_tests), desc="EC latency test"): + # Use a unique image each iteration so the cold run is truly cold + # (EC has never seen this mm_hash before). + image = _make_unique_image(base_image, i) + prompt = _build_image_prompt(image) + + # Cold run: encoder computes from scratch, saves to EC CPU. + llm.llm_engine.reset_encoder_cache() + start_time = time.time() + llm.generate([prompt], sampling_params, use_tqdm=False) + cold_time = time.time() - start_time + total_cold_time += cold_time + + # Ensure EC entry is marked ready. + _wait_for_ec_ready(llm) + + # Clear GPU encoder cache but keep EC CPU cache intact. + llm.llm_engine.reset_encoder_cache() + + # EC hit: scheduler loads from CPU mmap instead of re-encoding. + start_time = time.time() + llm.generate([prompt], sampling_params, use_tqdm=False) + ec_hit_time = time.time() - start_time + total_ec_hit_time += ec_hit_time + + if ec_hit_time < cold_time: + num_times_ec_better_than_cold += 1 + + print("Average times:") + print(f" Cold: {total_cold_time * 1000 / num_tests:.2f}ms") + print(f" EC hit: {total_ec_hit_time * 1000 / num_tests:.2f}ms") + + assert num_times_ec_better_than_cold >= 0.8 * num_tests, ( + f"EC hit was faster only {num_times_ec_better_than_cold}/{num_tests} " + f"times (expected >= 80%)" + ) + + +def _accuracy_test(llm: LLM) -> None: + """Verify EC CPU cache produces correct outputs.""" + sampling_params = SamplingParams(max_tokens=8, temperature=0) + image = ImageAsset("stop_sign").pil_image + prompt = _build_image_prompt(image) + + # Seed: generate with fresh encoder to establish baseline output. + llm.llm_engine.reset_encoder_cache() + baseline = llm.generate([prompt], sampling_params, use_tqdm=False) + baseline_text = baseline[0].outputs[0].text + + # Ensure EC entry is marked ready. + _wait_for_ec_ready(llm) + + # Clear GPU encoder cache so subsequent runs go through EC CPU path. + llm.llm_engine.reset_encoder_cache() + + # Generate multiple times from EC cache and check correctness. + test_count = 20 + results = llm.generate([prompt] * test_count, sampling_params, use_tqdm=False) + + success_count = sum(1 for r in results if r.outputs[0].text == baseline_text) + assert success_count >= 0.5 * test_count, ( + f"Only {success_count}/{test_count} outputs matched baseline " + f"(expected >= 50%). Baseline: {baseline_text!r}" + ) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +def test_ec_cpu_offloading() -> None: + """Tests ECCPUConnector accuracy and latency with a VLM model.""" + ec_transfer_config = ECTransferConfig( + ec_connector="ECCPUConnector", + ec_role="ec_both", + ec_connector_extra_config={"ec_cpu_bytes": EC_CPU_BYTES}, + ) + + llm = LLM( + model=MODEL, + max_model_len=2048, + gpu_memory_utilization=0.5, + enforce_eager=True, + ec_transfer_config=ec_transfer_config, + ) + + try: + _latency_test(llm) + _accuracy_test(llm) + finally: + del llm diff --git a/tests/v1/ec_connector/unit/test_metadata.py b/tests/v1/ec_connector/unit/test_metadata.py new file mode 100644 index 0000000000..55579e128c --- /dev/null +++ b/tests/v1/ec_connector/unit/test_metadata.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for ECCPUConnectorMetadata.""" + +from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata +from vllm.distributed.ec_transfer.ec_connector.cpu.common import ( + ECCPUConnectorMetadata, +) + + +def test_metadata_is_ec_connector_metadata(): + """ECCPUConnectorMetadata must be a subclass of ECConnectorMetadata.""" + meta = ECCPUConnectorMetadata() + assert isinstance(meta, ECConnectorMetadata) + + +def test_metadata_initialization(): + """ECCPUConnectorMetadata initializes with empty saves and loads dicts.""" + meta = ECCPUConnectorMetadata() + assert hasattr(meta, "saves") + assert hasattr(meta, "loads") + assert meta.saves == {} + assert meta.loads == {} + + +def test_metadata_saves_dict_operations(): + """The saves dict supports standard dict operations.""" + meta = ECCPUConnectorMetadata() + + # Add multiple entries + meta.saves["mm_hash_1"] = [0, 1, 2] + meta.saves["mm_hash_2"] = [10, 11] + + assert "mm_hash_1" in meta.saves + assert "mm_hash_2" in meta.saves + assert len(meta.saves) == 2 + assert meta.saves["mm_hash_1"] == [0, 1, 2] + assert meta.saves["mm_hash_2"] == [10, 11] + + +def test_metadata_loads_dict_operations(): + """The loads dict supports standard dict operations.""" + meta = ECCPUConnectorMetadata() + + # Add multiple entries + meta.loads["mm_hash_1"] = [5, 6, 7] + meta.loads["mm_hash_2"] = [100] + + assert "mm_hash_1" in meta.loads + assert "mm_hash_2" in meta.loads + assert len(meta.loads) == 2 + assert meta.loads["mm_hash_1"] == [5, 6, 7] + assert meta.loads["mm_hash_2"] == [100] + + +def test_metadata_saves_and_loads_are_independent(): + """saves and loads dicts are independent.""" + meta = ECCPUConnectorMetadata() + + meta.saves["key"] = [1, 2] + meta.loads["key"] = [3, 4] + + # Same key but different dicts should not interfere + assert meta.saves["key"] == [1, 2] + assert meta.loads["key"] == [3, 4] + + # Modifying one should not affect the other + meta.saves["other"] = [99] + assert "other" not in meta.loads + + +def test_metadata_block_indices_are_integers(): + """Block indices in saves/loads lists should be integers.""" + meta = ECCPUConnectorMetadata() + + meta.saves["hash"] = [0, 100, 999, 65535] + meta.loads["hash"] = [1, 2, 3] + + for idx in meta.saves["hash"]: + assert isinstance(idx, int) + for idx in meta.loads["hash"]: + assert isinstance(idx, int) + + +def test_metadata_multiple_instances_are_independent(): + """Different ECCPUConnectorMetadata instances should not share state.""" + meta1 = ECCPUConnectorMetadata() + meta2 = ECCPUConnectorMetadata() + + meta1.saves["x"] = [1] + meta2.saves["y"] = [2] + + assert "x" in meta1.saves + assert "x" not in meta2.saves + assert "y" not in meta1.saves + assert "y" in meta2.saves diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/__init__.py b/vllm/distributed/ec_transfer/ec_connector/cpu/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/common.py b/vllm/distributed/ec_transfer/ec_connector/cpu/common.py new file mode 100644 index 0000000000..771d4ca25a --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/common.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared types for the ECCPUConnector scheduler and worker delegates.""" + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import torch + +from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata +from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( + ECSharedRegion, +) + +if TYPE_CHECKING: + from vllm.config import VllmConfig + + +@dataclass +class ECCPUConnectorMetadata(ECConnectorMetadata): + """Per-step scheduler → worker payload for the ECCPUConnector. + + Populated by `ECCPUScheduler.build_connector_meta`; consumed by + `ECCPUWorker` via the mixin's `bind_connector_metadata`. + """ + + # Producer role: mm_hashes the scheduler has just allocated CPU + # blocks for this step; the worker's save_caches copies + # encoder_cache[mm_hash] → mmap at these block IDs. + saves: dict[str, list[int]] = field(default_factory=dict) + + # Consumer role: mm_hashes whose bytes are available in the local mmap; + # the worker's start_load_caches copies mmap[block_ids] → GPU + # encoder_cache. + loads: dict[str, list[int]] = field(default_factory=dict) + + +def _get_encoder_cache_hidden_dim(vllm_config: "VllmConfig") -> int: + """Return the per-token hidden dimension for encoder cache entries. + + For most models this equals the LLM's hidden size. Qwen3-VL (and any + future model with deepstack visual encoding) is an exception: the ViT + concatenates its own output with features from N decoder layers before + storing in encoder_cache, producing a tensor of width + ``out_hidden_size * (1 + N)`` per visual token. Using the plain LLM + hidden size would under-allocate EC blocks and silently truncate the + transferred data, leading to a shape mismatch on the consumer. + """ + model_config = vllm_config.model_config + hf_config = getattr(model_config, "hf_config", None) + vision_config = ( + getattr(hf_config, "vision_config", None) if hf_config is not None else None + ) + if vision_config is not None: + out_hidden_size = getattr(vision_config, "out_hidden_size", None) + deepstack_indexes = getattr(vision_config, "deepstack_visual_indexes", None) + if out_hidden_size is not None and deepstack_indexes: + return out_hidden_size * (1 + len(deepstack_indexes)) + return model_config.get_inputs_embeds_size() + + +def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion: + """Build the EC mmap region from `vllm_config`. + + Both `ECCPUScheduler` and `ECCPUWorker` call this to get the same + shared region (same engine_id, same block_size_bytes). + """ + ec_config = vllm_config.ec_transfer_config + assert ec_config is not None, "ec_transfer_config required to build region" + + dp_rank = vllm_config.parallel_config.data_parallel_rank + engine_id = f"{vllm_config.instance_id}_dp{dp_rank}" + + dtype = vllm_config.model_config.dtype + hidden_dim = _get_encoder_cache_hidden_dim(vllm_config) + element_size = torch.empty(0, dtype=dtype).element_size() + block_size_bytes = hidden_dim * element_size + + cpu_bytes = ec_config.ec_connector_extra_config.get("ec_cpu_bytes") + if not cpu_bytes: + raise ValueError("ec_cpu_bytes must be specified in ec_connector_extra_config") + cpu_bytes = int(cpu_bytes) + num_blocks = cpu_bytes // block_size_bytes + + return ECSharedRegion( + engine_id=engine_id, + num_blocks=num_blocks, + block_size_bytes=block_size_bytes, + ) diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/connector.py b/vllm/distributed/ec_transfer/ec_connector/cpu/connector.py new file mode 100644 index 0000000000..7de5c7fae3 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/connector.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ECCPUConnector — CPU encoder-cache offloading. + +A role-routed shell: one instance per process. The scheduler delegate owns the +mmap region and offload bookkeeping; the worker delegate owns the mmap view and +GPU copy plumbing. An ec_both instance reuses encoder outputs it has already +offloaded to CPU instead of recomputing them. +""" + +from typing import TYPE_CHECKING + +import torch + +from vllm.distributed.ec_transfer.ec_connector.base import ( + ECConnectorBase, + ECConnectorRole, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.common import ( + ECCPUConnectorMetadata, +) +from vllm.logger import init_logger + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.v1.core.sched.output import SchedulerOutput + from vllm.v1.request import Request + +logger = init_logger(__name__) + + +class ECCPUConnector(ECConnectorBase): + """EC connector that offloads encoder cache to a shared CPU mmap region.""" + + def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None: + super().__init__(vllm_config=vllm_config, role=role) + + self.connector_worker = None + self.connector_scheduler = None + + if role == ECConnectorRole.WORKER: + self.connector_worker = self._make_worker(vllm_config) + elif role == ECConnectorRole.SCHEDULER: + self.connector_scheduler = self._make_scheduler(vllm_config) + else: + raise ValueError(f"Unknown ECConnectorRole: {role}") + + # Construction seams. + def _make_worker(self, vllm_config: "VllmConfig"): + # Deferred import: the worker module touches torch/CUDA at import time + # via the region, so keep that cost off the scheduler path. + from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ( + ECCPUWorker, + ) + + return ECCPUWorker(vllm_config) + + def _make_scheduler(self, vllm_config: "VllmConfig"): + from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ( + ECCPUScheduler, + ) + + return ECCPUScheduler(vllm_config) + + # Worker-side forwarders. + def start_load_caches( + self, encoder_cache: dict[str, torch.Tensor], **kwargs + ) -> None: + assert self.connector_worker is not None + metadata = self._get_connector_metadata() + assert isinstance(metadata, ECCPUConnectorMetadata) + self.connector_worker.start_load_caches( + encoder_cache, connector_metadata=metadata + ) + + def save_caches( + self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs + ) -> None: + assert self.connector_worker is not None + metadata = self._get_connector_metadata() + assert isinstance(metadata, ECCPUConnectorMetadata) + self.connector_worker.save_caches( + encoder_cache, mm_hash, connector_metadata=metadata + ) + + # Scheduler-side forwarders. + def has_cache_item(self, identifier: str) -> bool: + assert self.connector_scheduler is not None + return self.connector_scheduler.has_cache_item(identifier) + + def ensure_cache_available( + self, request: "Request", num_computed_tokens: int + ) -> bool: + assert self.connector_scheduler is not None + return self.connector_scheduler.ensure_cache_available( + request, num_computed_tokens + ) + + def update_state_after_alloc(self, request: "Request", index: int) -> None: + assert self.connector_scheduler is not None + self.connector_scheduler.update_state_after_alloc(request, index) + + def build_connector_meta( + self, scheduler_output: "SchedulerOutput" + ) -> ECCPUConnectorMetadata: + assert self.connector_scheduler is not None + return self.connector_scheduler.build_connector_meta(scheduler_output) + + def get_finished( + self, finished_req_ids: set[str] + ) -> tuple[set[str] | None, set[str] | None]: + if self.connector_worker is not None: + self.connector_worker.flush_saves() + return None, None + + # Shared. + def shutdown(self) -> None: + if self.connector_scheduler is not None: + self.connector_scheduler.shutdown() + if self.connector_worker is not None: + self.connector_worker.shutdown() diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/ec_shared_region.py b/vllm/distributed/ec_transfer/ec_connector/cpu/ec_shared_region.py new file mode 100644 index 0000000000..093292142e --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/ec_shared_region.py @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Lightweight mmap-backed shared memory region for encoder cache (EC) data. + +Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified +for EC: flat shared layout, no multi-tensor cursor, no block_size_factor. +""" + +import mmap +import os +import time + +import torch + +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0): + """Spin-wait until the file reaches expected_size (creator truncated it).""" + deadline = time.monotonic() + timeout + while True: + if os.fstat(fd).st_size >= expected_size: + return + if time.monotonic() > deadline: + raise TimeoutError( + f"Timed out waiting for EC mmap file to reach {expected_size} bytes" + ) + time.sleep(0.005) + + +class ECSharedRegion: + """Flat mmap-backed memory region shared across TP workers for + encoder cache blocks. + + Layout: (num_blocks, block_size_bytes) — contiguous, no per-worker + interleaving. All workers map the same file and see identical data. + + File path: /dev/shm/vllm_ec_{engine_id}.mmap + + This class owns only the shared memory substrate (mmap lifecycle, the + `blocks` view, CUDA host registration). Block allocation and eviction + are tracked by `EmbeddingCache` in the scheduler process. + """ + + def __init__( + self, + engine_id: str, + num_blocks: int, + block_size_bytes: int, + ) -> None: + self.num_blocks = num_blocks + self.block_size_bytes = block_size_bytes + + total_size_bytes = num_blocks * block_size_bytes + # Path in /dev/shm (tmpfs); unique per engine instance. + self._mmap_path = f"/dev/shm/vllm_ec_{engine_id}.mmap" + # True for the process that created the file (responsible for unlink). + self._is_creator = False + # True after successful cudaHostRegister (cleanup must unregister). + self._is_pinned = False + + # File descriptor for the shared memory backing file. + try: + self._fd: int | None = os.open( + self._mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600 + ) + os.ftruncate(self._fd, total_size_bytes) + self._is_creator = True + logger.info( + "Created EC mmap file %s (%.2f MiB)", + self._mmap_path, + total_size_bytes / (1 << 20), + ) + except FileExistsError: + self._fd = os.open(self._mmap_path, os.O_RDWR) + try: + _wait_for_file_size(self._fd, total_size_bytes) + except Exception: + os.close(self._fd) + self._fd = None + raise + logger.info("Opened existing EC mmap file %s", self._mmap_path) + + # MAP_SHARED mmap over _fd; all processes see the same pages. + self._mmap_obj: mmap.mmap | None = mmap.mmap( + self._fd, + total_size_bytes, + flags=mmap.MAP_SHARED, + prot=mmap.PROT_READ | mmap.PROT_WRITE, + ) + + if self._is_creator: + _MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23) + self._mmap_obj.madvise(_MADV_POPULATE_WRITE, 0, total_size_bytes) + + # (num_blocks, block_size_bytes) int8 tensor over the mmap buffer. + self.blocks: torch.Tensor = torch.frombuffer( + memoryview(self._mmap_obj), dtype=torch.int8 + ).view(num_blocks, block_size_bytes) + # Cached for cudaHostRegister/Unregister and pointer math. + self._blocks_ptr: int = self.blocks.data_ptr() + self._blocks_nbytes: int = self.blocks.nbytes + + def pin_memory(self) -> None: + """Register the entire mmap as CUDA pinned memory for fast DMA. + + Each TP worker process owns its own virtual address for the same + shared physical pages, so every process must register independently. + No-op when CUDA is not available or already pinned. + """ + if self._is_pinned or not torch.cuda.is_available(): + return + result = torch.cuda.cudart().cudaHostRegister( + self._blocks_ptr, self._blocks_nbytes, 0 + ) + if result.value != 0: + logger.warning( + "cudaHostRegister failed (code=%d) — " + "transfers will still work but may be slower (unpinned DMA)", + result.value, + ) + else: + logger.debug("cudaHostRegister %.2f MB", self._blocks_nbytes / 1e6) + self._is_pinned = True + + def cleanup(self) -> None: + """Tear down the region. Lifecycle method; no concurrent access.""" + logger.info("Starting ECSharedRegion cleanup...") + if self._is_creator: + try: + os.unlink(self._mmap_path) + logger.info("Removed EC mmap file %s", self._mmap_path) + except Exception: + logger.warning( + "Failed to unlink path %s", self._mmap_path, exc_info=True + ) + self._is_creator = False + if self._is_pinned: + result = torch.cuda.cudart().cudaHostUnregister(self._blocks_ptr) + if result.value != 0: + logger.warning("cudaHostUnregister failed (code=%d)", result) + self._is_pinned = False + if hasattr(self, "blocks"): + del self.blocks + if self._mmap_obj: + try: + self._mmap_obj.close() + except Exception: + logger.warning("Failed to close mmap_obj", exc_info=True) + self._mmap_obj = None + if self._fd is not None: + try: + os.close(self._fd) + except Exception: + logger.warning("Failed to close fd %s", self._fd, exc_info=True) + self._fd = None diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py new file mode 100644 index 0000000000..132efc1df0 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ECCPUScheduler — CPU offload scheduler delegate. + +Owns the mmap region and the embedding cache, and handles the producer +(GPU->CPU offload) and consumer (CPU->GPU reload) scheduler-side logic +for the ECCPUConnector. +""" + +from typing import TYPE_CHECKING + +from vllm.distributed.ec_transfer.ec_connector.cpu.common import ( + ECCPUConnectorMetadata, + create_ec_shared_region, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import ( + EmbeddingCache, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import ( + StepTracker, +) +from vllm.logger import init_logger + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.v1.core.sched.output import SchedulerOutput + from vllm.v1.request import Request + +logger = init_logger(__name__) + + +class ECCPUScheduler: + """Scheduler delegate for the ECCPUConnector.""" + + def __init__(self, vllm_config: "VllmConfig") -> None: + ec_config = vllm_config.ec_transfer_config + assert ec_config is not None + self._is_producer: bool = ec_config.is_ec_producer + self._is_consumer: bool = ec_config.is_ec_consumer + + self._region = create_ec_shared_region(vllm_config) + # Block allocator + LRU eviction policy for the shared region. + self._cache = EmbeddingCache(self._region.num_blocks) + + max_batches = vllm_config.max_concurrent_batches + # Delays mark_ready until the GPU→mmap DMA is guaranteed complete. + self._ready_tracker = StepTracker(max_batches) + # Delays unpin until the mmap→GPU DMA is guaranteed complete. + self._unpin_tracker = StepTracker(max_batches) + + # mm_hash → block IDs allocated this step for GPU→mmap saves. + self._pending_saves: dict[str, list[int]] = {} + # mm_hash → block IDs to load from mmap→GPU this step. + self._pending_loads: dict[str, list[int]] = {} + + def has_cache_item(self, identifier: str) -> bool: + if not self._is_consumer: + return False + entry = self._cache.get(identifier) + return entry is not None and entry.ready + + def ensure_cache_available( + self, request: "Request", num_computed_tokens: int + ) -> bool: + return True + + def update_state_after_alloc(self, request: "Request", index: int) -> None: + feature = request.mm_features[index] + mm_hash = feature.identifier + + if self._is_producer and self._cache.get(mm_hash) is None: + entry = self._cache.alloc(mm_hash, feature.mm_position.length) + if entry is not None: + self._pending_saves[mm_hash] = list(entry.block_ids) + self._ready_tracker.add(mm_hash, request.request_id) + + if self._is_consumer and mm_hash not in self._pending_loads: + entry = self._cache.get(mm_hash) + if entry is not None and entry.ready: + self._cache.pin(mm_hash) + self._pending_loads[mm_hash] = list(entry.block_ids) + self._unpin_tracker.add(mm_hash, request.request_id) + + def build_connector_meta( + self, scheduler_output: "SchedulerOutput" + ) -> ECCPUConnectorMetadata: + finished = ( + scheduler_output.finished_req_ids if scheduler_output is not None else set() + ) + + for key in self._ready_tracker.step(finished): + entry = self._cache.get(key) + if entry is not None and not entry.ready: + self._cache.mark_ready(key) + + for key in self._unpin_tracker.step(finished): + self._cache.unpin(key) + + meta = ECCPUConnectorMetadata() + if self._is_producer: + meta.saves = self._pending_saves + self._pending_saves = {} + if self._is_consumer: + meta.loads = self._pending_loads + self._pending_loads = {} + return meta + + def shutdown(self) -> None: + # drain_all() covers both entries still in _current (never + # consumed by build_connector_meta) and entries in slots. + self._pending_loads.clear() + for mm_hash in self._unpin_tracker.drain_all(): + self._cache.unpin(mm_hash) + self._ready_tracker.drain_all() + + self._is_producer = False + self._is_consumer = False + + try: + self._region.cleanup() + except Exception: + logger.debug("ec: region cleanup failed", exc_info=True) diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/embedding_cache.py b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/embedding_cache.py new file mode 100644 index 0000000000..8cd5f5c8e7 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/embedding_cache.py @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""EmbeddingCache — named-entry block cache with FIFO eviction. + +Manages a fixed pool of block IDs keyed by content identity (mm_hash). +Entries transition through: not-ready → ready (evictable) → pinned. +Eviction targets ready + unpinned entries in FIFO order. + +All public methods are thread-safe. +""" + +import threading +from collections import OrderedDict + + +class CacheEntry: + """A single cache entry. Read `.block_ids` and `.ready` freely; + mutations only through EmbeddingCache methods.""" + + __slots__ = ("block_ids", "_pin_count") + + def __init__(self, block_ids: tuple[int, ...]) -> None: + self.block_ids = block_ids + self._pin_count = -1 # not ready + + @property + def ready(self) -> bool: + return self._pin_count >= 0 + + @property + def evictable(self) -> bool: + return self._pin_count == 0 + + def mark_ready(self): + self._pin_count = 0 + + def pin(self): + self._pin_count += 1 + + +class EmbeddingCache: + """Fixed-size block cache with FIFO eviction. + + Entries are keyed by content identity (e.g. mm_hash). Blocks are + allocated from a free-set; when space is needed, ready + unpinned + entries are evicted oldest-first. + + The caller is responsible for deciding *when* to call ``mark_ready`` + (e.g. after enough engine steps have elapsed for the worker to have + completed the write). + """ + + def __init__(self, num_blocks: int) -> None: + self._num_blocks = num_blocks + self._lock = threading.Lock() + # Available block IDs, LIFO stack. + self._free_blocks: list[int] = list(range(num_blocks)) + # Key → live entry (all states: not-ready, ready, pinned). + self._entries: dict[str, CacheEntry] = {} + # Ready + unpinned entries in insertion order (FIFO eviction). + self._entries_free_list: OrderedDict[str, None] = OrderedDict() + # Total blocks reclaimable via eviction (fail-fast check). + self._evictable_block_count: int = 0 + + def get(self, key: str) -> CacheEntry | None: + """Return the entry for *key*, or None if not present.""" + with self._lock: + return self._entries.get(key) + + def alloc(self, key: str, n_blocks: int) -> CacheEntry | None: + """Allocate *n_blocks* for *key*, evicting as needed. + + The entry starts not-ready. Returns None if there is not enough + space even after evicting all evictable entries. + """ + with self._lock: + assert key not in self._entries, ( + f"EmbeddingCache: duplicate alloc for {key!r}" + ) + assert n_blocks <= self._num_blocks, ( + f"EmbeddingCache: {n_blocks} blocks requested but capacity " + f"is {self._num_blocks}" + ) + if len(self._free_blocks) + self._evictable_block_count < n_blocks: + return None + self._evict_until(n_blocks) + block_ids = tuple(self._free_blocks.pop() for _ in range(n_blocks)) + entry = CacheEntry(block_ids) + self._entries[key] = entry + return entry + + def mark_ready(self, key: str) -> None: + """Mark an entry as ready (data is CPU-visible).""" + with self._lock: + entry = self._entries[key] + assert entry._pin_count == -1, ( + f"EmbeddingCache: mark_ready on already-ready entry {key!r}" + ) + entry.mark_ready() + self._entries_free_list[key] = None + self._evictable_block_count += len(entry.block_ids) + + def pin(self, key: str) -> None: + """Pin an entry (prevent eviction).""" + with self._lock: + entry = self._entries[key] + assert entry.ready, f"EmbeddingCache: pin of not-ready entry {key!r}" + if entry.evictable: + del self._entries_free_list[key] + self._evictable_block_count -= len(entry.block_ids) + entry.pin() + + def unpin(self, key: str) -> None: + """Unpin an entry. Asserts currently pinned.""" + with self._lock: + entry = self._entries[key] + assert entry._pin_count > 0, ( + f"EmbeddingCache: unpin of unpinned entry {key!r}" + ) + entry._pin_count -= 1 + if entry._pin_count == 0: + self._entries_free_list[key] = None + self._evictable_block_count += len(entry.block_ids) + + def _evict_until(self, n_blocks: int) -> None: + """Evict ready+unpinned entries FIFO until enough space. Lock held.""" + while len(self._free_blocks) < n_blocks: + key, _ = self._entries_free_list.popitem(last=False) + entry = self._entries.pop(key) + self._evictable_block_count -= len(entry.block_ids) + self._free_blocks.extend(entry.block_ids) diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/step_tracker.py b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/step_tracker.py new file mode 100644 index 0000000000..9ef7e87a7b --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/step_tracker.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""StepTracker — step-counting delay with first-finish fast-path. + +Manages a collection of (mm_hash, request_id) entries that need to be +processed after a configurable number of engine steps have elapsed. +Additionally, if the originating request finishes before the step-count +expires, the entry is processed immediately (the GPU transfer is +guaranteed complete by request finish time). + +Used by ECCPUScheduler: one instance for readiness marking, one for +unpins. +""" + +from collections import deque +from dataclasses import dataclass + + +@dataclass(slots=True) +class _PendingEntry: + mm_hash: str + request_id: str + processed: bool = False + + +class StepTracker: + """Track pending operations with step-count delay and first-finish. + + Each ``add(mm_hash, request_id)`` registers an operation that will be + returned from ``step()`` either when: + (a) ``max_concurrent_batches`` steps elapse (deque expiry), OR + (b) ``request_id`` appears in ``finished_req_ids`` (first-finish). + + Each add is processed exactly once — never both (a) and (b). + """ + + def __init__(self, max_concurrent_batches: int) -> None: + # One slot per past step; oldest at the right. When the deque is + # full the rightmost slot is popped and its entries expire. + self._slots: deque[list[_PendingEntry]] = deque(maxlen=max_concurrent_batches) + # Reverse index: request_id → entries across all slots + _current. + # Enables O(1) lookup for the first-finish fast-path. + self._req_index: dict[str, list[_PendingEntry]] = {} + # Entries added this step (via add()), committed to a new slot at + # the start of the next step() call. + self._current: list[_PendingEntry] = [] + + def add(self, mm_hash: str, request_id: str) -> None: + """Register an mm_hash needing processing after a delay.""" + entry = _PendingEntry(mm_hash=mm_hash, request_id=request_id) + self._current.append(entry) + self._req_index.setdefault(request_id, []).append(entry) + + def step(self, finished_req_ids: set[str]) -> list[str]: + """Advance one step. Return mm_hashes ready for processing. + + Returns: + List of mm_hashes to process. May contain duplicates if the + same mm_hash was added multiple times (from different requests). + """ + result: list[str] = [] + + # Phase 1: first-finish fast-path. + for req_id in finished_req_ids: + entries = self._req_index.pop(req_id, None) + if entries is None: + continue + for entry in entries: + if not entry.processed: + entry.processed = True + result.append(entry.mm_hash) + + # Phase 2: step-count expiry. + if len(self._slots) == self._slots.maxlen: + expired_slot = self._slots.pop() + for entry in expired_slot: + if not entry.processed: + entry.processed = True + result.append(entry.mm_hash) + self._cleanup_expired(expired_slot) + + # Phase 3: commit current entries to new slot. + self._slots.appendleft(self._current) + self._current = [] + + return result + + def drain_all(self) -> list[str]: + """Return all unprocessed entries and clear internal state.""" + result: list[str] = [] + for entry in self._current: + if not entry.processed: + entry.processed = True + result.append(entry.mm_hash) + self._current = [] + + for slot in self._slots: + for entry in slot: + if not entry.processed: + entry.processed = True + result.append(entry.mm_hash) + self._slots.clear() + self._req_index.clear() + return result + + def _cleanup_expired(self, slot: list[_PendingEntry]) -> None: + """Prune reverse index for entries in an expired slot.""" + for entry in slot: + entries_for_req = self._req_index.get(entry.request_id) + if entries_for_req is None: + # Already removed by fast-path (pop in step()). + continue + entries_for_req.remove(entry) + if not entries_for_req: + del self._req_index[entry.request_id] diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py new file mode 100644 index 0000000000..b923a2218d --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py @@ -0,0 +1,202 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Worker-side of the ECCPUConnector. + +Thin, stateless across steps: opens the shared mmap region and uses the +per-step connector metadata (`ECCPUConnectorMetadata`) to decide which +blocks to copy in each direction. +""" + +from typing import TYPE_CHECKING + +import torch + +from vllm._custom_ops import swap_blocks_batch +from vllm.distributed.ec_transfer.ec_connector.cpu.common import ( + ECCPUConnectorMetadata, + create_ec_shared_region, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.worker.descriptor_buffers import ( + DescriptorBufferPool, + DescriptorBuffers, +) +from vllm.distributed.parallel_state import ( + get_pcp_group, + get_tensor_model_parallel_rank, +) +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.utils.platform_utils import is_pin_memory_available + +if TYPE_CHECKING: + from vllm.config import VllmConfig + +logger = init_logger(__name__) + + +class ECCPUWorker: + """Worker-side delegate for the ECCPUConnector. + + - Producer role: copies `encoder_cache[mm_hash]` → `mmap[block_ids]` + for each entry in `metadata.saves`. Descriptor buffers are filled + directly in `save_caches`; the actual DMA is issued as a single + batched call in `flush_saves`. + - Consumer role: copies `mmap[block_ids]` → `encoder_cache[mm_hash]` + for all entries in `metadata.loads` via a single `swap_blocks_batch` + call on the load stream. + - On `ec_both` nodes both paths run back-to-back in a single step. + """ + + def __init__(self, vllm_config: "VllmConfig") -> None: + self._region = create_ec_shared_region(vllm_config) + # Model dtype; used to reinterpret raw int8 blocks on load. + self._dtype = vllm_config.model_config.dtype + + if is_pin_memory_available(): + self._region.pin_memory() + + # All TP/PCP ranks hold identical encoder output. Only one rank + # per mmap needs to write — saves host memory bandwidth. + # DCP is a subdivision of TP, so tp_rank==0 covers it. + self._is_save_rank = ( + get_tensor_model_parallel_rank() == 0 and get_pcp_group().rank_in_group == 0 + ) + + # Dedicated stream for async mmap→GPU loads (overlaps with compute). + self._load_stream = current_platform.Stream() + + # Descriptor buffer pool (recycled across steps, shared by both paths). + self._buf_pool = DescriptorBufferPool() + + # Active save buffer being filled during save_caches calls this step. + self._save_bufs: DescriptorBuffers | None = None + self._save_count: int = 0 + + def save_caches( + self, + encoder_cache: dict[str, torch.Tensor], + mm_hash: str, + connector_metadata: ECCPUConnectorMetadata, + ) -> None: + """Fill descriptor buffers directly for batched flush.""" + if not self._is_save_rank: + return + block_ids = connector_metadata.saves.get(mm_hash) + if block_ids is None: + return + + src = encoder_cache[mm_hash] + total_bytes = src.numel() * src.element_size() + block_size = self._region.block_size_bytes + allocated_bytes = len(block_ids) * block_size + assert total_bytes <= allocated_bytes, ( + f"EC: encoder output exceeds allocated blocks for mm_hash={mm_hash}: " + f"{total_bytes} bytes but only {allocated_bytes} allocated " + f"({len(block_ids)} blocks × {block_size}). " + f"shape={list(src.shape)}" + ) + + if self._save_bufs is None: + total = sum(len(v) for v in connector_metadata.saves.values()) + self._save_bufs = self._buf_pool.acquire(total) + + assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel() + + src_ptrs, dst_ptrs, sizes = self._save_bufs + src_base = src.view(-1).view(torch.uint8).data_ptr() + dst_base = self._region.blocks.data_ptr() + idx = self._save_count + + for i, block_idx in enumerate(block_ids): + start = i * block_size + src_ptrs[idx] = src_base + start + dst_ptrs[idx] = dst_base + block_idx * block_size + sizes[idx] = min(block_size, total_bytes - start) + idx += 1 + + self._save_count = idx + + def flush_saves(self) -> None: + """Flush all accumulated saves in a single swap_blocks_batch call.""" + if self._save_count == 0: + return + + bufs = self._save_bufs + assert bufs is not None + src_ptrs, dst_ptrs, sizes = bufs + n = self._save_count + swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n]) + + self._buf_pool.release(bufs) + self._save_bufs = None + self._save_count = 0 + + def start_load_caches( + self, + encoder_cache: dict[str, torch.Tensor], + connector_metadata: ECCPUConnectorMetadata, + ) -> None: + """Consumer path: single batched copy of all loads from mmap→GPU.""" + if not connector_metadata.loads: + return + + block_size = self._region.block_size_bytes + blocks = self._region.blocks + dtype = self._dtype + device_type = current_platform.device_type + src_base = blocks.data_ptr() + + # Pre-filter: only hashes not already in encoder_cache. + load_items = { + h: idxs + for h, idxs in connector_metadata.loads.items() + if h not in encoder_cache + } + if not load_items: + return + + total_blocks = sum(len(idxs) for idxs in load_items.values()) + + with current_platform.stream(self._load_stream): + # Single contiguous destination buffer for all loads. + dst_buf = torch.empty( + total_blocks, block_size, dtype=torch.int8, device=device_type + ) + dst_buf_base = dst_buf.data_ptr() + + bufs = self._buf_pool.acquire(total_blocks) + src_ptrs = bufs.src_ptrs[:total_blocks] + dst_ptrs = bufs.dst_ptrs[:total_blocks] + sizes = bufs.sizes[:total_blocks] + sizes[:] = block_size + + op_idx = 0 + for block_ids in load_items.values(): + for block_idx in block_ids: + src_ptrs[op_idx] = src_base + block_idx * block_size + dst_ptrs[op_idx] = dst_buf_base + op_idx * block_size + op_idx += 1 + + swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True) + + self._buf_pool.release(bufs) + + # Slice contiguous buffer into per-hash views. + offset = 0 + for mm_hash, block_ids in load_items.items(): + n = len(block_ids) + encoder_cache[mm_hash] = ( + dst_buf[offset : offset + n].view(dtype).reshape(n, -1) + ) + offset += n + + current_platform.current_stream().wait_stream(self._load_stream) + + def shutdown(self) -> None: + self._load_stream.synchronize() + self._save_bufs = None + self._save_count = 0 + try: + self._region.cleanup() + except Exception: + logger.debug("EC: worker region cleanup failed", exc_info=True) diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/worker/descriptor_buffers.py b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/descriptor_buffers.py new file mode 100644 index 0000000000..a75d99ee81 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/descriptor_buffers.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Reusable pool of (src_ptrs, dst_ptrs, sizes) int64 tensor triples. + +Used by ECCPUWorker to batch swap_blocks_batch descriptors without +per-step allocation overhead. +""" + +from typing import NamedTuple + +import torch + + +class DescriptorBuffers(NamedTuple): + src_ptrs: torch.Tensor + dst_ptrs: torch.Tensor + sizes: torch.Tensor + + +class DescriptorBufferPool: + """Pool of descriptor buffer triples for swap_blocks_batch. + + Each buffer is a `DescriptorBuffers` namedtuple of three 1-D int64 + tensors of equal length. Buffers are recycled across steps; if a + returned buffer is too small it is discarded and a fresh one allocated. + """ + + def __init__(self) -> None: + # LIFO stack of idle buffer triples. + self._pool: list[DescriptorBuffers] = [] + + def acquire(self, n: int) -> DescriptorBuffers: + """Get a buffer triple with capacity >= *n*.""" + if self._pool: + bufs = self._pool.pop() + if bufs.src_ptrs.numel() >= n: + return bufs + return DescriptorBuffers( + torch.empty(n, dtype=torch.int64), + torch.empty(n, dtype=torch.int64), + torch.empty(n, dtype=torch.int64), + ) + + def release(self, bufs: DescriptorBuffers) -> None: + """Return a buffer triple to the pool for reuse.""" + self._pool.append(bufs) diff --git a/vllm/distributed/ec_transfer/ec_connector/factory.py b/vllm/distributed/ec_transfer/ec_connector/factory.py index 32f36ffbb1..598b1b22e4 100644 --- a/vllm/distributed/ec_transfer/ec_connector/factory.py +++ b/vllm/distributed/ec_transfer/ec_connector/factory.py @@ -83,3 +83,9 @@ ECConnectorFactory.register_connector( "vllm.distributed.ec_transfer.ec_connector.example_connector", "ECExampleConnector", ) + +ECConnectorFactory.register_connector( + "ECCPUConnector", + "vllm.distributed.ec_transfer.ec_connector.cpu.connector", + "ECCPUConnector", +)