[EC Connector] CPU Offloading EC Connector (#47423)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Signed-off-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -0,0 +1,306 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for EmbeddingCache.
|
||||
|
||||
Pure integer bookkeeping — no mmap, torch, or CUDA — so these run anywhere.
|
||||
"""
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
|
||||
|
||||
def _cache(num_blocks: int = 8) -> EmbeddingCache:
|
||||
return EmbeddingCache(num_blocks)
|
||||
|
||||
|
||||
# ── alloc ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_alloc_returns_unique_block_ids():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 3)
|
||||
assert entry is not None
|
||||
assert len(entry.block_ids) == 3
|
||||
assert len(set(entry.block_ids)) == 3
|
||||
assert all(0 <= i < 8 for i in entry.block_ids)
|
||||
|
||||
|
||||
def test_alloc_returns_tuple():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 2)
|
||||
assert isinstance(entry.block_ids, tuple)
|
||||
|
||||
|
||||
def test_alloc_returns_none_when_full():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 4)
|
||||
assert cache.alloc("b", 1) is None
|
||||
|
||||
|
||||
def test_alloc_asserts_duplicate_key():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
with pytest.raises(AssertionError, match="duplicate alloc"):
|
||||
cache.alloc("a", 2)
|
||||
|
||||
|
||||
def test_alloc_asserts_n_blocks_exceeds_capacity():
|
||||
cache = _cache(num_blocks=4)
|
||||
with pytest.raises(AssertionError, match="capacity"):
|
||||
cache.alloc("a", 5)
|
||||
|
||||
|
||||
def test_alloc_zero_blocks():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 0)
|
||||
assert entry is not None
|
||||
assert entry.block_ids == ()
|
||||
|
||||
|
||||
def test_alloc_starts_not_ready():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 2)
|
||||
assert not entry.ready
|
||||
assert not entry.evictable
|
||||
|
||||
|
||||
# ── get ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_get_missing_returns_none():
|
||||
cache = _cache()
|
||||
assert cache.get("a") is None
|
||||
|
||||
|
||||
def test_get_returns_entry():
|
||||
cache = _cache()
|
||||
alloc_entry = cache.alloc("a", 2)
|
||||
got = cache.get("a")
|
||||
assert got is alloc_entry
|
||||
|
||||
|
||||
def test_get_returns_not_ready_entry():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
entry = cache.get("a")
|
||||
assert entry is not None
|
||||
assert not entry.ready
|
||||
|
||||
|
||||
# ── mark_ready ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_mark_ready_transitions():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
entry = cache.get("a")
|
||||
assert not entry.ready
|
||||
cache.mark_ready("a")
|
||||
assert entry.ready
|
||||
assert entry.evictable
|
||||
|
||||
|
||||
def test_mark_ready_asserts_already_ready():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
with pytest.raises(AssertionError, match="already-ready"):
|
||||
cache.mark_ready("a")
|
||||
|
||||
|
||||
def test_mark_ready_asserts_missing_key():
|
||||
cache = _cache()
|
||||
with pytest.raises(KeyError):
|
||||
cache.mark_ready("nonexistent")
|
||||
|
||||
|
||||
# ── pin / unpin ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_pin_asserts_not_ready():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
with pytest.raises(AssertionError, match="not-ready"):
|
||||
cache.pin("a")
|
||||
|
||||
|
||||
def test_pin_unpin_basic():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
entry = cache.get("a")
|
||||
assert not entry.evictable
|
||||
cache.unpin("a")
|
||||
assert entry.evictable
|
||||
|
||||
|
||||
def test_nested_pin():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
cache.pin("a")
|
||||
cache.unpin("a")
|
||||
entry = cache.get("a")
|
||||
assert not entry.evictable # still pinned (count=1)
|
||||
cache.unpin("a")
|
||||
assert entry.evictable
|
||||
|
||||
|
||||
def test_unpin_asserts_on_unpinned():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
with pytest.raises(AssertionError, match="unpinned"):
|
||||
cache.unpin("a")
|
||||
|
||||
|
||||
# ── eviction ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_eviction_frees_ready_unpinned_fifo():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
# Pool full. Alloc should evict "a" (oldest).
|
||||
entry_c = cache.alloc("c", 2)
|
||||
assert entry_c is not None
|
||||
assert cache.get("a") is None # evicted
|
||||
assert cache.get("b") is not None # still there
|
||||
|
||||
|
||||
def test_eviction_skips_pinned():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
# "a" is pinned, so "b" should be evicted instead.
|
||||
entry_c = cache.alloc("c", 2)
|
||||
assert entry_c is not None
|
||||
assert cache.get("a") is not None # pinned, survived
|
||||
assert cache.get("b") is None # evicted
|
||||
cache.unpin("a")
|
||||
|
||||
|
||||
def test_eviction_skips_not_ready():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2) # not ready
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
# "a" is not ready (non-evictable), "b" is evictable.
|
||||
entry_c = cache.alloc("c", 2)
|
||||
assert entry_c is not None
|
||||
assert cache.get("a") is not None # not ready, survived
|
||||
assert cache.get("b") is None # evicted
|
||||
cache.mark_ready("a") # clean up
|
||||
|
||||
|
||||
def test_eviction_returns_none_if_all_non_evictable():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
cache.alloc("b", 2) # not ready
|
||||
# Both entries are non-evictable (a=pinned, b=not-ready).
|
||||
assert cache.alloc("c", 1) is None
|
||||
# Fail-fast: nothing was evicted.
|
||||
assert cache.get("a") is not None
|
||||
assert cache.get("b") is not None
|
||||
cache.unpin("a")
|
||||
|
||||
|
||||
def test_fail_fast_no_wasted_eviction():
|
||||
cache = _cache(num_blocks=6)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a") # evictable: 2 blocks
|
||||
cache.alloc("b", 2) # not ready
|
||||
cache.alloc("c", 2) # not ready
|
||||
# free=0, evictable=2, need 3 -> cannot succeed
|
||||
assert cache.alloc("d", 3) is None
|
||||
# "a" was NOT evicted despite being evictable
|
||||
assert cache.get("a") is not None
|
||||
|
||||
|
||||
def test_eviction_multiple_entries():
|
||||
cache = _cache(num_blocks=6)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
cache.alloc("c", 2)
|
||||
cache.mark_ready("c")
|
||||
# Need 4 blocks: must evict both "a" and "b".
|
||||
entry_d = cache.alloc("d", 4)
|
||||
assert entry_d is not None
|
||||
assert cache.get("a") is None
|
||||
assert cache.get("b") is None
|
||||
assert cache.get("c") is not None
|
||||
|
||||
|
||||
# ── thread safety ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_concurrent_pin_unpin():
|
||||
cache = _cache(num_blocks=8)
|
||||
cache.alloc("a", 4)
|
||||
cache.mark_ready("a")
|
||||
errors: list[Exception] = []
|
||||
barrier = threading.Barrier(8)
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
try:
|
||||
for _ in range(500):
|
||||
cache.pin("a")
|
||||
cache.unpin("a")
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(8)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert not errors, errors
|
||||
entry = cache.get("a")
|
||||
assert entry.evictable # all pins balanced
|
||||
|
||||
|
||||
def test_concurrent_alloc_evict():
|
||||
cache = _cache(num_blocks=4)
|
||||
errors: list[Exception] = []
|
||||
barrier = threading.Barrier(4)
|
||||
counter = {"i": 0}
|
||||
counter_lock = threading.Lock()
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
try:
|
||||
for _ in range(200):
|
||||
with counter_lock:
|
||||
key = f"k{counter['i']}"
|
||||
counter["i"] += 1
|
||||
entry = cache.alloc(key, 1)
|
||||
if entry is not None:
|
||||
cache.mark_ready(key)
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(4)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert not errors, errors
|
||||
@@ -0,0 +1,480 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import uuid
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
|
||||
|
||||
_N = 16
|
||||
_BS = 64
|
||||
|
||||
|
||||
class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset = offset
|
||||
self.length = length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1, identifier=None):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = identifier if identifier is not None else mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
|
||||
|
||||
class _Request:
|
||||
_counter = 0
|
||||
|
||||
def __init__(self, features, request_id=None):
|
||||
self.mm_features = features
|
||||
if request_id is None:
|
||||
_Request._counter += 1
|
||||
self.request_id = f"req_{_Request._counter}"
|
||||
else:
|
||||
self.request_id = request_id
|
||||
|
||||
|
||||
def _make_scheduler(
|
||||
monkeypatch,
|
||||
num_blocks=_N,
|
||||
max_concurrent_batches=1,
|
||||
*,
|
||||
is_producer=True,
|
||||
is_consumer=True,
|
||||
) -> ECCPUScheduler:
|
||||
region = ECSharedRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=num_blocks,
|
||||
block_size_bytes=_BS,
|
||||
)
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
|
||||
_is_prod = is_producer
|
||||
_is_cons = is_consumer
|
||||
_mcb = max_concurrent_batches
|
||||
|
||||
class _EC:
|
||||
is_ec_producer = _is_prod
|
||||
is_ec_consumer = _is_cons
|
||||
|
||||
class _Cfg:
|
||||
ec_transfer_config = _EC()
|
||||
|
||||
@property
|
||||
def max_concurrent_batches(self):
|
||||
return _mcb
|
||||
|
||||
return ECCPUScheduler(_Cfg())
|
||||
|
||||
|
||||
class _SchedulerOutput:
|
||||
def __init__(self, finished_req_ids=None):
|
||||
self.finished_req_ids = finished_req_ids or set()
|
||||
|
||||
|
||||
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
|
||||
"""Pre-populate a ready cache entry backed by real blocks."""
|
||||
s._cache.alloc(mm_hash, n_blocks)
|
||||
s._cache.mark_ready(mm_hash)
|
||||
|
||||
|
||||
def test_offload_reuse_cycle(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
# Step A: first sight — allocate and save.
|
||||
assert s.has_cache_item("h1") is False
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_a = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta_a.saves
|
||||
assert meta_a.loads == {}
|
||||
# Not ready yet (max_concurrent_batches=1 means ready next step).
|
||||
assert s.has_cache_item("h1") is False
|
||||
|
||||
# Step B: _step_readiness marks h1 ready (no new allocs).
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("h1") is True
|
||||
|
||||
# Step C: reload now works — entry is ready at update_state_after_alloc time.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_c = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta_c.loads
|
||||
assert meta_c.loads["h1"] == meta_a.saves["h1"]
|
||||
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_has_cache_item_false_when_not_consumer(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch, is_consumer=False)
|
||||
assert s.has_cache_item("anything") is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
|
||||
"""The connector must key the encoder cache on feature.identifier (what
|
||||
has_cache_item is called with), NOT feature.mm_hash."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "ENC_KEY" in meta.saves
|
||||
assert "PROC_KEY" not in meta.saves
|
||||
# Not ready yet (1 step delay).
|
||||
assert s.has_cache_item("ENC_KEY") is False
|
||||
# After another step it becomes ready — keyed on identifier.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("ENC_KEY") is True
|
||||
assert s.has_cache_item("PROC_KEY") is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_cpu_region_fifo_eviction(monkeypatch):
|
||||
# Region holds exactly 2 one-block encodings.
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
|
||||
# Fill with A and B, marking each ready.
|
||||
for h in ("A", "B"):
|
||||
s.update_state_after_alloc(_Request([_Feature(h, length=1)]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
# After the step above, A's step was called in B's build. A is ready.
|
||||
# B was submitted in the last build, so one more step makes it ready.
|
||||
s.build_connector_meta(scheduler_output=None) # marks B ready
|
||||
assert s.has_cache_item("A") is True
|
||||
assert s.has_cache_item("B") is True
|
||||
|
||||
# Allocate C: must evict A (oldest) to make room.
|
||||
s.update_state_after_alloc(_Request([_Feature("C", length=1)]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("A") is False # evicted
|
||||
assert s.has_cache_item("B") is True
|
||||
|
||||
|
||||
def test_multiple_mm_items_per_request(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("h1", length=1), _Feature("h2", length=1)])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
s.update_state_after_alloc(req, 1)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert set(meta.saves) == {"h1", "h2"}
|
||||
|
||||
# Step delay: mark both ready.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("h1") is True
|
||||
assert s.has_cache_item("h2") is True
|
||||
|
||||
# Now reload both.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
s.update_state_after_alloc(req, 1)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert set(meta2.loads) == {"h1", "h2"}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_load_returns_correct_block_ids(monkeypatch):
|
||||
"""meta.loads must contain the same block IDs that meta.saves
|
||||
allocated — verified through the public API only."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("a", length=2)])
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_save = s.build_connector_meta(scheduler_output=None)
|
||||
assert "a" in meta_save.saves
|
||||
|
||||
# Readiness delay.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Reload.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_load = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta_load.loads["a"] == meta_save.saves["a"]
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
_seed_cached(s, "b", n_blocks=2)
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "a" in meta.loads
|
||||
assert "b" not in meta.loads
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_repeated_reload_same_step_loads_once(monkeypatch):
|
||||
"""The same mm_hash requested twice in one step must appear in
|
||||
meta.loads exactly once."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert list(meta.loads.keys()).count("a") == 1
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_load_not_emitted_for_uncached_entry(monkeypatch):
|
||||
"""A feature that was never saved must not appear in meta.loads."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
s.update_state_after_alloc(_Request([_Feature("missing")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta.loads == {}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_delayed_unpin_protects_blocks_during_worker_read(monkeypatch):
|
||||
"""Blocks being loaded must not be evictable until the delayed unpin
|
||||
fires. Observable: a save that needs eviction cannot reclaim a
|
||||
still-pinned loaded entry."""
|
||||
# 2 blocks, both occupied by "a" (ready).
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
# Load "a" — pins it.
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "a" in meta.loads
|
||||
|
||||
# Try to save "b" (needs 2 blocks, but "a" is pinned) — must fail.
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" not in meta2.saves # can't evict pinned "a"
|
||||
|
||||
# After the delay, "a" unpins. Now "b" can evict it.
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta3 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" in meta3.saves
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_delayed_unpin_depth_2(monkeypatch):
|
||||
"""With max_concurrent_batches=2, loaded blocks stay pinned for 2 steps
|
||||
after build. Observable via eviction failure."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
# Load "a" → pinned.
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Step 1 after load: still pinned — can't evict for "b".
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta1 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" not in meta1.saves
|
||||
|
||||
# Step 2 after load: still pinned.
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" not in meta2.saves
|
||||
|
||||
# Step 3: unpin fires, now eviction works.
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta3 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" in meta3.saves
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_shutdown_unpins_pending_loads(monkeypatch):
|
||||
"""Shutdown must unpin entries still in _pending_loads (not yet drained
|
||||
to the unpin deque). Observable: after shutdown, the entry is evictable
|
||||
(alloc can reclaim its blocks)."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
# Don't call build_connector_meta — "a" stays in _pending_loads.
|
||||
s.shutdown()
|
||||
|
||||
# After shutdown, scheduler reports no items (consumer role is off).
|
||||
assert s.has_cache_item("a") is False
|
||||
|
||||
|
||||
def test_shutdown_unpins_deferred_unpin_queue(monkeypatch):
|
||||
"""Shutdown must unpin entries queued in the deferred unpin deque
|
||||
(already drained from _pending_loads but not yet expired)."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
# Load "a" and build — moves to unpin deque.
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# "a" is in the deferred unpin deque (depth=2, not yet expired).
|
||||
# Shutdown must unpin it.
|
||||
s.shutdown()
|
||||
assert s.has_cache_item("a") is False
|
||||
|
||||
|
||||
def test_eviction_skips_entry_pinned_by_pending_load(monkeypatch):
|
||||
"""A loaded entry still within the unpin delay window must survive
|
||||
eviction attempts from new saves."""
|
||||
# 2 blocks: A occupies both, ready.
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "A", n_blocks=2)
|
||||
|
||||
# Load A → pinned.
|
||||
s.update_state_after_alloc(_Request([_Feature("A")]), 0)
|
||||
meta_load = s.build_connector_meta(scheduler_output=None)
|
||||
assert "A" in meta_load.loads
|
||||
|
||||
# Try to save C (needs 2 blocks) — can't evict pinned A.
|
||||
s.update_state_after_alloc(_Request([_Feature("C", length=2)]), 0)
|
||||
meta_save = s.build_connector_meta(scheduler_output=None)
|
||||
assert "C" not in meta_save.saves
|
||||
assert s.has_cache_item("A") is True # survived
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_region_full_skips_save_and_never_blocks(monkeypatch):
|
||||
"""When the region is fully occupied by pinned entries, new saves are
|
||||
silently skipped and ensure_cache_available never blocks."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=1)
|
||||
_seed_cached(s, "pinned", n_blocks=1)
|
||||
s._cache.pin("pinned")
|
||||
|
||||
req = _Request([_Feature("new", length=1)])
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "new" not in meta.saves
|
||||
assert s.has_cache_item("new") is False
|
||||
|
||||
s._cache.unpin("pinned")
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_step_readiness_depth_2(monkeypatch):
|
||||
"""With max_concurrent_batches=2, entries need 2 steps to become ready."""
|
||||
s = _make_scheduler(monkeypatch, max_concurrent_batches=2)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_0 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta_0.saves
|
||||
assert s.has_cache_item("h1") is False
|
||||
|
||||
# One step later: still not ready (depth=2).
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("h1") is False
|
||||
|
||||
# Two steps later: now ready.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("h1") is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_producer_only_never_emits_loads(monkeypatch):
|
||||
"""A producer-only scheduler must never populate meta.loads, even when
|
||||
entries are ready."""
|
||||
s = _make_scheduler(monkeypatch, is_consumer=False)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta.saves
|
||||
assert meta.loads == {}
|
||||
|
||||
# Mark ready.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Even if we re-encounter the same feature, no load.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta2.loads == {}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_consumer_only_never_emits_saves(monkeypatch):
|
||||
"""A consumer-only scheduler must never populate meta.saves."""
|
||||
s = _make_scheduler(monkeypatch, is_producer=False)
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
|
||||
req = _Request([_Feature("a", length=1)])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta.saves == {}
|
||||
assert "a" in meta.loads
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_consumer_only_has_cache_item(monkeypatch):
|
||||
"""Consumer-only: has_cache_item works normally for ready entries."""
|
||||
s = _make_scheduler(monkeypatch, is_producer=False)
|
||||
assert s.has_cache_item("a") is False
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
assert s.has_cache_item("a") is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_save_not_emitted_for_already_cached_entry(monkeypatch):
|
||||
"""An entry already in the cache (from a prior save) must not trigger
|
||||
a second allocation or appear in meta.saves again."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
# First encounter → save.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta1 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta1.saves
|
||||
|
||||
# Second encounter (same step or later) → no save.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" not in meta2.saves
|
||||
s.shutdown()
|
||||
|
||||
|
||||
# ── first-finish fast-path (idle engine) ────────────────────────────────────
|
||||
|
||||
|
||||
def test_first_finish_marks_ready_immediately(monkeypatch):
|
||||
"""When the originating request finishes, the save entry becomes ready
|
||||
without waiting for the step-count delay."""
|
||||
s = _make_scheduler(monkeypatch, max_concurrent_batches=5)
|
||||
req = _Request([_Feature("h1", length=1)], request_id="req_x")
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("h1") is False
|
||||
|
||||
# Request finishes — readiness should fire immediately.
|
||||
s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_x"}))
|
||||
assert s.has_cache_item("h1") is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_first_finish_unpins_immediately(monkeypatch):
|
||||
"""When the originating request finishes, the load entry is unpinned
|
||||
without waiting for the step-count delay."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=5)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
req = _Request([_Feature("a")], request_id="req_y")
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "a" in meta.loads
|
||||
|
||||
# "a" is pinned. Try to evict — must fail.
|
||||
s.update_state_after_alloc(
|
||||
_Request([_Feature("b", length=2)], request_id="req_z"), 0
|
||||
)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" not in meta2.saves
|
||||
|
||||
# req_y finishes — unpin fires in build_connector_meta (fast-path).
|
||||
s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_y"}))
|
||||
|
||||
# Next step: "a" is now evictable, so "b" can be allocated.
|
||||
s.update_state_after_alloc(
|
||||
_Request([_Feature("b", length=2)], request_id="req_w"), 0
|
||||
)
|
||||
meta3 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" in meta3.saves
|
||||
s.shutdown()
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for StepTracker."""
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
|
||||
StepTracker,
|
||||
)
|
||||
|
||||
|
||||
def test_basic_step_count_expiry_depth_1():
|
||||
"""With max_concurrent_batches=1, entry expires after 1 step."""
|
||||
t = StepTracker(max_concurrent_batches=1)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
# Step 1: entry was added, committed to slot. Deque fills to maxlen=1,
|
||||
# but no expiry yet (the entry is in the slot just committed).
|
||||
result = t.step(set())
|
||||
assert result == []
|
||||
|
||||
# Step 2: deque is full, oldest slot (containing h1) expires.
|
||||
result = t.step(set())
|
||||
assert result == ["h1"]
|
||||
|
||||
|
||||
def test_basic_step_count_expiry_depth_2():
|
||||
"""With max_concurrent_batches=2, entry expires after 2 steps."""
|
||||
t = StepTracker(max_concurrent_batches=2)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
assert t.step(set()) == [] # step 1: committed
|
||||
assert t.step(set()) == [] # step 2: deque fills but h1 in newer slot
|
||||
assert t.step(set()) == ["h1"] # step 3: h1's slot is oldest, expires
|
||||
|
||||
|
||||
def test_first_finish_fast_path():
|
||||
"""Entry is returned immediately when request_id appears in finished."""
|
||||
t = StepTracker(max_concurrent_batches=3)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
result = t.step({"req_a"})
|
||||
assert result == ["h1"]
|
||||
|
||||
|
||||
def test_no_double_processing_fast_path_then_expiry():
|
||||
"""Entry fast-pathed should NOT fire again on deque expiry."""
|
||||
t = StepTracker(max_concurrent_batches=1)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
# Fast-path fires h1.
|
||||
result = t.step({"req_a"})
|
||||
assert result == ["h1"]
|
||||
|
||||
# Normal expiry would fire now, but h1 is already processed.
|
||||
result = t.step(set())
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_no_double_processing_expiry_then_fast_path():
|
||||
"""Entry expired via step-count should NOT fire again via fast-path."""
|
||||
t = StepTracker(max_concurrent_batches=1)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
t.step(set()) # commit
|
||||
result = t.step(set()) # expiry fires h1
|
||||
assert result == ["h1"]
|
||||
|
||||
# req_a finishes later — must not re-fire.
|
||||
result = t.step({"req_a"})
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_multiplicity_same_hash_different_requests():
|
||||
"""Two adds of the same mm_hash (different requests) → returned twice."""
|
||||
t = StepTracker(max_concurrent_batches=3)
|
||||
t.add("h1", "req_a")
|
||||
t.add("h1", "req_b")
|
||||
|
||||
# Both requests finish in the same step.
|
||||
result = t.step({"req_a", "req_b"})
|
||||
assert sorted(result) == ["h1", "h1"]
|
||||
|
||||
|
||||
def test_mixed_expiry_and_fast_path():
|
||||
"""Some entries expire, others fast-path, in the same step."""
|
||||
t = StepTracker(max_concurrent_batches=1)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
t.step(set()) # commit h1's slot
|
||||
|
||||
# Add h2 in the next step.
|
||||
t.add("h2", "req_b")
|
||||
|
||||
# This step: h1's slot expires AND req_b finishes.
|
||||
result = t.step({"req_b"})
|
||||
assert set(result) == {"h1", "h2"}
|
||||
|
||||
|
||||
def test_drain_all_returns_unprocessed():
|
||||
"""drain_all returns everything pending and clears state."""
|
||||
t = StepTracker(max_concurrent_batches=3)
|
||||
t.add("h1", "req_a")
|
||||
t.add("h2", "req_b")
|
||||
t.step(set()) # commit both
|
||||
t.add("h3", "req_c") # in _current, not yet committed
|
||||
|
||||
result = t.drain_all()
|
||||
assert set(result) == {"h1", "h2", "h3"}
|
||||
|
||||
# After drain, everything is empty.
|
||||
assert t.step(set()) == []
|
||||
assert t.drain_all() == []
|
||||
|
||||
|
||||
def test_drain_all_skips_already_processed():
|
||||
"""drain_all should not re-return entries already fast-pathed."""
|
||||
t = StepTracker(max_concurrent_batches=3)
|
||||
t.add("h1", "req_a")
|
||||
t.add("h2", "req_b")
|
||||
|
||||
# Fast-path h1.
|
||||
result = t.step({"req_a"})
|
||||
assert result == ["h1"]
|
||||
|
||||
# drain should only return h2.
|
||||
result = t.drain_all()
|
||||
assert result == ["h2"]
|
||||
|
||||
|
||||
def test_empty_step_returns_nothing():
|
||||
"""step() with nothing pending returns empty list."""
|
||||
t = StepTracker(max_concurrent_batches=2)
|
||||
assert t.step(set()) == []
|
||||
assert t.step({"req_x"}) == []
|
||||
|
||||
|
||||
def test_add_after_step_goes_to_next_slot():
|
||||
"""Entries added after a step go into the next slot, not the current."""
|
||||
t = StepTracker(max_concurrent_batches=1)
|
||||
|
||||
t.step(set()) # empty step
|
||||
t.add("h1", "req_a")
|
||||
|
||||
# h1 is in _current. Next step commits it.
|
||||
t.step(set())
|
||||
|
||||
# Now the deque is full (1 slot with h1). Next step expires it.
|
||||
result = t.step(set())
|
||||
assert result == ["h1"]
|
||||
|
||||
|
||||
def test_fast_path_on_current_entries():
|
||||
"""Entries still in _current (not yet committed) can be fast-pathed."""
|
||||
t = StepTracker(max_concurrent_batches=3)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
# req_a finishes in the same step h1 was added.
|
||||
result = t.step({"req_a"})
|
||||
assert result == ["h1"]
|
||||
|
||||
# Entry is committed to slot but marked processed — won't re-fire.
|
||||
for _ in range(5):
|
||||
assert t.step(set()) == []
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
|
||||
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
|
||||
|
||||
|
||||
def _cfg():
|
||||
ec = MagicMock()
|
||||
ec.is_ec_producer = True
|
||||
ec.is_ec_consumer = True
|
||||
cfg = MagicMock()
|
||||
cfg.ec_transfer_config = ec
|
||||
return cfg
|
||||
|
||||
|
||||
def test_scheduler_role_builds_only_scheduler(monkeypatch):
|
||||
fake_sched = MagicMock()
|
||||
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
|
||||
assert c.connector_scheduler is fake_sched
|
||||
assert c.connector_worker is None
|
||||
c.has_cache_item("x")
|
||||
fake_sched.has_cache_item.assert_called_once_with("x")
|
||||
|
||||
|
||||
def test_worker_role_builds_only_worker(monkeypatch):
|
||||
fake_worker = MagicMock()
|
||||
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: fake_worker)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.WORKER)
|
||||
assert c.connector_worker is fake_worker
|
||||
assert c.connector_scheduler is None
|
||||
|
||||
|
||||
def test_request_finished_inherited_noop(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock()
|
||||
)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
|
||||
assert c.request_finished(MagicMock()) == (False, None)
|
||||
|
||||
|
||||
def test_factory_registered():
|
||||
cls = ECConnectorFactory._registry["ECCPUConnector"]()
|
||||
assert cls is ECCPUConnector
|
||||
@@ -0,0 +1,170 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECSharedRegion (the mmap substrate)."""
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
_wait_for_file_size,
|
||||
)
|
||||
|
||||
|
||||
def _make_region(num_blocks: int = 8, block_size_bytes: int = 64) -> ECSharedRegion:
|
||||
return ECSharedRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=num_blocks,
|
||||
block_size_bytes=block_size_bytes,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def region() -> ECSharedRegion:
|
||||
r = _make_region()
|
||||
yield r
|
||||
r.cleanup()
|
||||
|
||||
|
||||
# ── mmap sharing between two instances ───────────────────────────────────────
|
||||
|
||||
|
||||
def test_second_instance_opens_existing_file_and_shares_memory():
|
||||
instance_id = str(uuid.uuid4())
|
||||
r1 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
try:
|
||||
r2 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
try:
|
||||
assert r1._is_creator
|
||||
assert not r2._is_creator
|
||||
# Both map the same physical pages; writes via r1 are visible via r2.
|
||||
r1.blocks[0, :4] = torch.tensor([10, 20, 30, 40], dtype=torch.int8)
|
||||
assert r2.blocks[0, :4].tolist() == [10, 20, 30, 40]
|
||||
finally:
|
||||
r2.cleanup()
|
||||
finally:
|
||||
r1.cleanup()
|
||||
|
||||
|
||||
def test_only_creator_unlinks_file_on_cleanup():
|
||||
"""Critical contract: if the non-creator unlinks, the creator's mmap path
|
||||
becomes a dangling backing file and a third opener would create a new one
|
||||
out from under the creator."""
|
||||
instance_id = str(uuid.uuid4())
|
||||
r1 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
path = r1._mmap_path
|
||||
r2 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
|
||||
# Non-creator goes away first — file must still be on disk for r1.
|
||||
r2.cleanup()
|
||||
assert os.path.exists(path), "non-creator cleanup must not unlink the file"
|
||||
|
||||
# Creator goes away — file is removed.
|
||||
r1.cleanup()
|
||||
assert not os.path.exists(path), "creator cleanup must unlink the file"
|
||||
|
||||
|
||||
# ── _wait_for_file_size ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_wait_for_file_size_returns_when_already_big_enough(tmp_path):
|
||||
"""The fast path: file already at expected size — return immediately."""
|
||||
p = tmp_path / "f.bin"
|
||||
p.write_bytes(b"\x00" * 128)
|
||||
fd = os.open(str(p), os.O_RDONLY)
|
||||
try:
|
||||
_wait_for_file_size(fd, expected_size=128, timeout=1.0) # must not raise
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def test_wait_for_file_size_times_out_when_file_stays_empty(tmp_path):
|
||||
p = tmp_path / "f.bin"
|
||||
p.write_bytes(b"")
|
||||
fd = os.open(str(p), os.O_RDONLY)
|
||||
try:
|
||||
with pytest.raises(TimeoutError):
|
||||
_wait_for_file_size(fd, expected_size=4096, timeout=0.05)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
# ── pin_memory ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_pin_memory_success_sets_flag(region):
|
||||
"""When cudaHostRegister returns 0, _is_pinned flips to True
|
||||
and cleanup will correspondingly call cudaHostUnregister."""
|
||||
fake_cudart = MagicMock()
|
||||
success = MagicMock()
|
||||
success.value = 0
|
||||
fake_cudart.cudaHostRegister.return_value = success
|
||||
fake_cudart.cudaHostUnregister.return_value = success
|
||||
|
||||
with (
|
||||
patch("torch.cuda.is_available", return_value=True),
|
||||
patch("torch.cuda.cudart", return_value=fake_cudart),
|
||||
):
|
||||
region.pin_memory()
|
||||
assert region._is_pinned is True
|
||||
# cleanup must pair with cudaHostUnregister exactly once.
|
||||
region.cleanup()
|
||||
fake_cudart.cudaHostUnregister.assert_called_once()
|
||||
|
||||
|
||||
def test_pin_memory_failure_leaves_flag_false():
|
||||
"""If cudaHostRegister fails (non-zero), don't pretend it succeeded —
|
||||
cleanup must NOT call cudaHostUnregister on memory we never registered."""
|
||||
r = _make_region()
|
||||
try:
|
||||
fake_cudart = MagicMock()
|
||||
fail = MagicMock()
|
||||
fail.value = 1 # non-zero == error
|
||||
fake_cudart.cudaHostRegister.return_value = fail
|
||||
|
||||
with (
|
||||
patch("torch.cuda.is_available", return_value=True),
|
||||
patch("torch.cuda.cudart", return_value=fake_cudart),
|
||||
):
|
||||
r.pin_memory()
|
||||
assert r._is_pinned is False
|
||||
# Now run cleanup and verify cudaHostUnregister was NOT called.
|
||||
r.cleanup()
|
||||
fake_cudart.cudaHostUnregister.assert_not_called()
|
||||
finally:
|
||||
r.cleanup()
|
||||
|
||||
|
||||
def test_pin_memory_noop_without_cuda(region):
|
||||
"""pin_memory is a no-op when CUDA is not available."""
|
||||
with patch("torch.cuda.is_available", return_value=False):
|
||||
region.pin_memory()
|
||||
assert region._is_pinned is False
|
||||
|
||||
|
||||
# ── cleanup idempotency ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_cleanup_is_idempotent(region):
|
||||
region.cleanup()
|
||||
region.cleanup() # fixture calls a third time — must not raise
|
||||
@@ -0,0 +1,598 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ECCPUWorker.
|
||||
|
||||
The byte-level tests exercise real CUDA stream/event coordination
|
||||
against a real ``ECSharedRegion`` mmap and are skipped on hosts without
|
||||
CUDA. The lifecycle tests don't fire any ``torch.cuda.*`` primitives and
|
||||
run anywhere.
|
||||
|
||||
Mocking policy
|
||||
--------------
|
||||
- ``create_ec_shared_region`` is patched to inject a real ``ECSharedRegion``
|
||||
(mmap backed) with deterministic small dimensions, so byte assertions
|
||||
hit the actual mmap. The real responsibility of the worker — block
|
||||
index → mmap byte mapping, stream coordination, dtype/shape — is
|
||||
exercised end to end against a real GPU.
|
||||
- ``is_pin_memory_available`` is patched per-test so the rank-gating
|
||||
path can be probed deterministically without depending on host caps.
|
||||
- The shutdown test bypasses ``__init__`` via ``object.__new__`` and
|
||||
injects a ``MagicMock`` region.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker
|
||||
|
||||
# ── shape constants ──────────────────────────────────────────────────────────
|
||||
|
||||
# hidden_dim=8, dtype=fp16 (2 bytes) → block_size_bytes = 16, so one row of
|
||||
# encoder output fits in exactly one block.
|
||||
_HIDDEN_DIM = 8
|
||||
_DTYPE = torch.float16
|
||||
_BLOCK_SIZE_BYTES = _HIDDEN_DIM * _DTYPE.itemsize
|
||||
_NUM_BLOCKS = 8
|
||||
|
||||
_requires_cuda = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="exercises real CUDA stream/event coordination in ECCPUWorker",
|
||||
)
|
||||
|
||||
|
||||
def _make_region() -> ECSharedRegion:
|
||||
"""Fresh region backed by a real per-test mmap file."""
|
||||
return ECSharedRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=_NUM_BLOCKS,
|
||||
block_size_bytes=_BLOCK_SIZE_BYTES,
|
||||
)
|
||||
|
||||
|
||||
def _vllm_config(rank: int = 0) -> Mock:
|
||||
cfg = Mock(spec=VllmConfig)
|
||||
cfg.parallel_config = Mock(spec=ParallelConfig)
|
||||
cfg.parallel_config.rank = rank
|
||||
cfg.model_config = Mock()
|
||||
cfg.model_config.dtype = _DTYPE
|
||||
return cfg
|
||||
|
||||
|
||||
def _meta(
|
||||
*, saves: dict | None = None, loads: dict | None = None
|
||||
) -> ECCPUConnectorMetadata:
|
||||
return ECCPUConnectorMetadata(saves=saves or {}, loads=loads or {})
|
||||
|
||||
|
||||
# ── fixtures ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_worker():
|
||||
"""Factory that builds an ``ECCPUWorker`` over a real ``ECSharedRegion``.
|
||||
|
||||
Tracks the workers and regions it creates and tears them down at the
|
||||
end of the test, so each test starts from a fresh /dev/shm file.
|
||||
"""
|
||||
workers: list[ECCPUWorker] = []
|
||||
regions: list[ECSharedRegion] = []
|
||||
|
||||
def factory(
|
||||
*,
|
||||
rank: int = 0,
|
||||
tp_rank: int = 0,
|
||||
pcp_rank: int = 0,
|
||||
pin_memory_available: bool = False,
|
||||
) -> ECCPUWorker:
|
||||
region = _make_region()
|
||||
regions.append(region)
|
||||
pcp_group = Mock()
|
||||
pcp_group.rank_in_group = pcp_rank
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.create_ec_shared_region",
|
||||
return_value=region,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.is_pin_memory_available",
|
||||
return_value=pin_memory_available,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_tensor_model_parallel_rank",
|
||||
return_value=tp_rank,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_pcp_group",
|
||||
return_value=pcp_group,
|
||||
),
|
||||
):
|
||||
worker = ECCPUWorker(_vllm_config(rank=rank))
|
||||
workers.append(worker)
|
||||
return worker
|
||||
|
||||
yield factory
|
||||
|
||||
for worker in workers:
|
||||
with contextlib.suppress(Exception):
|
||||
worker.shutdown()
|
||||
for region in regions:
|
||||
with contextlib.suppress(Exception):
|
||||
region.cleanup()
|
||||
|
||||
|
||||
# ── save_caches ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@pytest.mark.parametrize(
|
||||
"n_elements,n_blocks",
|
||||
[
|
||||
(3 * _HIDDEN_DIM, 3), # exact fit: 3 full blocks, no remainder
|
||||
(3 * _HIDDEN_DIM + 4, 4), # partial last block: 4 fp16 trail in slot 3
|
||||
],
|
||||
ids=["exact-fit", "partial-last-block"],
|
||||
)
|
||||
def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks):
|
||||
"""``save_caches`` + ``flush_saves`` copies the source GPU tensor's bytes
|
||||
into the block IDs named by ``meta.saves``, in the order they appear,
|
||||
via a single batched GPU→CPU copy.
|
||||
|
||||
When ``total_bytes`` is not a multiple of ``block_size_bytes`` the last
|
||||
block is partially written; the unwritten tail must remain whatever
|
||||
was there before. Blocks not named in ``meta.saves`` must be untouched.
|
||||
"""
|
||||
worker = make_worker()
|
||||
sentinel = 0x5A
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
|
||||
src = torch.arange(n_elements, dtype=_DTYPE, device="cuda")
|
||||
expected_bytes = src.cpu().reshape(-1).view(torch.uint8)
|
||||
total_bytes = n_elements * _DTYPE.itemsize
|
||||
|
||||
block_ids = [7, 2, 5, 0][:n_blocks]
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
|
||||
worker.flush_saves()
|
||||
|
||||
for slot, block_idx in enumerate(block_ids):
|
||||
block_byte_start = slot * _BLOCK_SIZE_BYTES
|
||||
block_byte_end = min(block_byte_start + _BLOCK_SIZE_BYTES, total_bytes)
|
||||
n_written = block_byte_end - block_byte_start
|
||||
actual_bytes = worker._region.blocks[block_idx, :n_written].view(torch.uint8)
|
||||
assert torch.equal(
|
||||
actual_bytes, expected_bytes[block_byte_start:block_byte_end]
|
||||
), f"block {block_idx} (slot {slot}) bytes mismatch"
|
||||
if n_written < _BLOCK_SIZE_BYTES:
|
||||
tail = worker._region.blocks[block_idx, n_written:]
|
||||
assert torch.all(tail == sentinel), (
|
||||
f"block {block_idx} tail was overwritten"
|
||||
)
|
||||
|
||||
for idx in set(range(_NUM_BLOCKS)) - set(block_ids):
|
||||
assert torch.all(worker._region.blocks[idx] == sentinel), (
|
||||
f"block {idx} (unassigned) was overwritten"
|
||||
)
|
||||
|
||||
|
||||
def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
|
||||
"""When the scheduler hasn't pre-allocated blocks for ``mm_hash``,
|
||||
``save_caches`` + ``flush_saves`` is a pure no-op."""
|
||||
worker = make_worker()
|
||||
sentinel = 0x42
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
|
||||
worker.save_caches({}, "h", _meta(saves={}))
|
||||
worker.flush_saves()
|
||||
|
||||
assert torch.all(worker._region.blocks == sentinel)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_rank,pcp_rank",
|
||||
[(1, 0), (0, 1), (1, 1)],
|
||||
ids=["tp1-pcp0", "tp0-pcp1", "tp1-pcp1"],
|
||||
)
|
||||
def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
|
||||
"""Only TP rank 0 + PCP rank 0 writes to mmap. All other combinations
|
||||
must skip the save entirely."""
|
||||
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
|
||||
sentinel = 0x42
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
|
||||
worker.save_caches(
|
||||
{"h": torch.zeros(8, dtype=_DTYPE)}, "h", _meta(saves={"h": [0]})
|
||||
)
|
||||
worker.flush_saves()
|
||||
|
||||
assert torch.all(worker._region.blocks == sentinel)
|
||||
|
||||
|
||||
def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
|
||||
"""``save_caches`` must raise ``AssertionError`` when the encoder output
|
||||
is larger than the allocated block space."""
|
||||
worker = make_worker()
|
||||
src = torch.zeros(3 * _HIDDEN_DIM, dtype=_DTYPE)
|
||||
with pytest.raises(AssertionError, match="exceeds allocated blocks"):
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]}))
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_save_caches_batches_multiple_hashes(make_worker):
|
||||
"""Multiple save_caches calls are batched into a single flush."""
|
||||
worker = make_worker()
|
||||
sentinel = 0x5A
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
|
||||
src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
|
||||
src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device="cuda")
|
||||
|
||||
cache = {"a": src_a, "b": src_b}
|
||||
worker.save_caches(cache, "a", _meta(saves={"a": [1], "b": [3]}))
|
||||
worker.save_caches(cache, "b", _meta(saves={"a": [1], "b": [3]}))
|
||||
worker.flush_saves()
|
||||
|
||||
expected_a = src_a.cpu().view(torch.uint8)
|
||||
expected_b = src_b.cpu().view(torch.uint8)
|
||||
actual_a = worker._region.blocks[1].view(torch.uint8)
|
||||
actual_b = worker._region.blocks[3].view(torch.uint8)
|
||||
assert torch.equal(actual_a, expected_a)
|
||||
assert torch.equal(actual_b, expected_b)
|
||||
|
||||
# Unassigned blocks untouched.
|
||||
for idx in (0, 2, 4, 5, 6, 7):
|
||||
assert torch.all(worker._region.blocks[idx] == sentinel)
|
||||
|
||||
|
||||
# ── start_load_caches ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker):
|
||||
"""Single batched load across all hashes with correct byte→dtype→shape."""
|
||||
worker = make_worker()
|
||||
n_blocks = 3
|
||||
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
|
||||
|
||||
block_ids = [3, 1, 6]
|
||||
for i, idx in enumerate(block_ids):
|
||||
worker._region.blocks[idx].copy_(src_int8[i])
|
||||
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
|
||||
|
||||
out = encoder_cache["h"]
|
||||
assert out.is_cuda, "consumer worker must place the tensor on the GPU"
|
||||
assert out.shape == (n_blocks, _HIDDEN_DIM)
|
||||
assert out.dtype == _DTYPE
|
||||
assert torch.equal(out.cpu(), src_orig)
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker):
|
||||
"""If ``encoder_cache`` already holds the ``mm_hash``, the worker must
|
||||
not overwrite it."""
|
||||
worker = make_worker()
|
||||
worker._region.blocks[0].fill_(0x42)
|
||||
|
||||
sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device="cuda")
|
||||
encoder_cache = {"h": sentinel}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": [0]}))
|
||||
|
||||
assert encoder_cache["h"] is sentinel, (
|
||||
"existing encoder_cache entry must not be replaced"
|
||||
)
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_start_load_caches_noop_when_loads_is_empty(make_worker):
|
||||
"""When ``meta.loads`` is empty the early-return must fire."""
|
||||
worker = make_worker()
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={}))
|
||||
|
||||
assert encoder_cache == {}
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
|
||||
"""Cached entries are preserved while new ones are loaded."""
|
||||
worker = make_worker()
|
||||
n_blocks = 2
|
||||
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
|
||||
new_block_ids = [4, 2]
|
||||
for i, idx in enumerate(new_block_ids):
|
||||
worker._region.blocks[idx].copy_(src_int8[i])
|
||||
|
||||
cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device="cuda")
|
||||
encoder_cache: dict[str, torch.Tensor] = {"cached_h": cached_tensor}
|
||||
worker.start_load_caches(
|
||||
encoder_cache,
|
||||
_meta(loads={"cached_h": [0], "new_h": new_block_ids}),
|
||||
)
|
||||
|
||||
assert encoder_cache["cached_h"] is cached_tensor
|
||||
assert "new_h" in encoder_cache
|
||||
out = encoder_cache["new_h"]
|
||||
assert out.shape == (n_blocks, _HIDDEN_DIM)
|
||||
assert out.dtype == _DTYPE
|
||||
assert torch.equal(out.cpu(), src_orig)
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@pytest.mark.parametrize(
|
||||
"tp_rank,pcp_rank",
|
||||
[(0, 0), (1, 0), (0, 1), (1, 1)],
|
||||
ids=["tp0-pcp0", "tp1-pcp0", "tp0-pcp1", "tp1-pcp1"],
|
||||
)
|
||||
def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank):
|
||||
"""All TP/PCP ranks must load from mmap — loads are NOT gated like saves."""
|
||||
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
|
||||
n_blocks = 2
|
||||
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
|
||||
block_ids = [1, 3]
|
||||
for i, idx in enumerate(block_ids):
|
||||
worker._region.blocks[idx].copy_(src_int8[i])
|
||||
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
|
||||
|
||||
out = encoder_cache["h"]
|
||||
assert out.is_cuda
|
||||
assert torch.equal(out.cpu(), src_orig)
|
||||
|
||||
|
||||
# ── round-trip ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_save_then_load_round_trips_bytes(make_worker):
|
||||
"""Full producer→mmap→consumer byte path in one shot."""
|
||||
worker = make_worker()
|
||||
n_blocks = 3
|
||||
block_ids = [5, 1, 6]
|
||||
|
||||
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
|
||||
worker.flush_saves()
|
||||
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
|
||||
|
||||
out = encoder_cache["h"]
|
||||
assert out.shape == src.shape
|
||||
assert out.dtype == src.dtype
|
||||
assert torch.equal(out.cpu(), src.cpu())
|
||||
|
||||
|
||||
# ── buffer recycling ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_buffer_pool_is_reused_across_save_steps(make_worker):
|
||||
"""After flush_saves, descriptor buffers are returned to the pool and
|
||||
reused on the next flush — no reallocation."""
|
||||
worker = make_worker()
|
||||
src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
|
||||
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0]}))
|
||||
worker.flush_saves()
|
||||
|
||||
assert len(worker._buf_pool._pool) == 1
|
||||
buf_id = id(worker._buf_pool._pool[0].src_ptrs)
|
||||
|
||||
# Second step reuses the same buffer.
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": [1]}))
|
||||
worker.flush_saves()
|
||||
|
||||
assert len(worker._buf_pool._pool) == 1
|
||||
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_buffer_pool_is_reused_across_load_steps(make_worker):
|
||||
"""After start_load_caches, descriptor buffers are returned to the pool
|
||||
and reused on the next call."""
|
||||
worker = make_worker()
|
||||
worker._region.blocks[0].fill_(0x01)
|
||||
worker._region.blocks[1].fill_(0x02)
|
||||
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"a": [0]}))
|
||||
|
||||
assert len(worker._buf_pool._pool) == 1
|
||||
buf_id = id(worker._buf_pool._pool[0].src_ptrs)
|
||||
|
||||
encoder_cache2: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache2, _meta(loads={"b": [1]}))
|
||||
|
||||
assert len(worker._buf_pool._pool) == 1
|
||||
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
|
||||
|
||||
|
||||
# ── stream management ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_stream_initialized_at_construction(make_worker):
|
||||
"""``_load_stream`` must be a fully initialized CUDA stream."""
|
||||
worker = make_worker()
|
||||
assert isinstance(worker._load_stream, torch.cuda.Stream)
|
||||
|
||||
|
||||
# ── lifecycle ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pin_available,expected_pinned",
|
||||
[
|
||||
(True, True),
|
||||
(False, False),
|
||||
],
|
||||
ids=["available", "unavailable"],
|
||||
)
|
||||
def test_init_pins_memory_when_available(pin_available, expected_pinned):
|
||||
"""Every worker must call ``pin_memory()`` when the platform allows it."""
|
||||
region = _make_region()
|
||||
region.pin_memory = MagicMock() # type: ignore[method-assign]
|
||||
|
||||
pcp_group = Mock()
|
||||
pcp_group.rank_in_group = 0
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.create_ec_shared_region",
|
||||
return_value=region,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.is_pin_memory_available",
|
||||
return_value=pin_available,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_tensor_model_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_pcp_group",
|
||||
return_value=pcp_group,
|
||||
),
|
||||
):
|
||||
ECCPUWorker(_vllm_config(rank=0))
|
||||
|
||||
try:
|
||||
if expected_pinned:
|
||||
region.pin_memory.assert_called_once()
|
||||
else:
|
||||
region.pin_memory.assert_not_called()
|
||||
finally:
|
||||
region.cleanup()
|
||||
|
||||
|
||||
def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
|
||||
"""``shutdown`` must always call ``region.cleanup`` — and must never
|
||||
raise."""
|
||||
worker = object.__new__(ECCPUWorker)
|
||||
mock_region = Mock(spec=ECSharedRegion)
|
||||
worker._region = mock_region
|
||||
worker._load_stream = MagicMock()
|
||||
worker._save_bufs = None
|
||||
worker._save_count = 0
|
||||
|
||||
worker.shutdown()
|
||||
worker._load_stream.synchronize.assert_called_once()
|
||||
mock_region.cleanup.assert_called_once()
|
||||
|
||||
mock_region.cleanup.side_effect = RuntimeError("boom")
|
||||
with caplog_vllm.at_level(logging.DEBUG, logger="vllm"):
|
||||
worker.shutdown() # exception must be swallowed
|
||||
|
||||
assert mock_region.cleanup.call_count == 2
|
||||
assert any(
|
||||
"worker region cleanup failed" in r.message
|
||||
for r in caplog_vllm.records
|
||||
if r.levelno == logging.DEBUG
|
||||
)
|
||||
|
||||
|
||||
# ── e2e: scheduler + worker pipeline ────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
"""Full pipeline: scheduler allocates blocks, worker saves GPU tensor to
|
||||
mmap via flush_saves, scheduler marks ready after step delay, worker
|
||||
loads from mmap back to GPU, and the result matches the original.
|
||||
|
||||
Exercises the real scheduler + worker cooperation through a shared
|
||||
ECSharedRegion, with real CUDA transfers and stream coordination.
|
||||
"""
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
|
||||
ECCPUScheduler,
|
||||
)
|
||||
|
||||
# Build worker first (gets us a real region).
|
||||
worker = make_worker()
|
||||
region = worker._region
|
||||
|
||||
# Build scheduler sharing the same region.
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
|
||||
class _EC:
|
||||
is_ec_producer = True
|
||||
is_ec_consumer = True
|
||||
|
||||
class _Cfg:
|
||||
ec_transfer_config = _EC()
|
||||
max_concurrent_batches = 1
|
||||
|
||||
scheduler = ECCPUScheduler(_Cfg())
|
||||
|
||||
# -- Step 1: scheduler allocates, worker saves --
|
||||
n_blocks = 3
|
||||
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
|
||||
class _Pos:
|
||||
offset = 0
|
||||
length = n_blocks
|
||||
|
||||
class _Feature:
|
||||
identifier = "img_001"
|
||||
mm_position = _Pos()
|
||||
|
||||
class _Request:
|
||||
request_id = "req_e2e"
|
||||
mm_features = [_Feature()]
|
||||
|
||||
scheduler.update_state_after_alloc(_Request(), 0)
|
||||
meta_save = scheduler.build_connector_meta(scheduler_output=None)
|
||||
assert "img_001" in meta_save.saves
|
||||
|
||||
encoder_cache = {"img_001": src}
|
||||
worker.save_caches(encoder_cache, "img_001", meta_save)
|
||||
worker.flush_saves()
|
||||
|
||||
# -- Step 2: readiness delay (max_concurrent_batches=1) --
|
||||
meta_step2 = scheduler.build_connector_meta(scheduler_output=None)
|
||||
assert scheduler.has_cache_item("img_001") is True
|
||||
assert meta_step2.loads == {} # no load requested yet
|
||||
|
||||
# -- Step 3: scheduler emits load, worker loads --
|
||||
scheduler.update_state_after_alloc(_Request(), 0)
|
||||
meta_load = scheduler.build_connector_meta(scheduler_output=None)
|
||||
assert "img_001" in meta_load.loads
|
||||
assert meta_load.loads["img_001"] == meta_save.saves["img_001"]
|
||||
|
||||
load_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(load_cache, meta_load)
|
||||
|
||||
out = load_cache["img_001"]
|
||||
assert out.is_cuda
|
||||
assert out.shape == src.shape
|
||||
assert out.dtype == src.dtype
|
||||
assert torch.equal(out.cpu(), src.cpu())
|
||||
|
||||
scheduler.shutdown()
|
||||
@@ -0,0 +1,158 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E test for the ECCPUConnector.
|
||||
|
||||
Verifies:
|
||||
- Accuracy: outputs from EC CPU cache match fresh encoder computation.
|
||||
- Latency: loading from EC CPU cache is faster than a cold encoder run.
|
||||
|
||||
Requires a CUDA GPU and the Qwen2-VL-2B-Instruct model.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.config import ECTransferConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL = "Qwen/Qwen2-VL-2B-Instruct"
|
||||
EC_CPU_BYTES = 500 << 20 # 500 MiB
|
||||
|
||||
|
||||
def _build_image_prompt(image):
|
||||
"""Build a Qwen2-VL prompt with a single image."""
|
||||
return {
|
||||
"prompt": (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|image_pad|><|vision_end|>"
|
||||
"Describe this image briefly.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
"multi_modal_data": {"image": image},
|
||||
}
|
||||
|
||||
|
||||
def _make_unique_image(base_image, iteration: int):
|
||||
"""Create a unique image by modifying a pixel to change the mm_hash."""
|
||||
img = base_image.copy()
|
||||
img.putpixel((0, 0), (iteration % 256, iteration // 256 % 256, 0))
|
||||
return img
|
||||
|
||||
|
||||
def _wait_for_ec_ready(llm: LLM) -> None:
|
||||
"""Send a dummy request to force a scheduler step.
|
||||
|
||||
After generate() returns, the EC entry's data is in the mmap but
|
||||
readiness may not have fired yet (first-finish fires in the NEXT
|
||||
scheduler step). This forces that step.
|
||||
"""
|
||||
llm.generate("hi", SamplingParams(max_tokens=1), use_tqdm=False)
|
||||
|
||||
|
||||
def _latency_test(llm: LLM) -> None:
|
||||
"""Verify EC CPU cache hit is faster than cold encoder computation."""
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Latency test requires CUDA")
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=1, temperature=0)
|
||||
base_image = ImageAsset("stop_sign").pil_image
|
||||
|
||||
num_times_ec_better_than_cold = 0
|
||||
num_tests = 10
|
||||
total_cold_time = 0.0
|
||||
total_ec_hit_time = 0.0
|
||||
|
||||
for i in tqdm(range(num_tests), desc="EC latency test"):
|
||||
# Use a unique image each iteration so the cold run is truly cold
|
||||
# (EC has never seen this mm_hash before).
|
||||
image = _make_unique_image(base_image, i)
|
||||
prompt = _build_image_prompt(image)
|
||||
|
||||
# Cold run: encoder computes from scratch, saves to EC CPU.
|
||||
llm.llm_engine.reset_encoder_cache()
|
||||
start_time = time.time()
|
||||
llm.generate([prompt], sampling_params, use_tqdm=False)
|
||||
cold_time = time.time() - start_time
|
||||
total_cold_time += cold_time
|
||||
|
||||
# Ensure EC entry is marked ready.
|
||||
_wait_for_ec_ready(llm)
|
||||
|
||||
# Clear GPU encoder cache but keep EC CPU cache intact.
|
||||
llm.llm_engine.reset_encoder_cache()
|
||||
|
||||
# EC hit: scheduler loads from CPU mmap instead of re-encoding.
|
||||
start_time = time.time()
|
||||
llm.generate([prompt], sampling_params, use_tqdm=False)
|
||||
ec_hit_time = time.time() - start_time
|
||||
total_ec_hit_time += ec_hit_time
|
||||
|
||||
if ec_hit_time < cold_time:
|
||||
num_times_ec_better_than_cold += 1
|
||||
|
||||
print("Average times:")
|
||||
print(f" Cold: {total_cold_time * 1000 / num_tests:.2f}ms")
|
||||
print(f" EC hit: {total_ec_hit_time * 1000 / num_tests:.2f}ms")
|
||||
|
||||
assert num_times_ec_better_than_cold >= 0.8 * num_tests, (
|
||||
f"EC hit was faster only {num_times_ec_better_than_cold}/{num_tests} "
|
||||
f"times (expected >= 80%)"
|
||||
)
|
||||
|
||||
|
||||
def _accuracy_test(llm: LLM) -> None:
|
||||
"""Verify EC CPU cache produces correct outputs."""
|
||||
sampling_params = SamplingParams(max_tokens=8, temperature=0)
|
||||
image = ImageAsset("stop_sign").pil_image
|
||||
prompt = _build_image_prompt(image)
|
||||
|
||||
# Seed: generate with fresh encoder to establish baseline output.
|
||||
llm.llm_engine.reset_encoder_cache()
|
||||
baseline = llm.generate([prompt], sampling_params, use_tqdm=False)
|
||||
baseline_text = baseline[0].outputs[0].text
|
||||
|
||||
# Ensure EC entry is marked ready.
|
||||
_wait_for_ec_ready(llm)
|
||||
|
||||
# Clear GPU encoder cache so subsequent runs go through EC CPU path.
|
||||
llm.llm_engine.reset_encoder_cache()
|
||||
|
||||
# Generate multiple times from EC cache and check correctness.
|
||||
test_count = 20
|
||||
results = llm.generate([prompt] * test_count, sampling_params, use_tqdm=False)
|
||||
|
||||
success_count = sum(1 for r in results if r.outputs[0].text == baseline_text)
|
||||
assert success_count >= 0.5 * test_count, (
|
||||
f"Only {success_count}/{test_count} outputs matched baseline "
|
||||
f"(expected >= 50%). Baseline: {baseline_text!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
def test_ec_cpu_offloading() -> None:
|
||||
"""Tests ECCPUConnector accuracy and latency with a VLM model."""
|
||||
ec_transfer_config = ECTransferConfig(
|
||||
ec_connector="ECCPUConnector",
|
||||
ec_role="ec_both",
|
||||
ec_connector_extra_config={"ec_cpu_bytes": EC_CPU_BYTES},
|
||||
)
|
||||
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
max_model_len=2048,
|
||||
gpu_memory_utilization=0.5,
|
||||
enforce_eager=True,
|
||||
ec_transfer_config=ec_transfer_config,
|
||||
)
|
||||
|
||||
try:
|
||||
_latency_test(llm)
|
||||
_accuracy_test(llm)
|
||||
finally:
|
||||
del llm
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECCPUConnectorMetadata."""
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
)
|
||||
|
||||
|
||||
def test_metadata_is_ec_connector_metadata():
|
||||
"""ECCPUConnectorMetadata must be a subclass of ECConnectorMetadata."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
assert isinstance(meta, ECConnectorMetadata)
|
||||
|
||||
|
||||
def test_metadata_initialization():
|
||||
"""ECCPUConnectorMetadata initializes with empty saves and loads dicts."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
assert hasattr(meta, "saves")
|
||||
assert hasattr(meta, "loads")
|
||||
assert meta.saves == {}
|
||||
assert meta.loads == {}
|
||||
|
||||
|
||||
def test_metadata_saves_dict_operations():
|
||||
"""The saves dict supports standard dict operations."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
# Add multiple entries
|
||||
meta.saves["mm_hash_1"] = [0, 1, 2]
|
||||
meta.saves["mm_hash_2"] = [10, 11]
|
||||
|
||||
assert "mm_hash_1" in meta.saves
|
||||
assert "mm_hash_2" in meta.saves
|
||||
assert len(meta.saves) == 2
|
||||
assert meta.saves["mm_hash_1"] == [0, 1, 2]
|
||||
assert meta.saves["mm_hash_2"] == [10, 11]
|
||||
|
||||
|
||||
def test_metadata_loads_dict_operations():
|
||||
"""The loads dict supports standard dict operations."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
# Add multiple entries
|
||||
meta.loads["mm_hash_1"] = [5, 6, 7]
|
||||
meta.loads["mm_hash_2"] = [100]
|
||||
|
||||
assert "mm_hash_1" in meta.loads
|
||||
assert "mm_hash_2" in meta.loads
|
||||
assert len(meta.loads) == 2
|
||||
assert meta.loads["mm_hash_1"] == [5, 6, 7]
|
||||
assert meta.loads["mm_hash_2"] == [100]
|
||||
|
||||
|
||||
def test_metadata_saves_and_loads_are_independent():
|
||||
"""saves and loads dicts are independent."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
meta.saves["key"] = [1, 2]
|
||||
meta.loads["key"] = [3, 4]
|
||||
|
||||
# Same key but different dicts should not interfere
|
||||
assert meta.saves["key"] == [1, 2]
|
||||
assert meta.loads["key"] == [3, 4]
|
||||
|
||||
# Modifying one should not affect the other
|
||||
meta.saves["other"] = [99]
|
||||
assert "other" not in meta.loads
|
||||
|
||||
|
||||
def test_metadata_block_indices_are_integers():
|
||||
"""Block indices in saves/loads lists should be integers."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
meta.saves["hash"] = [0, 100, 999, 65535]
|
||||
meta.loads["hash"] = [1, 2, 3]
|
||||
|
||||
for idx in meta.saves["hash"]:
|
||||
assert isinstance(idx, int)
|
||||
for idx in meta.loads["hash"]:
|
||||
assert isinstance(idx, int)
|
||||
|
||||
|
||||
def test_metadata_multiple_instances_are_independent():
|
||||
"""Different ECCPUConnectorMetadata instances should not share state."""
|
||||
meta1 = ECCPUConnectorMetadata()
|
||||
meta2 = ECCPUConnectorMetadata()
|
||||
|
||||
meta1.saves["x"] = [1]
|
||||
meta2.saves["y"] = [2]
|
||||
|
||||
assert "x" in meta1.saves
|
||||
assert "x" not in meta2.saves
|
||||
assert "y" not in meta1.saves
|
||||
assert "y" in meta2.saves
|
||||
Reference in New Issue
Block a user