[KV Connector] Fix PD async scheduling race condition for hybrid attn models (#48481)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: llx-08 <2596671364@qq.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
llx-08
Nick Hill
Claude Fable 5
parent
a317bc5739
commit
530852f959
@@ -752,6 +752,94 @@ def test_nixl_metadata_hybrid_ssm_block_ids():
|
||||
assert len(req_meta.remote.block_ids[0]) != len(req_meta.remote.block_ids[1])
|
||||
|
||||
|
||||
class _FakeBlock:
|
||||
def __init__(self, block_id):
|
||||
self.block_id = block_id
|
||||
|
||||
|
||||
class _FakeSingleTypeManager:
|
||||
def __init__(self, records, block_size, block_ids):
|
||||
self.records_new_block_ids = records
|
||||
self.block_size = block_size
|
||||
self.req_to_blocks = {"req-1": [_FakeBlock(b) for b in block_ids]}
|
||||
self.new_block_ids: list[int] = []
|
||||
|
||||
def take_new_block_ids(self):
|
||||
ids = self.new_block_ids
|
||||
self.new_block_ids = []
|
||||
return ids
|
||||
|
||||
|
||||
def _make_fake_kv_cache_manager():
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
|
||||
manager = object.__new__(KVCacheManager)
|
||||
manager.coordinator = MagicMock()
|
||||
manager.coordinator.single_type_managers = (
|
||||
_FakeSingleTypeManager(True, 16, [10, 11, 12, 13, 14, 15]), # attention
|
||||
_FakeSingleTypeManager(False, 16, [20, 21, 22, 23, 24, 25]), # mamba
|
||||
)
|
||||
return manager
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_zeroing_block_ids_cover_only_loaded_attention_blocks():
|
||||
"""Only zero-recorded (attention) groups contribute, sliced to the
|
||||
externally-loaded token range; Mamba state blocks are never zeroed."""
|
||||
manager = _make_fake_kv_cache_manager()
|
||||
|
||||
# Tokens [0, 16) are locally cached; the load covers tokens [16, 56).
|
||||
assert manager.get_zeroing_block_ids_in_range("req-1", 16, 56) == [11, 12, 13]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_scheduler_filters_connector_loaded_blocks_from_zeroing():
|
||||
"""Blocks that will be loaded by the connector must not be zeroed."""
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
|
||||
class FakeKVCacheManager:
|
||||
def take_new_block_ids(self):
|
||||
return [9, 10, 11, 12]
|
||||
|
||||
scheduler = object.__new__(Scheduler)
|
||||
scheduler.needs_kv_cache_zeroing = True
|
||||
scheduler.kv_cache_manager = FakeKVCacheManager()
|
||||
scheduler._skip_zero_block_ids = {10, 12}
|
||||
|
||||
assert scheduler._get_new_block_ids_to_zero() == [9, 11]
|
||||
assert not scheduler._skip_zero_block_ids
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_failed_load_rezeroes_unwritten_skipped_blocks():
|
||||
"""A failed async load leaves zeroing-skipped blocks unwritten beyond
|
||||
the valid prefix; they must be zeroed before local recompute."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
|
||||
scheduler = object.__new__(Scheduler)
|
||||
scheduler.connector = MagicMock()
|
||||
scheduler.needs_kv_cache_zeroing = True
|
||||
scheduler.kv_cache_manager = _make_fake_kv_cache_manager()
|
||||
scheduler.kv_cache_manager.cache_blocks = MagicMock()
|
||||
scheduler.failed_recving_kv_req_ids = {"req-1"}
|
||||
scheduler.finished_recving_kv_req_ids = {"req-1"}
|
||||
|
||||
request = MagicMock()
|
||||
request.request_id = "req-1"
|
||||
request.num_computed_tokens = 48 # Truncated at the first invalid block.
|
||||
|
||||
scheduler._update_waiting_for_remote_kv(request)
|
||||
|
||||
# Attention blocks covering tokens >= 48 are re-recorded for zeroing
|
||||
# and flow into the next step's zero list; Mamba blocks are not.
|
||||
scheduler._skip_zero_block_ids = set()
|
||||
assert scheduler._get_new_block_ids_to_zero() == [13, 14, 15]
|
||||
|
||||
|
||||
# ── Mamba N-1 prefill tests ──────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user