[PD][NixlPush][Bugfix] Fix prefix caching (#48758)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -314,7 +314,10 @@ def test_apply_prefix_caching_mamba_hybrid(
|
||||
worker.kv_cache_config = make_kv_cache_config(block_size=16, mamba_enabled=True)
|
||||
|
||||
aligned_local, aligned_remote = worker._apply_prefix_caching(
|
||||
local_block_ids, remote_block_ids, remote_physical_per_logical
|
||||
local_block_ids,
|
||||
remote_block_ids,
|
||||
local_physical_per_logical,
|
||||
remote_physical_per_logical,
|
||||
)
|
||||
|
||||
assert aligned_local == expected_local, (
|
||||
@@ -413,7 +416,10 @@ def test_apply_prefix_caching_ssm_prefix_cache_hit(
|
||||
worker.kv_cache_config = make_kv_cache_config(block_size=16, mamba_enabled=True)
|
||||
|
||||
aligned_local, aligned_remote = worker._apply_prefix_caching(
|
||||
local_block_ids, remote_block_ids, remote_physical_per_logical
|
||||
local_block_ids,
|
||||
remote_block_ids,
|
||||
local_physical_per_logical,
|
||||
remote_physical_per_logical,
|
||||
)
|
||||
|
||||
assert aligned_local == expected_local, (
|
||||
@@ -442,7 +448,7 @@ def test_apply_prefix_caching_ssm_unpairable_slots_rejected():
|
||||
|
||||
with pytest.raises(AssertionError, match="unpairable SSM state slots"):
|
||||
worker._apply_prefix_caching(
|
||||
[list(range(10)), [4, 5, 6, 7]], [list(range(10)), [8, 9]], 10
|
||||
[list(range(10)), [4, 5, 6, 7]], [list(range(10)), [8, 9]], 10, 10
|
||||
)
|
||||
|
||||
|
||||
@@ -527,6 +533,7 @@ def test_mismatched_physical_per_logical_fails_with_prefix_caching(
|
||||
aligned_local, aligned_remote = worker._apply_prefix_caching(
|
||||
local_block_ids,
|
||||
remote_block_ids,
|
||||
local_physical_per_logical,
|
||||
remote_physical_per_logical,
|
||||
)
|
||||
|
||||
|
||||
@@ -341,6 +341,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
# Single non-hybrid attention group, matching the stub block id lists.
|
||||
w._has_mamba = False
|
||||
w._group_spec_types = (FullAttentionSpec,)
|
||||
w._engine_ttl = 0.0
|
||||
w._engine_last_active = {}
|
||||
|
||||
# Track _do_start_push_kv invocations.
|
||||
@@ -569,7 +570,7 @@ def test_do_start_push_kv_drops_request_on_handshake_failure():
|
||||
the failure is logged. Blocks are reclaimed by the lease/watchdog, matching
|
||||
the old blocking behaviour."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w._logical_to_kernel_block_ids = lambda x: x
|
||||
w._logical_to_kernel_block_ids = lambda x, ratio: x
|
||||
xfer_calls: list[dict[str, Any]] = []
|
||||
w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw)
|
||||
failures: list[dict[str, Any]] = []
|
||||
@@ -1157,3 +1158,93 @@ class TestPushWriterMlaReplication:
|
||||
# All of the request's WRITE handles must be tracked together, so the
|
||||
# engine thread never sees a partial set and double-frees the request.
|
||||
assert sorted(w._sending_transfers["p-req"]) == [1000, 1001]
|
||||
|
||||
|
||||
class TestPushPrefixCaching:
|
||||
"""Partial prefix-cache hit on D: D preallocates only its *uncomputed*
|
||||
blocks, so on a partial hit it registers fewer blocks than P's full
|
||||
sequence. P must WRITE only the *tail* of its sequence into D's slots --
|
||||
mirroring pull-mode ``_apply_prefix_caching`` (end-trim), never a front-trim
|
||||
which would write P's cached prefix into D's uncomputed suffix slots.
|
||||
|
||||
The trim runs inside ``_xfer_blocks``, so these tests drive the real
|
||||
``_xfer_blocks_for_req`` path and stub only the NIXL WRITE. With one region
|
||||
and 1:1 ratios ``_compute_desc_ids`` is the identity map, so the descs
|
||||
captured from ``make_prepped_xfer`` are exactly the (trimmed) block IDs.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _worker_driving_xfer(engine_id: str = "decode-engine"):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
TPMapping,
|
||||
)
|
||||
|
||||
w = _StubWriterWorker.fresh()
|
||||
w._has_mamba = False
|
||||
w.use_mla = False
|
||||
w.block_size = 16
|
||||
w.num_regions = 1
|
||||
w._physical_blocks_per_logical_kv_block = 1
|
||||
|
||||
w.transfer_topo = MagicMock()
|
||||
w.transfer_topo.get_engine_info.return_value = SimpleNamespace(
|
||||
remote_physical_blocks_per_logical=1,
|
||||
remote_block_size=16,
|
||||
remote_tp_size=1,
|
||||
)
|
||||
w.transfer_topo.tp_ratio.return_value = 1
|
||||
w.transfer_topo.block_size_ratio.return_value = 1
|
||||
w.tp_mappings = {
|
||||
engine_id: TPMapping(
|
||||
source_ranks_per_group=((0,),),
|
||||
all_source_ranks=(0,),
|
||||
rank_to_attention_slot={0: 0},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
}
|
||||
w.dst_num_blocks = {engine_id: 10_000, w.engine_id: 10_000}
|
||||
w.dst_xfer_side_handles = {engine_id: {0: 5000}}
|
||||
w.src_xfer_handles_by_block_size = {16: 2000}
|
||||
|
||||
# Stub only the NIXL WRITE; kernel expansion, prefix-cache trim, desc
|
||||
# computation and count assertions all run for real.
|
||||
w.nixl_wrapper = MagicMock()
|
||||
w.nixl_wrapper.make_prepped_xfer.return_value = 7
|
||||
w._ensure_handshake = lambda *a, **k: None
|
||||
w._logical_to_kernel_block_ids = lambda x, ratio: x
|
||||
return w, engine_id
|
||||
|
||||
@staticmethod
|
||||
def _written_block_ids(w) -> tuple[list[int], list[int]]:
|
||||
"""Return the (local, remote) block IDs handed to the NIXL WRITE."""
|
||||
args, _ = w.nixl_wrapper.make_prepped_xfer.call_args
|
||||
# ("WRITE", local_handle, local_descs, remote_handle, remote_descs)
|
||||
return list(args[2]), list(args[4])
|
||||
|
||||
def test_partial_prefix_hit_end_trims_producer_blocks(self):
|
||||
"""D registered only its 2 uncomputed suffix blocks; P finished the
|
||||
full 5-block sequence. P must WRITE its LAST 2 blocks into D's slots."""
|
||||
w, _ = self._worker_driving_xfer()
|
||||
reg = _registration_data("req-pc", local_block_ids=([500, 501],))
|
||||
|
||||
NixlPushConnectorWorker._do_start_push_kv(
|
||||
w, "req-pc", ([10, 11, 12, 13, 14],), reg
|
||||
)
|
||||
|
||||
local, remote = self._written_block_ids(w)
|
||||
# End-trim (suffix), NOT front-trim: [13, 14], not [10, 11].
|
||||
assert local == [13, 14]
|
||||
assert remote == [500, 501]
|
||||
|
||||
def test_no_prefix_hit_leaves_blocks_untrimmed(self):
|
||||
"""Equal counts (no prefix cache hit on D): nothing is trimmed."""
|
||||
w, _ = self._worker_driving_xfer()
|
||||
reg = _registration_data("req-full", local_block_ids=([500, 501, 502],))
|
||||
|
||||
NixlPushConnectorWorker._do_start_push_kv(w, "req-full", ([10, 11, 12],), reg)
|
||||
|
||||
local, remote = self._written_block_ids(w)
|
||||
assert local == [10, 11, 12]
|
||||
assert remote == [500, 501, 502]
|
||||
|
||||
Reference in New Issue
Block a user