[PD][NixlPush][Bugfix] Fix prefix caching (#48758)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-07 16:21:17 +02:00
committed by GitHub
parent f2bfad9167
commit d4ecb75ba2
5 changed files with 189 additions and 73 deletions
@@ -314,7 +314,10 @@ def test_apply_prefix_caching_mamba_hybrid(
worker.kv_cache_config = make_kv_cache_config(block_size=16, mamba_enabled=True)
aligned_local, aligned_remote = worker._apply_prefix_caching(
local_block_ids, remote_block_ids, remote_physical_per_logical
local_block_ids,
remote_block_ids,
local_physical_per_logical,
remote_physical_per_logical,
)
assert aligned_local == expected_local, (
@@ -413,7 +416,10 @@ def test_apply_prefix_caching_ssm_prefix_cache_hit(
worker.kv_cache_config = make_kv_cache_config(block_size=16, mamba_enabled=True)
aligned_local, aligned_remote = worker._apply_prefix_caching(
local_block_ids, remote_block_ids, remote_physical_per_logical
local_block_ids,
remote_block_ids,
local_physical_per_logical,
remote_physical_per_logical,
)
assert aligned_local == expected_local, (
@@ -442,7 +448,7 @@ def test_apply_prefix_caching_ssm_unpairable_slots_rejected():
with pytest.raises(AssertionError, match="unpairable SSM state slots"):
worker._apply_prefix_caching(
[list(range(10)), [4, 5, 6, 7]], [list(range(10)), [8, 9]], 10
[list(range(10)), [4, 5, 6, 7]], [list(range(10)), [8, 9]], 10, 10
)
@@ -527,6 +533,7 @@ def test_mismatched_physical_per_logical_fails_with_prefix_caching(
aligned_local, aligned_remote = worker._apply_prefix_caching(
local_block_ids,
remote_block_ids,
local_physical_per_logical,
remote_physical_per_logical,
)
@@ -341,6 +341,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
# Single non-hybrid attention group, matching the stub block id lists.
w._has_mamba = False
w._group_spec_types = (FullAttentionSpec,)
w._engine_ttl = 0.0
w._engine_last_active = {}
# Track _do_start_push_kv invocations.
@@ -569,7 +570,7 @@ def test_do_start_push_kv_drops_request_on_handshake_failure():
the failure is logged. Blocks are reclaimed by the lease/watchdog, matching
the old blocking behaviour."""
w = _StubWriterWorker.fresh()
w._logical_to_kernel_block_ids = lambda x: x
w._logical_to_kernel_block_ids = lambda x, ratio: x
xfer_calls: list[dict[str, Any]] = []
w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw)
failures: list[dict[str, Any]] = []
@@ -1157,3 +1158,93 @@ class TestPushWriterMlaReplication:
# All of the request's WRITE handles must be tracked together, so the
# engine thread never sees a partial set and double-frees the request.
assert sorted(w._sending_transfers["p-req"]) == [1000, 1001]
class TestPushPrefixCaching:
"""Partial prefix-cache hit on D: D preallocates only its *uncomputed*
blocks, so on a partial hit it registers fewer blocks than P's full
sequence. P must WRITE only the *tail* of its sequence into D's slots --
mirroring pull-mode ``_apply_prefix_caching`` (end-trim), never a front-trim
which would write P's cached prefix into D's uncomputed suffix slots.
The trim runs inside ``_xfer_blocks``, so these tests drive the real
``_xfer_blocks_for_req`` path and stub only the NIXL WRITE. With one region
and 1:1 ratios ``_compute_desc_ids`` is the identity map, so the descs
captured from ``make_prepped_xfer`` are exactly the (trimmed) block IDs.
"""
@staticmethod
def _worker_driving_xfer(engine_id: str = "decode-engine"):
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
)
w = _StubWriterWorker.fresh()
w._has_mamba = False
w.use_mla = False
w.block_size = 16
w.num_regions = 1
w._physical_blocks_per_logical_kv_block = 1
w.transfer_topo = MagicMock()
w.transfer_topo.get_engine_info.return_value = SimpleNamespace(
remote_physical_blocks_per_logical=1,
remote_block_size=16,
remote_tp_size=1,
)
w.transfer_topo.tp_ratio.return_value = 1
w.transfer_topo.block_size_ratio.return_value = 1
w.tp_mappings = {
engine_id: TPMapping(
source_ranks_per_group=((0,),),
all_source_ranks=(0,),
rank_to_attention_slot={0: 0},
rank_offset_factor=0,
)
}
w.dst_num_blocks = {engine_id: 10_000, w.engine_id: 10_000}
w.dst_xfer_side_handles = {engine_id: {0: 5000}}
w.src_xfer_handles_by_block_size = {16: 2000}
# Stub only the NIXL WRITE; kernel expansion, prefix-cache trim, desc
# computation and count assertions all run for real.
w.nixl_wrapper = MagicMock()
w.nixl_wrapper.make_prepped_xfer.return_value = 7
w._ensure_handshake = lambda *a, **k: None
w._logical_to_kernel_block_ids = lambda x, ratio: x
return w, engine_id
@staticmethod
def _written_block_ids(w) -> tuple[list[int], list[int]]:
"""Return the (local, remote) block IDs handed to the NIXL WRITE."""
args, _ = w.nixl_wrapper.make_prepped_xfer.call_args
# ("WRITE", local_handle, local_descs, remote_handle, remote_descs)
return list(args[2]), list(args[4])
def test_partial_prefix_hit_end_trims_producer_blocks(self):
"""D registered only its 2 uncomputed suffix blocks; P finished the
full 5-block sequence. P must WRITE its LAST 2 blocks into D's slots."""
w, _ = self._worker_driving_xfer()
reg = _registration_data("req-pc", local_block_ids=([500, 501],))
NixlPushConnectorWorker._do_start_push_kv(
w, "req-pc", ([10, 11, 12, 13, 14],), reg
)
local, remote = self._written_block_ids(w)
# End-trim (suffix), NOT front-trim: [13, 14], not [10, 11].
assert local == [13, 14]
assert remote == [500, 501]
def test_no_prefix_hit_leaves_blocks_untrimmed(self):
"""Equal counts (no prefix cache hit on D): nothing is trimmed."""
w, _ = self._worker_driving_xfer()
reg = _registration_data("req-full", local_block_ids=([500, 501, 502],))
NixlPushConnectorWorker._do_start_push_kv(w, "req-full", ([10, 11, 12],), reg)
local, remote = self._written_block_ids(w)
assert local == [10, 11, 12]
assert remote == [500, 501, 502]