[PD][NixlPush][Bugfix] Fix prefix caching (#48758)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -2397,93 +2397,101 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
def _apply_prefix_caching(
|
||||
self,
|
||||
local_block_ids: BlockIds,
|
||||
remote_block_ids: BlockIds,
|
||||
remote_physical_per_logical: int,
|
||||
) -> tuple[BlockIds, list]:
|
||||
"""Apply prefix caching by trimming local/remote block ID lists.
|
||||
decode_block_ids: BlockIds,
|
||||
prefill_block_ids: BlockIds,
|
||||
decode_physical_per_logical: int,
|
||||
prefill_physical_per_logical: int,
|
||||
) -> tuple[BlockIds, BlockIds]:
|
||||
"""Trim block ID lists so only the uncomputed suffix is transferred.
|
||||
|
||||
For non-Mamba models: end-trim remote to match local count, so that
|
||||
Inputs are *kernel* (physical) block IDs, already expanded from logical IDs
|
||||
with each side's physical-per-logical ratio. Both pull and push call this after
|
||||
that expansion so the trim happens at kernel granularity.
|
||||
|
||||
The prefix-cache hit is always on the decode (D) side, so ``decode``
|
||||
holds only its uncomputed blocks while prefill (P) holds the full
|
||||
sequence. This is mode-independent: pull passes its own (D) blocks as
|
||||
``decode``, push passes the remote D registration as ``decode``.
|
||||
|
||||
For non-Mamba models: end-trim ``prefill`` to match ``decode`` count, so
|
||||
already-cached prefix blocks are skipped in the transfer.
|
||||
|
||||
For Mamba hybrid (prefix caching not yet supported): front-trim both
|
||||
to the minimum count to handle kernel block count discrepancies from
|
||||
logical block rounding in heterogeneous TP.
|
||||
For Mamba hybrid: SSM groups pair state slots by position and FA
|
||||
groups end-trim to the uncomputed suffix when physical-per-logical
|
||||
matches.
|
||||
"""
|
||||
# Partial prefix cache hit: just read uncomputed blocks.
|
||||
# Partial prefix cache hit: just transfer uncomputed blocks.
|
||||
# Skip mamba groups — their blocks represent full state (conv+ssm),
|
||||
# not per-token data, so trimming would corrupt the transfer.
|
||||
remote_block_ids = list(remote_block_ids)
|
||||
prefill_block_ids = list(prefill_block_ids)
|
||||
if not self._has_mamba:
|
||||
for i, remote_group in enumerate(remote_block_ids):
|
||||
num_local_blocks = len(local_block_ids[i])
|
||||
assert num_local_blocks <= len(remote_group)
|
||||
if num_local_blocks < len(remote_group):
|
||||
remote_block_ids[i] = remote_group[-num_local_blocks:]
|
||||
for i, prefill_group in enumerate(prefill_block_ids):
|
||||
num_decode_blocks = len(decode_block_ids[i])
|
||||
assert num_decode_blocks <= len(prefill_group)
|
||||
if num_decode_blocks < len(prefill_group):
|
||||
prefill_block_ids[i] = prefill_group[-num_decode_blocks:]
|
||||
else:
|
||||
# (NOTE: ZhanqiuHu) Mamba hybrid: no prefix caching support so far.HeteroTP
|
||||
# can cause different kernel block counts due to logical block rounding.
|
||||
# (NOTE: ZhanqiuHu) HeteroTP can cause different kernel block counts
|
||||
# due to logical block rounding.
|
||||
# Example: 640 prompt tokens, kernel_block_size=64
|
||||
# remote physical_per_logical=10, local physical_per_logical=6
|
||||
# remote logical ids from kv_transfer_params = [0]
|
||||
# local logical ids allocated = [0, 1]
|
||||
# remote kernel blocks: [0..9] (1*10=10)
|
||||
# local kernel blocks: [0..11] (2*6=12)
|
||||
# prefill physical_per_logical=10, decode physical_per_logical=6
|
||||
# prefill logical ids from kv_transfer_params = [0]
|
||||
# decode logical ids allocated = [0, 1]
|
||||
# prefill kernel blocks: [0..9] (1*10=10)
|
||||
# decode kernel blocks: [0..11] (2*6=12)
|
||||
# actual data blocks = ceil(640/64) = 10, trim both to 10
|
||||
# Vice versa (remote physical_per_logical=6, local=10):
|
||||
# remote logical ids = [0, 1], local logical ids = [0]
|
||||
# remote kernel blocks: [0..11] (2*6=12)
|
||||
# local kernel blocks: [0..9] (1*10=10)
|
||||
# Vice versa (prefill physical_per_logical=6, decode=10):
|
||||
# prefill logical ids = [0, 1], decode logical ids = [0]
|
||||
# prefill kernel blocks: [0..11] (2*6=12)
|
||||
# decode kernel blocks: [0..9] (1*10=10)
|
||||
# actual data blocks = ceil(640/64) = 10, trim both to 10
|
||||
local_block_ids = list(local_block_ids)
|
||||
for i, remote_group in enumerate(remote_block_ids):
|
||||
num_local_blocks = len(local_block_ids[i])
|
||||
num_remote_blocks = len(remote_group)
|
||||
decode_block_ids = list(decode_block_ids)
|
||||
for i, prefill_group in enumerate(prefill_block_ids):
|
||||
num_decode_blocks = len(decode_block_ids[i])
|
||||
num_prefill_blocks = len(prefill_group)
|
||||
if _is_ssm_spec(self._group_spec_types[i]):
|
||||
if num_local_blocks == num_remote_blocks:
|
||||
if num_decode_blocks == num_prefill_blocks:
|
||||
continue
|
||||
# Only state-bearing slots reach here, single-state modes
|
||||
# just one (see get_exchange_clipped_blocks), so differing
|
||||
# counts mean position-indexed "all"-mode lists. A longer
|
||||
# remote list carries earlier positions the local side
|
||||
# already has (prefix hit) -> read its tail; a longer local
|
||||
# prefill list carries earlier positions the decode side
|
||||
# already has (prefix hit) -> take its tail; a longer decode
|
||||
# list holds the position D recomputes itself, which gets
|
||||
# no remote state.
|
||||
assert num_local_blocks - num_remote_blocks <= 1, (
|
||||
# no prefill state.
|
||||
assert num_decode_blocks - num_prefill_blocks <= 1, (
|
||||
f"Group {i}: unpairable SSM state slots, "
|
||||
f"local={num_local_blocks} remote={num_remote_blocks}"
|
||||
f"decode={num_decode_blocks} prefill={num_prefill_blocks}"
|
||||
)
|
||||
num_blocks = min(num_local_blocks, num_remote_blocks)
|
||||
if num_local_blocks < num_remote_blocks:
|
||||
remote_block_ids[i] = remote_group[-num_blocks:]
|
||||
num_blocks = min(num_decode_blocks, num_prefill_blocks)
|
||||
if num_decode_blocks < num_prefill_blocks:
|
||||
prefill_block_ids[i] = prefill_group[-num_blocks:]
|
||||
else:
|
||||
local_block_ids[i] = local_block_ids[i][:num_blocks]
|
||||
decode_block_ids[i] = decode_block_ids[i][:num_blocks]
|
||||
elif (
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
== remote_physical_per_logical
|
||||
and num_local_blocks < num_remote_blocks
|
||||
decode_physical_per_logical == prefill_physical_per_logical
|
||||
and num_decode_blocks < num_prefill_blocks
|
||||
):
|
||||
# Partial prefix cache hit for FA group.
|
||||
remote_block_ids[i] = remote_group[-num_local_blocks:]
|
||||
prefill_block_ids[i] = prefill_group[-num_decode_blocks:]
|
||||
else:
|
||||
# TODO Handle prefix caching with different block_sizes
|
||||
# Allocation rounding legitimately leaves up to
|
||||
# ppl - 1 trailing dead kernel blocks per side (plus one
|
||||
# extra local block for the recomputed final token), so
|
||||
# extra decode block for the recomputed final token), so
|
||||
# the counts may differ by up to the sum of the two
|
||||
# ratios; anything larger indicates mismatched lists.
|
||||
max_padding = (
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
+ remote_physical_per_logical
|
||||
decode_physical_per_logical + prefill_physical_per_logical
|
||||
)
|
||||
assert abs(num_local_blocks - num_remote_blocks) <= max_padding, (
|
||||
f"Group {i}: |{num_local_blocks} - "
|
||||
f"{num_remote_blocks}| > {max_padding}"
|
||||
assert abs(num_decode_blocks - num_prefill_blocks) <= max_padding, (
|
||||
f"Group {i}: |{num_decode_blocks} - "
|
||||
f"{num_prefill_blocks}| > {max_padding}"
|
||||
)
|
||||
num_blocks = min(num_local_blocks, num_remote_blocks)
|
||||
local_block_ids[i] = local_block_ids[i][:num_blocks]
|
||||
remote_block_ids[i] = remote_group[:num_blocks]
|
||||
return local_block_ids, remote_block_ids
|
||||
num_blocks = min(num_decode_blocks, num_prefill_blocks)
|
||||
decode_block_ids[i] = decode_block_ids[i][:num_blocks]
|
||||
prefill_block_ids[i] = prefill_group[:num_blocks]
|
||||
return decode_block_ids, prefill_block_ids
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""
|
||||
|
||||
@@ -289,9 +289,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
== len(local_block_ids)
|
||||
== len(self.kv_cache_config.kv_cache_groups)
|
||||
)
|
||||
remote_physical_per_logical = remote_info.remote_physical_blocks_per_logical
|
||||
local_block_ids, remote_block_ids = self._apply_prefix_caching(
|
||||
local_block_ids, remote_block_ids, remote_physical_per_logical
|
||||
decode_block_ids=local_block_ids,
|
||||
prefill_block_ids=remote_block_ids,
|
||||
decode_physical_per_logical=self._physical_blocks_per_logical_kv_block,
|
||||
prefill_physical_per_logical=remote_info.remote_physical_blocks_per_logical,
|
||||
)
|
||||
|
||||
# NOTE (nicolo) With homogeneous TP, each TP worker loads KV from
|
||||
|
||||
@@ -454,10 +454,6 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
# how pull-mode transfers touch _engine_last_active in start_load_kv.
|
||||
self._engine_last_active[decode_engine_id] = time.perf_counter()
|
||||
|
||||
# Both sides stay logical here; ``_xfer_blocks_for_req`` converts each
|
||||
# to physical with its own physical-blocks-per-logical ratio -- P uses
|
||||
# ``self._physical_blocks_per_logical_kv_block``, D's is learned during
|
||||
# the NIXL handshake.
|
||||
logical_local = self._as_grouped_block_ids(local_block_ids)
|
||||
logical_remote = self._as_grouped_block_ids(remote_block_ids)
|
||||
physical_local = self._logical_to_kernel_block_ids(
|
||||
@@ -629,16 +625,28 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
logger.warning("No blocks to push for request %s", request_id)
|
||||
return None
|
||||
|
||||
# Align per-group block counts for push.
|
||||
# Prefix caching: D allocated only uncached blocks, so on a partial hit it
|
||||
# sends fewer than P's. End-trim P's blocks to that same suffix so we WRITE only
|
||||
# the uncomputed tail into D's slots. Runs on kernel ids, post-expansion.
|
||||
remote_block_ids, local_block_ids = self._apply_prefix_caching(
|
||||
decode_block_ids=remote_block_ids,
|
||||
prefill_block_ids=local_block_ids,
|
||||
decode_physical_per_logical=remote_info.remote_physical_blocks_per_logical,
|
||||
prefill_physical_per_logical=self._physical_blocks_per_logical_kv_block,
|
||||
)
|
||||
|
||||
local_block_ids = list(local_block_ids)
|
||||
remote_block_ids = list(remote_block_ids)
|
||||
for i in range(min(len(local_block_ids), len(remote_block_ids))):
|
||||
num_local = len(local_block_ids[i])
|
||||
num_remote = len(remote_block_ids[i])
|
||||
if num_local > num_remote:
|
||||
local_block_ids[i] = local_block_ids[i][:num_remote]
|
||||
elif num_local < num_remote:
|
||||
remote_block_ids[i] = remote_block_ids[i][:num_local]
|
||||
assert len(local_block_ids) == len(remote_block_ids), (
|
||||
f"push group-count mismatch for {request_id}: {len(local_block_ids)} "
|
||||
f"local vs {len(remote_block_ids)} remote groups"
|
||||
)
|
||||
for i in range(len(local_block_ids)):
|
||||
assert len(local_block_ids[i]) == len(remote_block_ids[i]), (
|
||||
f"push block-count mismatch for {request_id} group {i}: "
|
||||
f"{len(local_block_ids[i])} local vs "
|
||||
f"{len(remote_block_ids[i])} remote blocks"
|
||||
)
|
||||
|
||||
# Get descs ids.
|
||||
remote_block_descs_ids = self._compute_desc_ids(
|
||||
|
||||
Reference in New Issue
Block a user