[KV Connector] Fix NIXL mamba state pairing for multi-slot block tables (#50153)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -135,6 +135,20 @@ class NixlBaseConnectorScheduler:
|
||||
for n_tokens, block_size in sw_sizes_tokens
|
||||
]
|
||||
|
||||
# Trailing scratch slots that mamba managers co-allocate per request
|
||||
# for speculative decoding; None for non-SSM groups.
|
||||
self._ssm_spec_blocks = [
|
||||
g.kv_cache_spec.num_speculative_blocks
|
||||
if isinstance(g.kv_cache_spec, MambaSpec)
|
||||
else None
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
]
|
||||
# Only "all" mode keeps a state per block position; the other modes
|
||||
# keep a single running state in the last non-speculative slot.
|
||||
self._ssm_state_slots_are_positional = (
|
||||
vllm_config.cache_config.mamba_cache_mode == "all"
|
||||
)
|
||||
|
||||
# Threshold to decide whether to compute kv cache locally
|
||||
# or pull from a remote node: minimum number of remote
|
||||
# tokens to amortize the xfer latencies
|
||||
@@ -218,13 +232,25 @@ class NixlBaseConnectorScheduler:
|
||||
# Clean up empty engines so we don't leak a key when remote dies.
|
||||
del self._heartbeat_by_engine[engine_id]
|
||||
|
||||
def get_sw_clipped_blocks(self, block_ids: BlockIds) -> BlockIds:
|
||||
"""
|
||||
Clip the number of blocks to the sliding window size for each kv cache group
|
||||
that employs SWA.
|
||||
This is necessary because the KV Cache manager initially allocates blocks for
|
||||
the entire sequence length, and successively cleans up blocks that are outside
|
||||
the window prior to the `request_finished_all_groups` hook.
|
||||
def get_exchange_clipped_blocks(
|
||||
self, block_ids: BlockIds, clip_ssm: bool = True
|
||||
) -> BlockIds:
|
||||
"""Clip a request's block lists down to the transferable blocks.
|
||||
|
||||
Sliding-window groups keep only the in-window tail: the KV cache
|
||||
manager allocates blocks for the entire sequence length and cleans up
|
||||
out-of-window blocks only prior to the `request_finished_all_groups`
|
||||
hook.
|
||||
|
||||
SSM groups keep only their state-bearing slots: the trailing
|
||||
speculative scratch slots always go, and in single-state cache modes
|
||||
so does everything before the running state (null placeholders and
|
||||
the previous step's superseded state). "all" mode keeps its remaining
|
||||
slots, which the worker pairs position-wise.
|
||||
|
||||
Use this at every block-id exchange point. Pass ``clip_ssm=False``
|
||||
for per-step partial lists (host-buffer save), where the SSM strip
|
||||
does not apply.
|
||||
"""
|
||||
if len(block_ids) == 0 or not self._is_hma_required:
|
||||
# No blocks to clip eg Full prefix cache hit or not a hybrid model.
|
||||
@@ -235,15 +261,22 @@ class NixlBaseConnectorScheduler:
|
||||
assert len(block_ids) == len(self.blocks_per_sw), (
|
||||
"Number of KV cache groups must match"
|
||||
)
|
||||
# For non-SWA groups, blocks_per_sw is 0 so we return all block_ids unchanged
|
||||
return tuple(
|
||||
[
|
||||
blocks[-self.blocks_per_sw[i] :]
|
||||
if self.blocks_per_sw[i] > 0
|
||||
else blocks
|
||||
for i, blocks in enumerate(block_ids)
|
||||
]
|
||||
)
|
||||
clipped = []
|
||||
for i, blocks in enumerate(block_ids):
|
||||
if n_sw := self.blocks_per_sw[i]:
|
||||
blocks = blocks[-n_sw:]
|
||||
elif (
|
||||
clip_ssm
|
||||
and blocks
|
||||
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
|
||||
):
|
||||
if n_spec := min(n_spec_blocks, len(blocks) - 1):
|
||||
blocks = blocks[:-n_spec]
|
||||
if not self._ssm_state_slots_are_positional:
|
||||
# Never empty: downstream reads that as a full prefix hit.
|
||||
blocks = blocks[-1:]
|
||||
clipped.append(blocks)
|
||||
return tuple(clipped)
|
||||
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
@@ -380,7 +413,9 @@ class NixlBaseConnectorScheduler:
|
||||
req = req_to_save
|
||||
|
||||
assert req.kv_transfer_params is not None
|
||||
clipped_block_id_groups = self.get_sw_clipped_blocks(new_block_id_groups)
|
||||
clipped_block_id_groups = self.get_exchange_clipped_blocks(
|
||||
new_block_id_groups, clip_ssm=False
|
||||
)
|
||||
meta.add_new_req_to_save(
|
||||
request_id=req_id,
|
||||
local_block_ids=clipped_block_id_groups,
|
||||
|
||||
@@ -2439,16 +2439,25 @@ class NixlBaseConnectorWorker:
|
||||
for i, remote_group in enumerate(remote_block_ids):
|
||||
num_local_blocks = len(local_block_ids[i])
|
||||
num_remote_blocks = len(remote_group)
|
||||
if (
|
||||
_is_ssm_spec(self._group_spec_types[i])
|
||||
and num_local_blocks < num_remote_blocks
|
||||
):
|
||||
# NOTE (NickLucche): With prefix caching on SSM, (remote) blocks
|
||||
# prior to the last one are placeholders (null blocks). Mind that
|
||||
# this doesn't really impact transfer, as we only still care about
|
||||
# the last "block", the full in-place state.
|
||||
assert num_local_blocks == 1, "SSM can only have one local block"
|
||||
remote_block_ids[i] = remote_group[-num_local_blocks:]
|
||||
if _is_ssm_spec(self._group_spec_types[i]):
|
||||
if num_local_blocks == num_remote_blocks:
|
||||
continue
|
||||
# Only state-bearing slots reach here, single-state modes
|
||||
# just one (see get_exchange_clipped_blocks), so differing
|
||||
# counts mean position-indexed "all"-mode lists. A longer
|
||||
# remote list carries earlier positions the local side
|
||||
# already has (prefix hit) -> read its tail; a longer local
|
||||
# list holds the position D recomputes itself, which gets
|
||||
# no remote state.
|
||||
assert num_local_blocks - num_remote_blocks <= 1, (
|
||||
f"Group {i}: unpairable SSM state slots, "
|
||||
f"local={num_local_blocks} remote={num_remote_blocks}"
|
||||
)
|
||||
num_blocks = min(num_local_blocks, num_remote_blocks)
|
||||
if num_local_blocks < num_remote_blocks:
|
||||
remote_block_ids[i] = remote_group[-num_blocks:]
|
||||
else:
|
||||
local_block_ids[i] = local_block_ids[i][:num_blocks]
|
||||
elif (
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
== remote_physical_per_logical
|
||||
|
||||
@@ -155,7 +155,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
if num_external_tokens > 0
|
||||
else ()
|
||||
)
|
||||
local_block_ids = self.get_sw_clipped_blocks(
|
||||
local_block_ids = self.get_exchange_clipped_blocks(
|
||||
unhashed_local_block_ids
|
||||
)
|
||||
|
||||
@@ -262,7 +262,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
# trimming down after allocating for the whole sequence length. Empty
|
||||
# blocks are always at the start of the list.
|
||||
# Here we "unpad" blocks to send the actual remote blocks to be read.
|
||||
block_ids = self.get_sw_clipped_blocks(block_ids)
|
||||
block_ids = self.get_exchange_clipped_blocks(block_ids)
|
||||
|
||||
remote_num_tokens = request.num_computed_tokens
|
||||
|
||||
|
||||
@@ -170,7 +170,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
request.request_id,
|
||||
)
|
||||
local_block_ids: BlockIds = blocks.get_unhashed_block_ids_all_groups()
|
||||
local_block_ids = self.get_sw_clipped_blocks(local_block_ids)
|
||||
local_block_ids = self.get_exchange_clipped_blocks(local_block_ids)
|
||||
|
||||
# ``remote_*`` fields are P's coordinates (from D's perspective).
|
||||
# ``decode_*`` fields are D's own info that P needs for the
|
||||
@@ -271,7 +271,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
time.perf_counter() + self._kv_lease_duration
|
||||
)
|
||||
|
||||
block_ids = self.get_sw_clipped_blocks(block_ids)
|
||||
block_ids = self.get_exchange_clipped_blocks(block_ids)
|
||||
remote_num_tokens = request.num_computed_tokens
|
||||
|
||||
# Store finished blocks for worker-level matching with D
|
||||
|
||||
Reference in New Issue
Block a user