[PD][Nixl] Mamba prefix caching mode support (#42554)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -2333,9 +2333,25 @@ class NixlConnectorWorker:
|
||||
for i, remote_group in enumerate(remote_block_ids):
|
||||
num_local_blocks = len(local_block_ids[i])
|
||||
num_remote_blocks = len(remote_group)
|
||||
if _is_ssm_spec(self._group_spec_types[i]):
|
||||
assert num_local_blocks == num_remote_blocks
|
||||
if (
|
||||
_is_ssm_spec(self._group_spec_types[i])
|
||||
and num_local_blocks < num_remote_blocks
|
||||
):
|
||||
# NOTE (NickLucche): With prefix caching on SSM, (remote) blocks
|
||||
# prior to the last one are placeholders (null blocks). Mind that
|
||||
# this doesn't really impact transfer, as we only still care about
|
||||
# the last "block", the full in-place state.
|
||||
assert num_local_blocks == 1, "SSM can only have one local block"
|
||||
remote_block_ids[i] = remote_group[-num_local_blocks:]
|
||||
elif (
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
== remote_physical_per_logical
|
||||
and num_local_blocks < num_remote_blocks
|
||||
):
|
||||
# Partial prefix cache hit for FA group.
|
||||
remote_block_ids[i] = remote_group[-num_local_blocks:]
|
||||
else:
|
||||
# TODO Handle prefix caching with different block_sizes
|
||||
max_padding = max(
|
||||
self._physical_blocks_per_logical_kv_block,
|
||||
remote_physical_per_logical,
|
||||
|
||||
Reference in New Issue
Block a user