[NIXL][HeteroTP] Enable KV transfer from HND prefill to NHD decode (#26556)
Signed-off-by: Chendi Xue <chendi.xue@intel.com>
This commit is contained in:
@@ -563,6 +563,7 @@ class NixlConnectorWorker:
|
||||
self.world_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_group = get_tp_group()
|
||||
self.num_blocks = 0
|
||||
self.enable_permute_local_kv = False
|
||||
|
||||
# KV Caches and nixl tracking data.
|
||||
self.device_type = current_platform.device_type
|
||||
@@ -1094,6 +1095,23 @@ class NixlConnectorWorker:
|
||||
is_kv_replicated = self._tp_size[engine_id] // total_num_kv_heads >= 1
|
||||
|
||||
remote_block_len = nixl_agent_meta.block_lens[0]
|
||||
if nixl_agent_meta.kv_cache_layout != self.kv_cache_layout:
|
||||
if (
|
||||
self.vllm_config.kv_transfer_config is not None
|
||||
and self.vllm_config.kv_transfer_config.enable_permute_local_kv
|
||||
and nixl_agent_meta.kv_cache_layout == "HND"
|
||||
):
|
||||
logger.info(
|
||||
"Remote is HND and local is NHD, enabled additional permute "
|
||||
"on local device KV."
|
||||
)
|
||||
self.enable_permute_local_kv = True
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Heterogeneous TP expects same kv_cache_layout. "
|
||||
"Or enable experimental feature to use HND to NHD support by "
|
||||
"setting 'enable_permute_local_kv'=True in --kv-transfer-config."
|
||||
)
|
||||
if self.use_mla or is_kv_replicated:
|
||||
# With replicated KV cache, only the number of blocks can differ.
|
||||
assert self.block_len_per_layer == nixl_agent_meta.block_lens, (
|
||||
@@ -1114,7 +1132,10 @@ class NixlConnectorWorker:
|
||||
remote_block_size //= 2
|
||||
if tp_ratio > 1:
|
||||
# Heterogeneous TP expects same kv_cache_layout.
|
||||
assert nixl_agent_meta.kv_cache_layout == self.kv_cache_layout
|
||||
if nixl_agent_meta.kv_cache_layout == "NHD":
|
||||
raise ValueError(
|
||||
"Heterogeneous TP is not supported for remote with NHD."
|
||||
)
|
||||
if self.device_type == "xpu":
|
||||
raise ValueError("Heterogeneous TP is not supported on XPU")
|
||||
|
||||
@@ -1226,6 +1247,41 @@ class NixlConnectorWorker:
|
||||
"d2h",
|
||||
)
|
||||
|
||||
def permute_device_kv(self, block_ids: list[int]):
|
||||
"""Transforms the layout of received KV cache blocks to the local format.
|
||||
|
||||
This method corrects layout mismatches from direct memory copies by
|
||||
permuting the tensor dimensions.
|
||||
|
||||
- **Source Layout:** `[num_blocks, n_kv_head, block_size, head_dim]`
|
||||
- **Target Layout:** `[num_blocks, block_size, n_kv_head, head_dim]`
|
||||
|
||||
Args:
|
||||
block_ids: A list of block IDs to update and permute.
|
||||
|
||||
Implementation:
|
||||
- x = blocks_to_update.reshape(src_shape) # view local kv with sender layout
|
||||
- permuted_blocks = x.permute(*inv_order) # transpose n_kv_heads, block_size
|
||||
- cache.index_copy_(0, indices, permuted_blocks) # copy permuted kv back
|
||||
|
||||
"""
|
||||
split_k_and_v = not (self.use_mla or self._use_pallas or self._use_flashinfer)
|
||||
inv_order = [0, 2, 1, 3]
|
||||
sample_cache = list(self.device_kv_caches.values())[0][0]
|
||||
target_shape = list(sample_cache.shape)
|
||||
target_shape[0] = -1
|
||||
src_shape = tuple(target_shape[i] for i in inv_order)
|
||||
indices = torch.tensor(block_ids, device=sample_cache.device)
|
||||
|
||||
for _, cache_or_caches in self.device_kv_caches.items():
|
||||
cache_list = cache_or_caches if split_k_and_v else [cache_or_caches]
|
||||
for cache in cache_list:
|
||||
blocks_to_update = cache.index_select(0, indices)
|
||||
permuted_blocks = blocks_to_update.reshape(src_shape).permute(
|
||||
*inv_order
|
||||
)
|
||||
cache.index_copy_(0, indices, permuted_blocks)
|
||||
|
||||
def get_finished(self) -> tuple[set[str], set[str]]:
|
||||
"""
|
||||
Get requests that are done sending or recving on this specific worker.
|
||||
@@ -1273,6 +1329,15 @@ class NixlConnectorWorker:
|
||||
del self._reqs_to_send[req_id]
|
||||
done_sending.add(req_id)
|
||||
|
||||
if self.enable_permute_local_kv and len(done_recving) > 0:
|
||||
block_ids = []
|
||||
for req_id in done_recving:
|
||||
meta = self._recving_metadata.pop(req_id)
|
||||
assert meta, f"{req_id} not found in recving_metadata list"
|
||||
block_ids += meta.local_block_ids
|
||||
|
||||
self.permute_device_kv(block_ids)
|
||||
|
||||
return done_sending, done_recving
|
||||
|
||||
def _get_new_notifs(self) -> set[str]:
|
||||
|
||||
Reference in New Issue
Block a user