[NIXL] Fix after virtual block_size for host_buffer with heter kv_layout (#29122)
Signed-off-by: Chendi Xue <chendi.xue@intel.com>
This commit is contained in:
@@ -1042,10 +1042,12 @@ class NixlConnectorWorker:
|
||||
NOT directly supported by NIXL (e.g., tpu)
|
||||
"""
|
||||
xfer_buffers: dict[str, torch.Tensor] = {}
|
||||
inv_order = [0, 1, 3, 2, 4]
|
||||
try:
|
||||
for layer_name, kv_cache in kv_caches.items():
|
||||
kv_shape = kv_cache.shape
|
||||
kv_dtype = kv_cache.dtype
|
||||
permute_shape = False
|
||||
if (
|
||||
self.kv_cache_layout == "NHD"
|
||||
and self.vllm_config.kv_transfer_config is not None
|
||||
@@ -1059,10 +1061,20 @@ class NixlConnectorWorker:
|
||||
# Since NHD will not support Decode/Prefill TP_ratio > 1,
|
||||
# we can leverage host_buffer for permute
|
||||
self.host_buffer_kv_cache_layout = "HND"
|
||||
kv_shape = tuple(kv_shape[i] for i in [0, 1, 3, 2, 4])
|
||||
kv_shape = (
|
||||
tuple(kv_shape[i] for i in inv_order)
|
||||
if not self.use_mla
|
||||
else kv_shape
|
||||
)
|
||||
permute_shape = not self.use_mla
|
||||
|
||||
xfer_buffers[layer_name] = torch.empty(
|
||||
kv_shape, dtype=kv_dtype, device="cpu"
|
||||
)
|
||||
if permute_shape:
|
||||
xfer_buffers[layer_name] = xfer_buffers[layer_name].permute(
|
||||
inv_order
|
||||
)
|
||||
except MemoryError as e:
|
||||
logger.error("NIXLConnectorWorker gets %s.", e)
|
||||
raise
|
||||
|
||||
Reference in New Issue
Block a user