From 798185d438c030f9b4fd62687440889e7b195251 Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Sat, 27 Jun 2026 21:01:45 -0400 Subject: [PATCH] [KV-Offloading] Fix tensors_per_block stride (#46888) Signed-off-by: <> Co-authored-by: Varun Sundar Rabindranath --- .../unit/offloading_connector/utils.py | 12 +++++++++++- .../kv_connector/v1/offloading/worker.py | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/utils.py b/tests/v1/kv_connector/unit/offloading_connector/utils.py index 482a2f25a5..c2884649bd 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/utils.py +++ b/tests/v1/kv_connector/unit/offloading_connector/utils.py @@ -41,6 +41,7 @@ from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, + KVCacheTensor, ) from vllm.v1.kv_offload.base import ( CanonicalKVCaches, @@ -241,9 +242,18 @@ class RequestRunner: ) ] + kv_cache_tensors = [ + KVCacheTensor( + size=group.kv_cache_spec.page_size_bytes * num_gpu_blocks, + shared_by=[layer_name], + ) + for group in kv_cache_groups + for layer_name in group.layer_names + ] + kv_cache_config = KVCacheConfig( num_blocks=num_gpu_blocks, - kv_cache_tensors=[], + kv_cache_tensors=kv_cache_tensors, kv_cache_groups=kv_cache_groups, ) vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index 254e0dec09..29914e7388 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -53,6 +53,16 @@ class OffloadingConnectorWorker: kv_cache_config = self.spec.kv_cache_config num_blocks = kv_cache_config.num_blocks + # Packed layouts (e.g. DSv4) set block_stride > 0; their tensors use + # stride(0) as the manager-block stride (equals total_num_bytes_per_block). + # General (non-packed) layouts size the tensor at page_size_bytes per + # manager block, so page_size_bytes is the correct offloading stride. + layer_is_packed: dict[str, bool] = { + ln: bool(kv_tensor.block_stride) + for kv_tensor in kv_cache_config.kv_cache_tensors + for ln in kv_tensor.shared_by + } + # layer_name -> (num_blocks, page_size_bytes) tensor tensors_per_block: dict[str, tuple[torch.Tensor, ...]] = {} # layer_name -> size of (un-padded) page in bytes @@ -77,7 +87,11 @@ class OffloadingConnectorWorker: page = layer_kv_cache_spec.page_size_bytes elem_size = layer_kv_cache.element_size() byte_offset = layer_kv_cache.storage_offset() * elem_size - block_stride_bytes = layer_kv_cache.stride(0) * elem_size + block_stride_bytes = ( + layer_kv_cache.stride(0) * elem_size + if layer_is_packed[layer_name] + else page + ) tensors_per_block[layer_name] = ( torch.tensor( [],