From b6e7c1f1f0430b5d4784aea391c42067581b7f76 Mon Sep 17 00:00:00 2001 From: Itay Etelis <92247226+Etelis@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:22:49 +0300 Subject: [PATCH] [kv_offload] Skip scratch groups (#57145) Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Or Ozeri --- .../unit/offloading_connector/test_config.py | 33 +++++++++++++++ .../unit/offloading_connector/test_worker.py | 42 +++++++++++++++++++ .../kv_connector/v1/offloading/config.py | 6 +-- .../kv_connector/v1/offloading/worker.py | 5 ++- 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_config.py b/tests/v1/kv_connector/unit/offloading_connector/test_config.py index acf55972df..fe0d258462 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_config.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_config.py @@ -19,6 +19,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( from vllm.platforms import current_platform from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, HiddenStateCacheSpec, KVCacheConfig, @@ -340,6 +341,38 @@ def test_worker_kv_bytes_preserves_tensor_layout(packed: bool): assert offloading_config.cache.blocks_per_chunk == 2 +def test_offloading_skips_scratch_group(): + kv_cache_config = _make_mamba_hybrid_kv_cache_config() + kv_cache_config.kv_cache_groups.insert( + 1, + KVCacheGroupSpec( + ["scratch"], + CircularBufferSpec( + block_size=4, num_kv_heads=1, head_size=128, dtype=torch.float32 + ), + ), + ) + page_size = kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes + kv_cache_config.kv_cache_tensors = [ + KVCacheTensor( + size=page_size * kv_cache_config.num_blocks, + layers=["full_layer", "scratch", "mamba_layer"], + layer_stride=0, + block_stride=page_size, + ) + ] + config = _make_vllm_config() + + offloading_config = build_offloading_config(config, kv_cache_config) + scheduler_config = SchedulerOffloadConfig.from_spec( + MockOffloadingSpec(offloading_config), config, kv_cache_config + ) + + assert [group.group_id for group in offloading_config.groups] == [0, 2] + assert [group.group_idx for group in scheduler_config.kv_group_configs] == [0, 2] + assert offloading_config.worker_kv_bytes_per_block == page_size + + def test_hisparse_offloads_only_indexer_group(): source = KVCacheGroupSpec( ["source"], diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py index 64f481d02e..32bd4f3eca 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py @@ -13,6 +13,7 @@ from vllm.platforms import current_platform from vllm.utils.torch_utils import get_dtype_size from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, @@ -520,6 +521,47 @@ def test_register_kv_caches(backend): assert actual.mapping is not None +def test_register_packed_kv_caches_skips_scratch_group(): + attn_spec = FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + ) + page = attn_spec.page_size_bytes + layers = ["layer0", "layer1"] + kv_cache_config = KVCacheConfig( + num_blocks=NUM_BLOCKS, + kv_cache_tensors=[ + KVCacheTensor( + size=2 * page * NUM_BLOCKS, + layers=layers, + layer_stride=page, + block_stride=2 * page, + ) + ], + kv_cache_groups=[ + KVCacheGroupSpec(layers, attn_spec), + KVCacheGroupSpec( + ["scratch"], + CircularBufferSpec( + block_size=4, num_kv_heads=1, head_size=1, dtype=DTYPE + ), + ), + ], + ) + kv_caches = _allocate_kv_caches( + kv_cache_config, [], device=torch.device(f"{DEVICE_TYPE}:0") + ) + worker, spec = _make_worker(kv_cache_config) + worker.register_kv_caches(kv_caches) + + canonical = spec.get_worker.call_args[0][0] + assert len(canonical.tensors) == 1 + assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, 2 * page) + assert canonical.group_data_refs == [[CanonicalKVCacheRef(0, 2 * page)]] + + @pytest.mark.parametrize("backend", ATTN_BACKENDS) def test_register_kv_caches_uniform_type(backend): """Test register_kv_caches with UniformTypeKVCacheSpecs. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py index ca503d4e67..e20c1189af 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py @@ -35,7 +35,7 @@ if TYPE_CHECKING: def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]: if kv_cache_config.hisparse_host_num_blocks is None: - return tuple(range(len(kv_cache_config.kv_cache_groups))) + return kv_cache_config.prefix_cacheable_group_ids return tuple( group_id for group_id, group in enumerate(kv_cache_config.kv_cache_groups) @@ -135,12 +135,12 @@ def build_offloading_config( ) worker_kv_bytes_per_block = 0 - all_groups_selected = len(selected_groups) == len(kv_cache_config.kv_cache_groups) if ( - all_groups_selected + kv_cache_config.hisparse_host_num_blocks is None and kv_cache_config.num_blocks > 0 and kv_cache_config.kv_cache_tensors ): + # Scratch filtering must preserve the scheduler/worker allocation stride. # Every KVCacheTensor describes placement within the same backing allocation, # so its size is the total, not a per-tensor share. total_gpu_kv_bytes = kv_cache_config.kv_cache_tensors[0].size diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index 5b14e50dda..2bb00d88f0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -141,8 +141,9 @@ class OffloadingConnectorWorker: ), None, ) - if packed_layer_name is not None and len(selected_groups) == len( - kv_cache_config.kv_cache_groups + if ( + packed_layer_name is not None + and kv_cache_config.hisparse_host_num_blocks is None ): (tensor,) = tensors_per_block[packed_layer_name] num_blocks = tensor.shape[0]