[kv_offload] Skip scratch groups (#57145)

Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Itay Etelis
2026-09-17 13:22:49 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 40b40d1d39
commit b6e7c1f1f0
4 changed files with 81 additions and 5 deletions
@@ -19,6 +19,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
from vllm.platforms import current_platform
from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
KVCacheConfig,
@@ -340,6 +341,38 @@ def test_worker_kv_bytes_preserves_tensor_layout(packed: bool):
assert offloading_config.cache.blocks_per_chunk == 2
def test_offloading_skips_scratch_group():
kv_cache_config = _make_mamba_hybrid_kv_cache_config()
kv_cache_config.kv_cache_groups.insert(
1,
KVCacheGroupSpec(
["scratch"],
CircularBufferSpec(
block_size=4, num_kv_heads=1, head_size=128, dtype=torch.float32
),
),
)
page_size = kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes
kv_cache_config.kv_cache_tensors = [
KVCacheTensor(
size=page_size * kv_cache_config.num_blocks,
layers=["full_layer", "scratch", "mamba_layer"],
layer_stride=0,
block_stride=page_size,
)
]
config = _make_vllm_config()
offloading_config = build_offloading_config(config, kv_cache_config)
scheduler_config = SchedulerOffloadConfig.from_spec(
MockOffloadingSpec(offloading_config), config, kv_cache_config
)
assert [group.group_id for group in offloading_config.groups] == [0, 2]
assert [group.group_idx for group in scheduler_config.kv_group_configs] == [0, 2]
assert offloading_config.worker_kv_bytes_per_block == page_size
def test_hisparse_offloads_only_indexer_group():
source = KVCacheGroupSpec(
["source"],
@@ -13,6 +13,7 @@ from vllm.platforms import current_platform
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
@@ -520,6 +521,47 @@ def test_register_kv_caches(backend):
assert actual.mapping is not None
def test_register_packed_kv_caches_skips_scratch_group():
attn_spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
)
page = attn_spec.page_size_bytes
layers = ["layer0", "layer1"]
kv_cache_config = KVCacheConfig(
num_blocks=NUM_BLOCKS,
kv_cache_tensors=[
KVCacheTensor(
size=2 * page * NUM_BLOCKS,
layers=layers,
layer_stride=page,
block_stride=2 * page,
)
],
kv_cache_groups=[
KVCacheGroupSpec(layers, attn_spec),
KVCacheGroupSpec(
["scratch"],
CircularBufferSpec(
block_size=4, num_kv_heads=1, head_size=1, dtype=DTYPE
),
),
],
)
kv_caches = _allocate_kv_caches(
kv_cache_config, [], device=torch.device(f"{DEVICE_TYPE}:0")
)
worker, spec = _make_worker(kv_cache_config)
worker.register_kv_caches(kv_caches)
canonical = spec.get_worker.call_args[0][0]
assert len(canonical.tensors) == 1
assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, 2 * page)
assert canonical.group_data_refs == [[CanonicalKVCacheRef(0, 2 * page)]]
@pytest.mark.parametrize("backend", ATTN_BACKENDS)
def test_register_kv_caches_uniform_type(backend):
"""Test register_kv_caches with UniformTypeKVCacheSpecs.
@@ -35,7 +35,7 @@ if TYPE_CHECKING:
def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]:
if kv_cache_config.hisparse_host_num_blocks is None:
return tuple(range(len(kv_cache_config.kv_cache_groups)))
return kv_cache_config.prefix_cacheable_group_ids
return tuple(
group_id
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
@@ -135,12 +135,12 @@ def build_offloading_config(
)
worker_kv_bytes_per_block = 0
all_groups_selected = len(selected_groups) == len(kv_cache_config.kv_cache_groups)
if (
all_groups_selected
kv_cache_config.hisparse_host_num_blocks is None
and kv_cache_config.num_blocks > 0
and kv_cache_config.kv_cache_tensors
):
# Scratch filtering must preserve the scheduler/worker allocation stride.
# Every KVCacheTensor describes placement within the same backing allocation,
# so its size is the total, not a per-tensor share.
total_gpu_kv_bytes = kv_cache_config.kv_cache_tensors[0].size
@@ -141,8 +141,9 @@ class OffloadingConnectorWorker:
),
None,
)
if packed_layer_name is not None and len(selected_groups) == len(
kv_cache_config.kv_cache_groups
if (
packed_layer_name is not None
and kv_cache_config.hisparse_host_num_blocks is None
):
(tensor,) = tensors_per_block[packed_layer_name]
num_blocks = tensor.shape[0]