[kv_offload] Skip scratch groups (#57145)
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -19,6 +19,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KVCacheConfig,
|
||||
@@ -340,6 +341,38 @@ def test_worker_kv_bytes_preserves_tensor_layout(packed: bool):
|
||||
assert offloading_config.cache.blocks_per_chunk == 2
|
||||
|
||||
|
||||
def test_offloading_skips_scratch_group():
|
||||
kv_cache_config = _make_mamba_hybrid_kv_cache_config()
|
||||
kv_cache_config.kv_cache_groups.insert(
|
||||
1,
|
||||
KVCacheGroupSpec(
|
||||
["scratch"],
|
||||
CircularBufferSpec(
|
||||
block_size=4, num_kv_heads=1, head_size=128, dtype=torch.float32
|
||||
),
|
||||
),
|
||||
)
|
||||
page_size = kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes
|
||||
kv_cache_config.kv_cache_tensors = [
|
||||
KVCacheTensor(
|
||||
size=page_size * kv_cache_config.num_blocks,
|
||||
layers=["full_layer", "scratch", "mamba_layer"],
|
||||
layer_stride=0,
|
||||
block_stride=page_size,
|
||||
)
|
||||
]
|
||||
config = _make_vllm_config()
|
||||
|
||||
offloading_config = build_offloading_config(config, kv_cache_config)
|
||||
scheduler_config = SchedulerOffloadConfig.from_spec(
|
||||
MockOffloadingSpec(offloading_config), config, kv_cache_config
|
||||
)
|
||||
|
||||
assert [group.group_id for group in offloading_config.groups] == [0, 2]
|
||||
assert [group.group_idx for group in scheduler_config.kv_group_configs] == [0, 2]
|
||||
assert offloading_config.worker_kv_bytes_per_block == page_size
|
||||
|
||||
|
||||
def test_hisparse_offloads_only_indexer_group():
|
||||
source = KVCacheGroupSpec(
|
||||
["source"],
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -520,6 +521,47 @@ def test_register_kv_caches(backend):
|
||||
assert actual.mapping is not None
|
||||
|
||||
|
||||
def test_register_packed_kv_caches_skips_scratch_group():
|
||||
attn_spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
page = attn_spec.page_size_bytes
|
||||
layers = ["layer0", "layer1"]
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=NUM_BLOCKS,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=2 * page * NUM_BLOCKS,
|
||||
layers=layers,
|
||||
layer_stride=page,
|
||||
block_stride=2 * page,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layers, attn_spec),
|
||||
KVCacheGroupSpec(
|
||||
["scratch"],
|
||||
CircularBufferSpec(
|
||||
block_size=4, num_kv_heads=1, head_size=1, dtype=DTYPE
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
kv_caches = _allocate_kv_caches(
|
||||
kv_cache_config, [], device=torch.device(f"{DEVICE_TYPE}:0")
|
||||
)
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
canonical = spec.get_worker.call_args[0][0]
|
||||
assert len(canonical.tensors) == 1
|
||||
assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, 2 * page)
|
||||
assert canonical.group_data_refs == [[CanonicalKVCacheRef(0, 2 * page)]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ATTN_BACKENDS)
|
||||
def test_register_kv_caches_uniform_type(backend):
|
||||
"""Test register_kv_caches with UniformTypeKVCacheSpecs.
|
||||
|
||||
@@ -35,7 +35,7 @@ if TYPE_CHECKING:
|
||||
|
||||
def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]:
|
||||
if kv_cache_config.hisparse_host_num_blocks is None:
|
||||
return tuple(range(len(kv_cache_config.kv_cache_groups)))
|
||||
return kv_cache_config.prefix_cacheable_group_ids
|
||||
return tuple(
|
||||
group_id
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
@@ -135,12 +135,12 @@ def build_offloading_config(
|
||||
)
|
||||
|
||||
worker_kv_bytes_per_block = 0
|
||||
all_groups_selected = len(selected_groups) == len(kv_cache_config.kv_cache_groups)
|
||||
if (
|
||||
all_groups_selected
|
||||
kv_cache_config.hisparse_host_num_blocks is None
|
||||
and kv_cache_config.num_blocks > 0
|
||||
and kv_cache_config.kv_cache_tensors
|
||||
):
|
||||
# Scratch filtering must preserve the scheduler/worker allocation stride.
|
||||
# Every KVCacheTensor describes placement within the same backing allocation,
|
||||
# so its size is the total, not a per-tensor share.
|
||||
total_gpu_kv_bytes = kv_cache_config.kv_cache_tensors[0].size
|
||||
|
||||
@@ -141,8 +141,9 @@ class OffloadingConnectorWorker:
|
||||
),
|
||||
None,
|
||||
)
|
||||
if packed_layer_name is not None and len(selected_groups) == len(
|
||||
kv_cache_config.kv_cache_groups
|
||||
if (
|
||||
packed_layer_name is not None
|
||||
and kv_cache_config.hisparse_host_num_blocks is None
|
||||
):
|
||||
(tensor,) = tensors_per_block[packed_layer_name]
|
||||
num_blocks = tensor.shape[0]
|
||||
|
||||
Reference in New Issue
Block a user