[KV Connector] Canonical CPU layout for parallelism-agnostic KV offload (#48414)

Signed-off-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
This commit is contained in:
Itay Etelis
2026-08-10 10:44:54 +03:00
committed by GitHub
co-authored by Itay Etelis
parent 31cd109f18
commit 81840a172f
12 changed files with 891 additions and 103 deletions
@@ -47,6 +47,9 @@ def _make_vllm_config(
config.cache_config.cache_dtype = torch.float16
config.model_config.model = "test-model"
config.model_config.use_mla = False
# _full_attention_spec's heads at tp=1: the parallelism-agnostic gate
# requires the head shard to cover the model's KV heads exactly
config.model_config.get_total_num_kv_heads.return_value = 4
world_size = (
tensor_parallel_size * pipeline_parallel_size * prefill_context_parallel_size
)
@@ -603,6 +606,61 @@ def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
assert not _parallelism_agnostic(kv_cache_groups)
def test_canonical_layout_widens_parallelism_agnostic_to_mla():
"""The canonical layout dedups the TP-replicated MLA latent into one
portable copy, so the gate admits MLA — but only when canonical_layout
is requested."""
mla_groups = [KVCacheGroupSpec(["l0"], _mla_spec(head_size=576))]
assert not _parallelism_agnostic(mla_groups)
config = _make_vllm_config(extra_config={"canonical_layout": True})
kv_cache_config = KVCacheConfig(
num_blocks=0,
kv_cache_tensors=[],
kv_cache_groups=mla_groups,
)
offloading_config = build_offloading_config(config, kv_cache_config)
assert offloading_config.parallel.is_parallelism_agnostic
assert offloading_config.canonical_layout
# hybrid groupings stay out: their non-full-attention layers can only
# derive opaque (exact-topology) mappings
hybrid_config = KVCacheConfig(
num_blocks=0,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["l0"], _full_attention_spec()),
KVCacheGroupSpec(["l1"], _full_attention_spec()),
],
)
assert not build_offloading_config(
config, hybrid_config
).parallel.is_parallelism_agnostic
def test_canonical_layout_certifies_v2_model_runner():
"""Canonical bytes are certified per layer against live tensor strides at
registration, so the static gate must not depend on the model-runner
version — the v2 runner is the case the canonical layout exists for."""
kv_cache_config = KVCacheConfig(
num_blocks=0,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["l0"], _full_attention_spec())],
)
config = _make_vllm_config()
config.use_v2_model_runner = True
assert not build_offloading_config(
config, kv_cache_config
).parallel.is_parallelism_agnostic
config = _make_vllm_config(extra_config={"canonical_layout": True})
config.use_v2_model_runner = True
assert build_offloading_config(
config, kv_cache_config
).parallel.is_parallelism_agnostic
def test_parallelism_agnostic_disabled_on_v2_model_runner():
config = _make_vllm_config()
config.use_v2_model_runner = True
@@ -0,0 +1,274 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
import uuid
import numpy as np
import pytest
import torch
from vllm.v1.kv_offload.base import (
CanonicalKVCacheRef,
CanonicalPageMapping,
CopyRun,
GPULoadStoreSpec,
)
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.cpu.gpu_worker import (
SingleDirectionOffloadingHandler,
_build_copy_plan,
_canonical_block_sizes,
_canonical_page_ids,
pin_mmap_region,
)
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
def _ref(mapping: CanonicalPageMapping, tensor_idx: int = 0) -> CanonicalKVCacheRef:
return CanonicalKVCacheRef(
tensor_idx=tensor_idx,
page_size_bytes=mapping.local_page_size_bytes,
mapping=mapping,
)
def _nhd_mapping() -> CanonicalPageMapping:
# 4-token NHD page at tp=4, rank 2: K and V runs of 4x128B fragments
runs = (
CopyRun(0, 256, 128, 4, 128, 512),
CopyRun(512, 2304, 128, 4, 128, 512),
)
return CanonicalPageMapping(4096, 1024, runs, 1, 0, True)
def test_copy_plan_unrolls_runs():
plan = _build_copy_plan(_ref(_nhd_mapping()), gpu_to_cpu=True)
k_dst = [256, 768, 1280, 1792]
assert plan.frag_offsets_src.tolist() == [0, 128, 256, 384, 512, 640, 768, 896]
assert plan.frag_offsets_dst.tolist() == k_dst + [2048 + o for o in k_dst]
assert plan.frag_sizes.tolist() == [128] * 8
assert plan.num_frags == 8
def test_load_direction_swaps_offsets():
store = _build_copy_plan(_ref(_nhd_mapping()), gpu_to_cpu=True)
load = _build_copy_plan(_ref(_nhd_mapping()), gpu_to_cpu=False)
assert np.array_equal(store.frag_offsets_src, load.frag_offsets_dst)
assert np.array_equal(store.frag_offsets_dst, load.frag_offsets_src)
assert np.array_equal(store.frag_sizes, load.frag_sizes)
def test_writer_rotation_matches_is_writer():
# Replicas take turns writing shared canonical pages, keyed by the
# CPU-side canonical page id; the enumeration must agree with is_writer
identity = CopyRun(0, 0, 256, 1, 256, 256)
mapping = CanonicalPageMapping(256, 256, (identity,), 2, 1, True)
ids = _canonical_page_ids(
np.array([3, 7, 9]), blocks_per_chunk=4, count=10, skip_count=2
)
assert ids.tolist() == [14, 15, 28, 29, 30, 31, 36, 37, 38, 39]
mask = ids % mapping.num_writers == mapping.writer_index
assert mask.tolist() == [mapping.is_writer(int(b)) for b in ids]
def test_canonical_block_sizes_take_max_per_tensor():
identity = CopyRun(0, 0, 512, 1, 512, 512)
small = CanonicalPageMapping(2048, 512, (identity,), 1, 0, False)
refs = [[_ref(_nhd_mapping(), 0), _ref(small, 0)], [_ref(small, 1)]]
assert _canonical_block_sizes(refs, 2) == [4096, 2048]
def _tp2_rank_mapping(rank: int) -> CanonicalPageMapping:
# 4-token NHD page, 4 total heads of 64B, tp=2: rank holds 2 heads,
# so K and V each scatter as 4 per-token 128B fragments
runs = (
CopyRun(0, rank * 128, 128, 4, 128, 256),
CopyRun(512, 1024 + rank * 128, 128, 4, 128, 256),
)
return CanonicalPageMapping(2048, 1024, runs, 1, 0, True)
def _whole_page_mapping() -> CanonicalPageMapping:
identity = CopyRun(0, 0, 2048, 1, 2048, 2048)
return CanonicalPageMapping(2048, 2048, (identity,), 1, 0, True)
def _transfer(handler, num_blocks: int, gpu_to_cpu: bool) -> None:
block_ids = list(range(num_blocks))
gpu_spec = GPULoadStoreSpec(
block_ids, group_sizes=(num_blocks,), block_indices=(0,)
)
cpu_spec = CPULoadStoreSpec(block_ids)
src, dst = (gpu_spec, cpu_spec) if gpu_to_cpu else (cpu_spec, gpu_spec)
assert handler.transfer_async(0, src, dst)
deadline = time.time() + 30
while time.time() < deadline:
if handler.get_finished():
return
time.sleep(0.001)
raise TimeoutError("transfer did not complete")
def _canonical_handler(gpu_tensor, cpu_tensor, mapping, gpu_to_cpu):
page = mapping.local_page_size_bytes
refs = [[CanonicalKVCacheRef(tensor_idx=0, page_size_bytes=page, mapping=mapping)]]
return SingleDirectionOffloadingHandler(
gpu_tensors=[gpu_tensor],
cpu_tensors=[cpu_tensor],
blocks_per_chunk=1,
layer_refs_per_group=refs,
gpu_to_cpu=gpu_to_cpu,
canonical_layout=True,
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_gpu_roundtrip_assembles_canonical_page_across_ranks():
"""Two TP2-style rank handlers must scatter into one shared canonical
CPU page such that each rank reloads bit-exact and a whole-page (TP1)
reader sees both shards — the cross-topology contract."""
torch.manual_seed(0)
num_blocks = 4
gpu_rank = [
torch.randint(-128, 128, (num_blocks, 1024), dtype=torch.int8, device="cuda")
for _ in range(2)
]
cpu_canonical = torch.zeros(num_blocks, 2048, dtype=torch.int8, pin_memory=True)
for rank in (0, 1):
store = _canonical_handler(
gpu_rank[rank], cpu_canonical, _tp2_rank_mapping(rank), gpu_to_cpu=True
)
_transfer(store, num_blocks, gpu_to_cpu=True)
torch.accelerator.synchronize()
# independent oracle: replay each rank's runs in numpy
expected = np.zeros((num_blocks, 2048), dtype=np.int8)
for rank in (0, 1):
local = gpu_rank[rank].cpu().numpy()
for run in _tp2_rank_mapping(rank).runs:
for i in range(run.num_fragments):
lo = run.local_offset + i * run.local_stride
co = run.canonical_offset + i * run.canonical_stride
expected[:, co : co + run.fragment_size] = local[
:, lo : lo + run.fragment_size
]
assert np.array_equal(cpu_canonical.numpy(), expected)
# each rank reloads its shard bit-exact
gpu_back = torch.zeros(num_blocks, 1024, dtype=torch.int8, device="cuda")
load = _canonical_handler(
gpu_back, cpu_canonical, _tp2_rank_mapping(0), gpu_to_cpu=False
)
_transfer(load, num_blocks, gpu_to_cpu=False)
torch.accelerator.synchronize()
assert torch.equal(gpu_back, gpu_rank[0])
# a whole-page reader (TP1 topology) sees the assembled page
gpu_full = torch.zeros(num_blocks, 2048, dtype=torch.int8, device="cuda")
load_full = _canonical_handler(
gpu_full, cpu_canonical, _whole_page_mapping(), gpu_to_cpu=False
)
_transfer(load_full, num_blocks, gpu_to_cpu=False)
torch.accelerator.synchronize()
assert torch.equal(gpu_full.cpu(), cpu_canonical)
# 4-token NHD page, 4 total heads of 64 bytes: canonical page holds
# [K: token x head][V: token x head] = 2048 bytes per block
_TOTAL_HEADS = 4
_HEAD_BYTES = 64
_BLOCK_TOKENS = 4
_CANONICAL_PAGE = 2 * _BLOCK_TOKENS * _TOTAL_HEADS * _HEAD_BYTES
def _nhd_shard_mapping(tp: int, rank: int) -> CanonicalPageMapping:
"""Rank's head-shard mapping into the canonical NHD page at the given tp."""
local_heads = _TOTAL_HEADS // tp
frag = local_heads * _HEAD_BYTES
canonical_row = _TOTAL_HEADS * _HEAD_BYTES
k_run = CopyRun(0, rank * frag, frag, _BLOCK_TOKENS, frag, canonical_row)
v_run = CopyRun(
_BLOCK_TOKENS * frag,
_BLOCK_TOKENS * canonical_row + rank * frag,
frag,
_BLOCK_TOKENS,
frag,
canonical_row,
)
return CanonicalPageMapping(
_CANONICAL_PAGE, 2 * _BLOCK_TOKENS * frag, (k_run, v_run), 1, 0, True
)
def _head_shard(full_kv: torch.Tensor, tp: int, rank: int) -> torch.Tensor:
"""This rank's local page rows out of the (blocks, 2, tokens, heads,
head_bytes) ground truth."""
local_heads = _TOTAL_HEADS // tp
shard = full_kv[:, :, :, rank * local_heads : (rank + 1) * local_heads, :]
return shard.reshape(full_kv.shape[0], -1).contiguous()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("writer_tp,reader_tp", [(2, 4), (4, 2), (2, 1), (4, 4)])
def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int):
"""KV written at one tp must be readable at another: writer ranks scatter
head shards into a shared canonical region, reader ranks gather their own
shards, and every reader must see the writers' ground-truth bytes."""
torch.manual_seed(0)
num_blocks = 3
full_kv = torch.randint(
-128,
128,
(num_blocks, 2, _BLOCK_TOKENS, _TOTAL_HEADS, _HEAD_BYTES),
dtype=torch.int8,
)
engine_id = str(uuid.uuid4())
row_stride = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT
assert row_stride >= _CANONICAL_PAGE
regions: list[SharedOffloadRegion] = []
def canonical_view(rank: int, world_size: int) -> torch.Tensor:
region = SharedOffloadRegion(
engine_id=engine_id,
num_blocks=num_blocks,
rank=rank,
kv_bytes_per_block=row_stride,
cpu_page_size=row_stride // world_size,
)
regions.append(region)
# The Triton load path dereferences CPU pointers on the GPU, which is
# only legal on pinned memory; production pins via CPUOffloadingWorker
pin_mmap_region(region)
return region.create_next_canonical_view(_CANONICAL_PAGE)
try:
for rank in range(writer_tp):
store = _canonical_handler(
_head_shard(full_kv, writer_tp, rank).cuda(),
canonical_view(rank, writer_tp),
_nhd_shard_mapping(writer_tp, rank),
gpu_to_cpu=True,
)
_transfer(store, num_blocks, gpu_to_cpu=True)
torch.accelerator.synchronize()
for rank in range(reader_tp):
expected = _head_shard(full_kv, reader_tp, rank)
gpu_out = torch.zeros_like(expected, device="cuda")
load = _canonical_handler(
gpu_out,
canonical_view(rank, reader_tp),
_nhd_shard_mapping(reader_tp, rank),
gpu_to_cpu=False,
)
_transfer(load, num_blocks, gpu_to_cpu=False)
torch.accelerator.synchronize()
assert torch.equal(gpu_out.cpu(), expected), (
f"reader tp={reader_tp} rank={rank} bytes diverge from the "
f"tp={writer_tp} writers' ground truth"
)
finally:
for region in regions:
region.cleanup()
+2 -2
View File
@@ -195,7 +195,7 @@ def test_transfer(
assert finished[0].success
assert finished[0].transfer_size == (
len(gpu_blocks)
* sum([x.page_size_bytes for x in handler.kv_cache_groups_data_refs[0]])
* sum([x.page_size_bytes for x in handler.layer_refs_per_group[0]])
)
assert finished[0].transfer_time > 0
assert finished[0].transfer_time < (time.time() - start_time)
@@ -409,7 +409,7 @@ def test_transfer_multi_group(
expected_bytes = sum(
group_size * sum([x.page_size_bytes for x in data_refs])
for group_size, data_refs in zip(
group_sizes, handler.kv_cache_groups_data_refs
group_sizes, handler.layer_refs_per_group
)
)
assert finished[0].transfer_size == expected_bytes
@@ -173,7 +173,7 @@ def _mp_race_construct_and_write(
kv_bytes_per_block=num_workers * cpu_page_size,
cpu_page_size=cpu_page_size,
)
t = region.create_next_view(cpu_page_size)
t = region.create_next_worker_view(cpu_page_size)
t[:, :] = fill_value
done_queue.put({"rank": rank, "error": None})
cleanup_queue.get() # wait for parent's verification to finish
@@ -190,110 +190,112 @@ def iid():
# ---------------------------------------------------------------------------
# create_next_view — shape, stride and storage offset
# create_next_worker_view — shape, stride and storage offset
# ---------------------------------------------------------------------------
def test_create_next_view_shape_and_stride(iid):
def test_create_next_worker_view_shape_and_stride(iid):
"""Returned tensor must have shape (num_blocks, tensor_page_size) and
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers."""
with _region(iid, num_blocks=4, cpu_page_size=2 * PAGE_SIZE) as r:
t = r.create_next_view(PAGE_SIZE)
t = r.create_next_worker_view(PAGE_SIZE)
assert t.shape == (4, PAGE_SIZE)
# num_workers=1 → row_stride = cpu_page_size
assert t.stride() == (2 * PAGE_SIZE, 1)
del t
def test_create_next_view_storage_offset_rank0(iid):
def test_create_next_worker_view_storage_offset_rank0(iid):
"""rank=0 worker's first tensor must start at byte 0 of the mmap."""
with _region(iid, cpu_page_size=PAGE_SIZE, num_workers=2, rank=0) as r:
t = r.create_next_view(PAGE_SIZE)
t = r.create_next_worker_view(PAGE_SIZE)
assert t.data_ptr() == r._base.data_ptr() # storage_offset == 0
del t
def test_create_next_view_storage_offset_rank1(iid):
def test_create_next_worker_view_storage_offset_rank1(iid):
"""rank=1 worker's first tensor must start cpu_page_size bytes into the mmap."""
with _multi_region(iid, num_workers=2, num_blocks=4) as (r0, r1):
t1 = r1.create_next_view(PAGE_SIZE)
t1 = r1.create_next_worker_view(PAGE_SIZE)
assert t1.data_ptr() == r1._base.data_ptr() + PAGE_SIZE
del t1
def test_create_next_view_row_stride_with_multiple_workers(iid):
def test_create_next_worker_view_row_stride_with_multiple_workers(iid):
"""With num_workers=4, row_stride must be 4 * cpu_page_size."""
with _region(iid, num_blocks=2, num_workers=4) as r:
t = r.create_next_view(PAGE_SIZE)
t = r.create_next_worker_view(PAGE_SIZE)
assert t.stride(0) == 4 * PAGE_SIZE
del t
# ---------------------------------------------------------------------------
# create_next_view — cursor advancement
# create_next_worker_view — cursor advancement
# ---------------------------------------------------------------------------
def test_create_next_view_cursor_advances(iid):
"""Each call to create_next_view must advance _worker_offset by tensor_page_size."""
def test_create_next_worker_view_cursor_advances(iid):
"""Each create_next_worker_view call must advance _worker_offset by
tensor_page_size."""
with _region(iid, cpu_page_size=3 * PAGE_SIZE) as r:
assert r._worker_offset == 0
r.create_next_view(PAGE_SIZE)
r.create_next_worker_view(PAGE_SIZE)
assert r._worker_offset == PAGE_SIZE
r.create_next_view(PAGE_SIZE)
r.create_next_worker_view(PAGE_SIZE)
assert r._worker_offset == 2 * PAGE_SIZE
r.create_next_view(PAGE_SIZE)
r.create_next_worker_view(PAGE_SIZE)
assert r._worker_offset == 3 * PAGE_SIZE # exactly at area end
def test_create_next_view_exact_fill_succeeds(iid):
def test_create_next_worker_view_exact_fill_succeeds(iid):
"""Allocations whose total exactly equals cpu_page_size must all succeed."""
with _region(iid, cpu_page_size=2 * PAGE_SIZE) as r:
r.create_next_view(PAGE_SIZE) # first half
r.create_next_view(PAGE_SIZE) # fills to area end — must not raise
r.create_next_worker_view(PAGE_SIZE) # first half
r.create_next_worker_view(PAGE_SIZE) # fills to area end — must not raise
# ---------------------------------------------------------------------------
# create_next_view — overflow guard
# create_next_worker_view — overflow guard
# ---------------------------------------------------------------------------
def test_create_next_view_single_overflow_raises(iid):
def test_create_next_worker_view_single_overflow_raises(iid):
"""A single allocation larger than cpu_page_size must raise AssertionError."""
with (
_region(iid) as r,
pytest.raises(AssertionError, match="exceeds worker area end"),
):
r.create_next_view(PAGE_SIZE + 1)
r.create_next_worker_view(PAGE_SIZE + 1)
def test_create_next_view_cumulative_overflow_raises(iid):
def test_create_next_worker_view_cumulative_overflow_raises(iid):
"""Successive allocations that cumulatively exceed cpu_page_size must raise."""
with _region(iid, cpu_page_size=2 * PAGE_SIZE) as r:
r.create_next_view(PAGE_SIZE) # ok — half used
r.create_next_view(PAGE_SIZE) # ok — full
r.create_next_worker_view(PAGE_SIZE) # ok — half used
r.create_next_worker_view(PAGE_SIZE) # ok — full
with pytest.raises(AssertionError, match="exceeds worker area end"):
r.create_next_view(1) # one byte too many
r.create_next_worker_view(1) # one byte too many
def test_create_next_view_overflow_does_not_mutate_cursor(iid):
"""A failed create_next_view must leave _worker_offset unchanged."""
def test_create_next_worker_view_overflow_does_not_mutate_cursor(iid):
"""A failed create_next_worker_view must leave _worker_offset unchanged."""
with _region(iid) as r:
offset_before = r._worker_offset
with pytest.raises(AssertionError):
r.create_next_view(PAGE_SIZE + 1)
r.create_next_worker_view(PAGE_SIZE + 1)
assert r._worker_offset == offset_before
# ---------------------------------------------------------------------------
# create_next_view — data correctness and layout
# create_next_worker_view — data correctness and layout
# ---------------------------------------------------------------------------
def test_create_next_view_write_visible_in_raw_mmap(iid):
"""Writes into a create_next_view view must appear at the correct raw mmap offset"""
def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
"""Writes into a create_next_worker_view view must appear at the correct
raw mmap offset"""
with _region(iid, num_blocks=4) as r:
t = r.create_next_view(PAGE_SIZE)
t = r.create_next_worker_view(PAGE_SIZE)
t[2, :] = 42 # write to block row 2
raw = memoryview(r.mmap_obj)
@@ -303,11 +305,11 @@ def test_create_next_view_write_visible_in_raw_mmap(iid):
del raw, t
def test_create_next_view_multi_tensor_layout(iid):
def test_create_next_worker_view_multi_tensor_layout(iid):
"""Two tensors from the same worker land at consecutive byte offsets per row."""
with _region(iid, num_blocks=2, cpu_page_size=2 * PAGE_SIZE) as r:
ta = r.create_next_view(PAGE_SIZE)
tb = r.create_next_view(PAGE_SIZE)
ta = r.create_next_worker_view(PAGE_SIZE)
tb = r.create_next_worker_view(PAGE_SIZE)
ta[:, :] = 1
tb[:, :] = 2
@@ -322,8 +324,8 @@ def test_create_next_view_multi_tensor_layout(iid):
del raw, ta, tb
def test_create_next_view_multiprocess_slots(iid):
"""Each worker process calls create_next_view and writes distinct data;
def test_create_next_worker_view_multiprocess_slots(iid):
"""Each worker process calls create_next_worker_view and writes distinct data;
the parent verifies each slot lands at the correct interleaved offset."""
num_workers = 2
num_blocks = 4
@@ -356,7 +358,7 @@ def test_create_next_view_multiprocess_slots(iid):
)
child.start()
t0 = region.create_next_view(PAGE_SIZE)
t0 = region.create_next_worker_view(PAGE_SIZE)
t0[:, :] = 11
result = done_queue.get(timeout=30)
@@ -379,13 +381,13 @@ def test_create_next_view_multiprocess_slots(iid):
_cleanup_file(region.mmap_path)
def test_create_next_view_worker_isolation(iid):
def test_create_next_worker_view_worker_isolation(iid):
"""Writes by worker 0 must not affect worker 1's slot and vice versa."""
num_workers = 2
num_blocks = 4
with _multi_region(iid, num_workers=num_workers, num_blocks=num_blocks) as regions:
t0 = regions[0].create_next_view(PAGE_SIZE)
t1 = regions[1].create_next_view(PAGE_SIZE)
t0 = regions[0].create_next_worker_view(PAGE_SIZE)
t1 = regions[1].create_next_worker_view(PAGE_SIZE)
t0[:, :] = 11
t1[:, :] = 22
@@ -712,14 +714,14 @@ def test_cleanup_idempotent(iid):
r.cleanup() # must be a no-op
def test_cleanup_after_create_next_view_releases_mmap(iid):
"""cleanup() must close the mmap even after create_next_view was called.
create_next_view returns a view that shares storage with _base; both must be
def test_cleanup_after_create_next_worker_view_releases_mmap(iid):
"""cleanup() must close the mmap even after create_next_worker_view was called.
create_next_worker_view returns a view that shares storage with _base; both must be
released before mmap.close() can succeed."""
r = _make_region(iid)
mmap_obj = r.mmap_obj
t = r.create_next_view(PAGE_SIZE)
t = r.create_next_worker_view(PAGE_SIZE)
del t
r.cleanup()
+17
View File
@@ -52,6 +52,7 @@ def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
is_parallelism_agnostic=kwargs.get("is_parallelism_agnostic", False),
),
replicated_layout=kwargs.get("replicated_layout", False),
canonical_layout=kwargs.get("canonical_layout", False),
)
spec = MagicMock(spec=OffloadingSpec)
spec.config = config
@@ -208,6 +209,22 @@ def test_parallel_agnostic_separates_persistent_layouts():
assert specific.fields["parallel_agnostic"] is False
def test_canonical_layout_changes_storage_namespace():
# Canonical bytes are not interchangeable with the direct layout, so the
# format id must fork the storage namespace.
from vllm.v1.attention.backends.utils import set_kv_cache_layout
set_kv_cache_layout("NHD")
try:
direct = make_mapper_from_offloading_spec()
canonical = make_mapper_from_offloading_spec(canonical_layout=True)
finally:
set_kv_cache_layout(None)
assert "canonical_format" not in direct.fields
assert canonical.fields["canonical_format"] == "v1-nhd"
assert direct.base_path != canonical.base_path
# ---------------------------------------------------------------------------
# replicated_layout: OR'd into parallel-agnostic identity for compact rows
# ---------------------------------------------------------------------------