[KV Offload] Make compact secondary identity TP-independent (#49858)

Signed-off-by: Change72 <changg@nvidia.com>
Co-authored-by: GPT-5.6 Sol <noreply@openai.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Chang Guo
2026-07-28 07:10:20 +03:00
committed by GitHub
co-authored by GPT-5.6 Sol Or Ozeri
parent fbb1ef6803
commit a8f296083f
4 changed files with 226 additions and 11 deletions
+105
View File
@@ -51,6 +51,7 @@ def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
data_parallel_index=0,
is_parallelism_agnostic=kwargs.get("is_parallelism_agnostic", False),
),
replicated_layout=kwargs.get("replicated_layout", False),
)
spec = MagicMock(spec=OffloadingSpec)
spec.config = config
@@ -205,3 +206,107 @@ def test_parallel_agnostic_separates_persistent_layouts():
assert agnostic.base_path != specific.base_path
assert "parallel_agnostic" not in agnostic.fields
assert specific.fields["parallel_agnostic"] is False
# ---------------------------------------------------------------------------
# replicated_layout: OR'd into parallel-agnostic identity for compact rows
# ---------------------------------------------------------------------------
def test_replicated_layout_collapses_parallel_identity():
shared = dict(
model_name="mla-model",
groups=((16, "mla_layer"),),
replicated_layout=True,
parallel_agnostic=True,
)
tp2 = make_mapper_from_offloading_spec(tp_size=2, world_size=2, rank=1, **shared)
tp4 = make_mapper_from_offloading_spec(tp_size=4, world_size=4, rank=3, **shared)
assert tp2.base_path == tp4.base_path
for fm in (tp2, tp4):
assert fm.fields["tp_size"] == 1
assert fm.fields["pp_size"] == 1
assert fm.fields["pcp_size"] == 1
assert fm.fields["dcp_size"] == 1
assert fm.rank == 0
assert "parallel_agnostic" not in fm.fields
assert fm.fields["replicated_layout"] is True
assert fm.get_file_name(make_offload_key(b"\x01" * 8, 0)).startswith(
f"{fm.base_path}_r0/"
)
def test_replicated_layout_requires_caller_opt_in():
fm = make_mapper_from_offloading_spec(
tp_size=2,
world_size=2,
rank=1,
replicated_layout=True,
parallel_agnostic=False,
)
assert fm.fields["tp_size"] == 2
assert fm.rank == 1
assert fm.fields["parallel_agnostic"] is False
assert "replicated_layout" not in fm.fields
baseline = make_mapper_from_offloading_spec(
tp_size=2,
world_size=2,
rank=1,
replicated_layout=False,
parallel_agnostic=False,
)
assert fm.base_path == baseline.base_path
def test_non_replicated_keeps_parallel_identity():
fm = make_mapper_from_offloading_spec(
tp_size=4,
world_size=4,
rank=2,
replicated_layout=False,
is_parallelism_agnostic=False,
parallel_agnostic=True,
)
assert fm.fields["tp_size"] == 4
assert fm.rank == 2
assert fm.fields["parallel_agnostic"] is False
assert fm.get_file_name(make_offload_key(b"\x02" * 8, 0)).startswith(
f"{fm.base_path}_r2/"
)
def test_replicated_and_parallelism_agnostic_separate_layouts():
shared = dict(
model_name="shared-model",
groups=((16, "layer0"),),
tp_size=2,
world_size=2,
rank=1,
parallel_agnostic=True,
)
via_agnostic = make_mapper_from_offloading_spec(
is_parallelism_agnostic=True,
replicated_layout=False,
**shared,
)
via_replicated = make_mapper_from_offloading_spec(
is_parallelism_agnostic=False,
replicated_layout=True,
**shared,
)
assert via_agnostic.base_path != via_replicated.base_path
assert "replicated_layout" not in via_agnostic.fields
assert via_replicated.fields["replicated_layout"] is True
def test_replicated_layout_run_config_tp_invariant():
shared = dict(
model_name="mla-model",
groups=((16, "mla_layer"),),
replicated_layout=True,
parallel_agnostic=True,
)
tp2 = make_mapper_from_offloading_spec(tp_size=2, world_size=2, rank=0, **shared)
tp4 = make_mapper_from_offloading_spec(tp_size=4, world_size=4, rank=2, **shared)
assert tp2.get_run_config() == tp4.get_run_config()
+61 -5
View File
@@ -52,8 +52,18 @@ _DTYPE: torch.dtype = torch.float32
_CTX = ReqContext(req_id="test")
def _make_offloading_spec(enable_kv_cache_events: bool) -> MagicMock:
def _make_offloading_spec(
enable_kv_cache_events: bool = False,
*,
tp_size: int = 1,
rank: int = 0,
world_size: int | None = None,
replicated_layout: bool = False,
is_parallelism_agnostic: bool = False,
) -> MagicMock:
"""Mock spec with an explicit global KV events flag."""
if world_size is None:
world_size = tp_size
spec = MagicMock()
spec.config = OffloadingConfig(
groups=(),
@@ -64,15 +74,16 @@ def _make_offloading_spec(enable_kv_cache_events: bool) -> MagicMock:
model=OffloadingModelConfig(name="test-model", dtype="float32"),
cache=OffloadingCacheConfig(tokens_per_hash=16, blocks_per_chunk=1),
parallel=OffloadingParallelConfig(
rank=0,
world_size=1,
tp_size=1,
rank=rank,
world_size=world_size,
tp_size=tp_size,
pp_size=1,
pcp_size=1,
dcp_size=1,
data_parallel_index=0,
is_parallelism_agnostic=False,
is_parallelism_agnostic=is_parallelism_agnostic,
),
replicated_layout=replicated_layout,
)
spec.blocks_per_chunk = 1
spec.kv_events_config = OffloadingKVEventsConfig(
@@ -725,3 +736,48 @@ def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
assert not fs_events[0].removed
finally:
tier.shutdown()
def test_fs_tier_cross_tp_round_trip(tmp_path):
"""TP=2 replicated writer and TP=4 reader share namespace and bytes."""
root = str(tmp_path)
writer_tensor = _page_aligned_rand_tensor(4, _BLOCK_ELEMENTS)
expected = writer_tensor[0].clone()
writer = FileSystemTierManager(
offloading_spec=_make_offloading_spec(
tp_size=2, world_size=2, rank=0, replicated_layout=True
),
primary_kv_view=memoryview(writer_tensor.numpy()),
tier_type="fs",
root_dir=root,
n_read_threads=2,
n_write_threads=2,
)
try:
writer.submit_store(make_job(1, [key(7)], [0]))
assert all(r.success for r in drain(writer))
writer_base = writer.file_mapper.base_path
writer_path = writer.file_mapper.get_file_name(key(7))
finally:
writer.shutdown()
reader_tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
reader = FileSystemTierManager(
offloading_spec=_make_offloading_spec(
tp_size=4, world_size=4, rank=3, replicated_layout=True
),
primary_kv_view=memoryview(reader_tensor.numpy()),
tier_type="fs",
root_dir=root,
n_read_threads=2,
n_write_threads=2,
)
try:
assert reader.file_mapper.base_path == writer_base
assert reader.file_mapper.get_file_name(key(7)) == writer_path
assert lookup_and_wait(reader, [key(7)]) == [LookupResult.HIT]
reader.submit_load(make_job(2, [key(7)], [1], is_promotion=True))
assert all(r.success for r in drain(reader))
assert torch.allclose(reader_tensor[1], expected)
finally:
reader.shutdown()
+50 -5
View File
@@ -43,7 +43,17 @@ from vllm.v1.kv_offload.tiering.obj.manager import ObjectStoreSecondaryTierManag
# ---------------------------------------------------------------------------
def _make_offloading_config(enable_kv_cache_events: bool) -> OffloadingConfig:
def _make_offloading_config(
enable_kv_cache_events: bool,
*,
tp_size: int = 1,
rank: int = 0,
world_size: int | None = None,
replicated_layout: bool = False,
is_parallelism_agnostic: bool = False,
) -> OffloadingConfig:
if world_size is None:
world_size = tp_size
return OffloadingConfig(
groups=(),
worker_kv_bytes_per_block=0,
@@ -53,15 +63,16 @@ def _make_offloading_config(enable_kv_cache_events: bool) -> OffloadingConfig:
model=OffloadingModelConfig(name="test/model", dtype="float16"),
cache=OffloadingCacheConfig(tokens_per_hash=16, blocks_per_chunk=1),
parallel=OffloadingParallelConfig(
rank=0,
world_size=1,
tp_size=1,
rank=rank,
world_size=world_size,
tp_size=tp_size,
pp_size=1,
pcp_size=1,
dcp_size=1,
data_parallel_index=0,
is_parallelism_agnostic=False,
is_parallelism_agnostic=is_parallelism_agnostic,
),
replicated_layout=replicated_layout,
)
@@ -617,3 +628,37 @@ class TestObjStoreConfig:
params = cfg.to_nixl_params()
assert params["ca_bundle"] == "/path/to/ca.pem"
assert "access_key" not in params
def test_obj_tier_replicated_layout_collapses_mapper_identity():
"""TP=2 and TP=4 replicated configs share the obj FileMapper namespace."""
tp2_spec = SimpleNamespace(
config=_make_offloading_config(
False, tp_size=2, world_size=2, rank=1, replicated_layout=True
),
kv_events_config=OffloadingKVEventsConfig(
enable_kv_cache_events=False,
self_describing_kv_events=False,
),
)
tp4_spec = SimpleNamespace(
config=_make_offloading_config(
False, tp_size=4, world_size=4, rank=3, replicated_layout=True
),
kv_events_config=OffloadingKVEventsConfig(
enable_kv_cache_events=False,
self_describing_kv_events=False,
),
)
tp2_tier, _ = _make_tier(offloading_spec=tp2_spec)
tp4_tier, _ = _make_tier(offloading_spec=tp4_spec)
try:
assert tp2_tier._file_mapper.base_path == tp4_tier._file_mapper.base_path
assert tp2_tier._file_mapper.rank == 0
assert tp4_tier._file_mapper.rank == 0
assert tp2_tier._file_mapper.get_run_config() == (
tp4_tier._file_mapper.get_run_config()
)
finally:
tp2_tier.shutdown()
tp4_tier.shutdown()
+10 -1
View File
@@ -35,6 +35,7 @@ class FileMapper:
kv_cache_groups: list[dict] | None = None,
inference_engine: str = "vllm",
parallel_agnostic: bool = False,
replicated_layout: bool = False,
):
"""
Initialize the file mapper. Each worker constructs its own, but
@@ -60,6 +61,10 @@ class FileMapper:
}
if not parallel_agnostic:
self.fields["parallel_agnostic"] = False
# Only written when True so existing deployments' hashed fields are
# unchanged (False is the historical default and must not appear).
if replicated_layout:
self.fields["replicated_layout"] = True
self.base_path: str = self._compute_base_path(root_dir, self.fields)
@classmethod
@@ -92,7 +97,11 @@ class FileMapper:
rank=parallel.rank,
dtype=config.model.dtype,
kv_cache_groups=kv_cache_groups,
parallel_agnostic=(parallel_agnostic and parallel.is_parallelism_agnostic),
parallel_agnostic=(
parallel_agnostic
and (parallel.is_parallelism_agnostic or config.replicated_layout)
),
replicated_layout=(parallel_agnostic and config.replicated_layout),
)
def get_file_name(self, key: OffloadKey) -> str: