[KV Offload] Make compact secondary identity TP-independent (#49858)
Signed-off-by: Change72 <changg@nvidia.com> Co-authored-by: GPT-5.6 Sol <noreply@openai.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
GPT-5.6 Sol
Or Ozeri
parent
fbb1ef6803
commit
a8f296083f
@@ -51,6 +51,7 @@ def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
|
||||
data_parallel_index=0,
|
||||
is_parallelism_agnostic=kwargs.get("is_parallelism_agnostic", False),
|
||||
),
|
||||
replicated_layout=kwargs.get("replicated_layout", False),
|
||||
)
|
||||
spec = MagicMock(spec=OffloadingSpec)
|
||||
spec.config = config
|
||||
@@ -205,3 +206,107 @@ def test_parallel_agnostic_separates_persistent_layouts():
|
||||
assert agnostic.base_path != specific.base_path
|
||||
assert "parallel_agnostic" not in agnostic.fields
|
||||
assert specific.fields["parallel_agnostic"] is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# replicated_layout: OR'd into parallel-agnostic identity for compact rows
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_replicated_layout_collapses_parallel_identity():
|
||||
shared = dict(
|
||||
model_name="mla-model",
|
||||
groups=((16, "mla_layer"),),
|
||||
replicated_layout=True,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
tp2 = make_mapper_from_offloading_spec(tp_size=2, world_size=2, rank=1, **shared)
|
||||
tp4 = make_mapper_from_offloading_spec(tp_size=4, world_size=4, rank=3, **shared)
|
||||
|
||||
assert tp2.base_path == tp4.base_path
|
||||
for fm in (tp2, tp4):
|
||||
assert fm.fields["tp_size"] == 1
|
||||
assert fm.fields["pp_size"] == 1
|
||||
assert fm.fields["pcp_size"] == 1
|
||||
assert fm.fields["dcp_size"] == 1
|
||||
assert fm.rank == 0
|
||||
assert "parallel_agnostic" not in fm.fields
|
||||
assert fm.fields["replicated_layout"] is True
|
||||
assert fm.get_file_name(make_offload_key(b"\x01" * 8, 0)).startswith(
|
||||
f"{fm.base_path}_r0/"
|
||||
)
|
||||
|
||||
|
||||
def test_replicated_layout_requires_caller_opt_in():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
world_size=2,
|
||||
rank=1,
|
||||
replicated_layout=True,
|
||||
parallel_agnostic=False,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 2
|
||||
assert fm.rank == 1
|
||||
assert fm.fields["parallel_agnostic"] is False
|
||||
assert "replicated_layout" not in fm.fields
|
||||
baseline = make_mapper_from_offloading_spec(
|
||||
tp_size=2,
|
||||
world_size=2,
|
||||
rank=1,
|
||||
replicated_layout=False,
|
||||
parallel_agnostic=False,
|
||||
)
|
||||
assert fm.base_path == baseline.base_path
|
||||
|
||||
|
||||
def test_non_replicated_keeps_parallel_identity():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
tp_size=4,
|
||||
world_size=4,
|
||||
rank=2,
|
||||
replicated_layout=False,
|
||||
is_parallelism_agnostic=False,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
assert fm.fields["tp_size"] == 4
|
||||
assert fm.rank == 2
|
||||
assert fm.fields["parallel_agnostic"] is False
|
||||
assert fm.get_file_name(make_offload_key(b"\x02" * 8, 0)).startswith(
|
||||
f"{fm.base_path}_r2/"
|
||||
)
|
||||
|
||||
|
||||
def test_replicated_and_parallelism_agnostic_separate_layouts():
|
||||
shared = dict(
|
||||
model_name="shared-model",
|
||||
groups=((16, "layer0"),),
|
||||
tp_size=2,
|
||||
world_size=2,
|
||||
rank=1,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
via_agnostic = make_mapper_from_offloading_spec(
|
||||
is_parallelism_agnostic=True,
|
||||
replicated_layout=False,
|
||||
**shared,
|
||||
)
|
||||
via_replicated = make_mapper_from_offloading_spec(
|
||||
is_parallelism_agnostic=False,
|
||||
replicated_layout=True,
|
||||
**shared,
|
||||
)
|
||||
assert via_agnostic.base_path != via_replicated.base_path
|
||||
assert "replicated_layout" not in via_agnostic.fields
|
||||
assert via_replicated.fields["replicated_layout"] is True
|
||||
|
||||
|
||||
def test_replicated_layout_run_config_tp_invariant():
|
||||
shared = dict(
|
||||
model_name="mla-model",
|
||||
groups=((16, "mla_layer"),),
|
||||
replicated_layout=True,
|
||||
parallel_agnostic=True,
|
||||
)
|
||||
tp2 = make_mapper_from_offloading_spec(tp_size=2, world_size=2, rank=0, **shared)
|
||||
tp4 = make_mapper_from_offloading_spec(tp_size=4, world_size=4, rank=2, **shared)
|
||||
assert tp2.get_run_config() == tp4.get_run_config()
|
||||
|
||||
@@ -52,8 +52,18 @@ _DTYPE: torch.dtype = torch.float32
|
||||
_CTX = ReqContext(req_id="test")
|
||||
|
||||
|
||||
def _make_offloading_spec(enable_kv_cache_events: bool) -> MagicMock:
|
||||
def _make_offloading_spec(
|
||||
enable_kv_cache_events: bool = False,
|
||||
*,
|
||||
tp_size: int = 1,
|
||||
rank: int = 0,
|
||||
world_size: int | None = None,
|
||||
replicated_layout: bool = False,
|
||||
is_parallelism_agnostic: bool = False,
|
||||
) -> MagicMock:
|
||||
"""Mock spec with an explicit global KV events flag."""
|
||||
if world_size is None:
|
||||
world_size = tp_size
|
||||
spec = MagicMock()
|
||||
spec.config = OffloadingConfig(
|
||||
groups=(),
|
||||
@@ -64,15 +74,16 @@ def _make_offloading_spec(enable_kv_cache_events: bool) -> MagicMock:
|
||||
model=OffloadingModelConfig(name="test-model", dtype="float32"),
|
||||
cache=OffloadingCacheConfig(tokens_per_hash=16, blocks_per_chunk=1),
|
||||
parallel=OffloadingParallelConfig(
|
||||
rank=0,
|
||||
world_size=1,
|
||||
tp_size=1,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
tp_size=tp_size,
|
||||
pp_size=1,
|
||||
pcp_size=1,
|
||||
dcp_size=1,
|
||||
data_parallel_index=0,
|
||||
is_parallelism_agnostic=False,
|
||||
is_parallelism_agnostic=is_parallelism_agnostic,
|
||||
),
|
||||
replicated_layout=replicated_layout,
|
||||
)
|
||||
spec.blocks_per_chunk = 1
|
||||
spec.kv_events_config = OffloadingKVEventsConfig(
|
||||
@@ -725,3 +736,48 @@ def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
|
||||
assert not fs_events[0].removed
|
||||
finally:
|
||||
tier.shutdown()
|
||||
|
||||
|
||||
def test_fs_tier_cross_tp_round_trip(tmp_path):
|
||||
"""TP=2 replicated writer and TP=4 reader share namespace and bytes."""
|
||||
root = str(tmp_path)
|
||||
writer_tensor = _page_aligned_rand_tensor(4, _BLOCK_ELEMENTS)
|
||||
expected = writer_tensor[0].clone()
|
||||
writer = FileSystemTierManager(
|
||||
offloading_spec=_make_offloading_spec(
|
||||
tp_size=2, world_size=2, rank=0, replicated_layout=True
|
||||
),
|
||||
primary_kv_view=memoryview(writer_tensor.numpy()),
|
||||
tier_type="fs",
|
||||
root_dir=root,
|
||||
n_read_threads=2,
|
||||
n_write_threads=2,
|
||||
)
|
||||
try:
|
||||
writer.submit_store(make_job(1, [key(7)], [0]))
|
||||
assert all(r.success for r in drain(writer))
|
||||
writer_base = writer.file_mapper.base_path
|
||||
writer_path = writer.file_mapper.get_file_name(key(7))
|
||||
finally:
|
||||
writer.shutdown()
|
||||
|
||||
reader_tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
|
||||
reader = FileSystemTierManager(
|
||||
offloading_spec=_make_offloading_spec(
|
||||
tp_size=4, world_size=4, rank=3, replicated_layout=True
|
||||
),
|
||||
primary_kv_view=memoryview(reader_tensor.numpy()),
|
||||
tier_type="fs",
|
||||
root_dir=root,
|
||||
n_read_threads=2,
|
||||
n_write_threads=2,
|
||||
)
|
||||
try:
|
||||
assert reader.file_mapper.base_path == writer_base
|
||||
assert reader.file_mapper.get_file_name(key(7)) == writer_path
|
||||
assert lookup_and_wait(reader, [key(7)]) == [LookupResult.HIT]
|
||||
reader.submit_load(make_job(2, [key(7)], [1], is_promotion=True))
|
||||
assert all(r.success for r in drain(reader))
|
||||
assert torch.allclose(reader_tensor[1], expected)
|
||||
finally:
|
||||
reader.shutdown()
|
||||
|
||||
@@ -43,7 +43,17 @@ from vllm.v1.kv_offload.tiering.obj.manager import ObjectStoreSecondaryTierManag
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_offloading_config(enable_kv_cache_events: bool) -> OffloadingConfig:
|
||||
def _make_offloading_config(
|
||||
enable_kv_cache_events: bool,
|
||||
*,
|
||||
tp_size: int = 1,
|
||||
rank: int = 0,
|
||||
world_size: int | None = None,
|
||||
replicated_layout: bool = False,
|
||||
is_parallelism_agnostic: bool = False,
|
||||
) -> OffloadingConfig:
|
||||
if world_size is None:
|
||||
world_size = tp_size
|
||||
return OffloadingConfig(
|
||||
groups=(),
|
||||
worker_kv_bytes_per_block=0,
|
||||
@@ -53,15 +63,16 @@ def _make_offloading_config(enable_kv_cache_events: bool) -> OffloadingConfig:
|
||||
model=OffloadingModelConfig(name="test/model", dtype="float16"),
|
||||
cache=OffloadingCacheConfig(tokens_per_hash=16, blocks_per_chunk=1),
|
||||
parallel=OffloadingParallelConfig(
|
||||
rank=0,
|
||||
world_size=1,
|
||||
tp_size=1,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
tp_size=tp_size,
|
||||
pp_size=1,
|
||||
pcp_size=1,
|
||||
dcp_size=1,
|
||||
data_parallel_index=0,
|
||||
is_parallelism_agnostic=False,
|
||||
is_parallelism_agnostic=is_parallelism_agnostic,
|
||||
),
|
||||
replicated_layout=replicated_layout,
|
||||
)
|
||||
|
||||
|
||||
@@ -617,3 +628,37 @@ class TestObjStoreConfig:
|
||||
params = cfg.to_nixl_params()
|
||||
assert params["ca_bundle"] == "/path/to/ca.pem"
|
||||
assert "access_key" not in params
|
||||
|
||||
|
||||
def test_obj_tier_replicated_layout_collapses_mapper_identity():
|
||||
"""TP=2 and TP=4 replicated configs share the obj FileMapper namespace."""
|
||||
tp2_spec = SimpleNamespace(
|
||||
config=_make_offloading_config(
|
||||
False, tp_size=2, world_size=2, rank=1, replicated_layout=True
|
||||
),
|
||||
kv_events_config=OffloadingKVEventsConfig(
|
||||
enable_kv_cache_events=False,
|
||||
self_describing_kv_events=False,
|
||||
),
|
||||
)
|
||||
tp4_spec = SimpleNamespace(
|
||||
config=_make_offloading_config(
|
||||
False, tp_size=4, world_size=4, rank=3, replicated_layout=True
|
||||
),
|
||||
kv_events_config=OffloadingKVEventsConfig(
|
||||
enable_kv_cache_events=False,
|
||||
self_describing_kv_events=False,
|
||||
),
|
||||
)
|
||||
tp2_tier, _ = _make_tier(offloading_spec=tp2_spec)
|
||||
tp4_tier, _ = _make_tier(offloading_spec=tp4_spec)
|
||||
try:
|
||||
assert tp2_tier._file_mapper.base_path == tp4_tier._file_mapper.base_path
|
||||
assert tp2_tier._file_mapper.rank == 0
|
||||
assert tp4_tier._file_mapper.rank == 0
|
||||
assert tp2_tier._file_mapper.get_run_config() == (
|
||||
tp4_tier._file_mapper.get_run_config()
|
||||
)
|
||||
finally:
|
||||
tp2_tier.shutdown()
|
||||
tp4_tier.shutdown()
|
||||
|
||||
@@ -35,6 +35,7 @@ class FileMapper:
|
||||
kv_cache_groups: list[dict] | None = None,
|
||||
inference_engine: str = "vllm",
|
||||
parallel_agnostic: bool = False,
|
||||
replicated_layout: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the file mapper. Each worker constructs its own, but
|
||||
@@ -60,6 +61,10 @@ class FileMapper:
|
||||
}
|
||||
if not parallel_agnostic:
|
||||
self.fields["parallel_agnostic"] = False
|
||||
# Only written when True so existing deployments' hashed fields are
|
||||
# unchanged (False is the historical default and must not appear).
|
||||
if replicated_layout:
|
||||
self.fields["replicated_layout"] = True
|
||||
self.base_path: str = self._compute_base_path(root_dir, self.fields)
|
||||
|
||||
@classmethod
|
||||
@@ -92,7 +97,11 @@ class FileMapper:
|
||||
rank=parallel.rank,
|
||||
dtype=config.model.dtype,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
parallel_agnostic=(parallel_agnostic and parallel.is_parallelism_agnostic),
|
||||
parallel_agnostic=(
|
||||
parallel_agnostic
|
||||
and (parallel.is_parallelism_agnostic or config.replicated_layout)
|
||||
),
|
||||
replicated_layout=(parallel_agnostic and config.replicated_layout),
|
||||
)
|
||||
|
||||
def get_file_name(self, key: OffloadKey) -> str:
|
||||
|
||||
Reference in New Issue
Block a user