From 7b942936276d59cc1912185a442b97dea8d2386c Mon Sep 17 00:00:00 2001 From: Ren Yuzhou <54501155+yuzhouo7@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:35:23 +0800 Subject: [PATCH] [Bugfix][ROCm][KV Offload] Use private pinned tensors for CPU KV offload (#57160) Signed-off-by: Yuzhou Ren --- tests/v1/kv_offload/test_factory.py | 49 +++++++++++++++++++---------- vllm/v1/kv_offload/cpu/spec.py | 2 +- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/tests/v1/kv_offload/test_factory.py b/tests/v1/kv_offload/test_factory.py index 16539677c4..01ef756cae 100644 --- a/tests/v1/kv_offload/test_factory.py +++ b/tests/v1/kv_offload/test_factory.py @@ -275,11 +275,12 @@ def test_tiering_spec_create_worker_folds_device_index_for_sharded_layout(monkey @pytest.mark.parametrize("world_size", [2, 4, 8]) def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: int): - # On shared-region (CUDA-alike) platforms the default spec now honors + # On CUDA the default spec honors # replicated layout: a single MLA copy (num_copies=1), matching tiering. import vllm.v1.kv_offload.cpu.spec as cpu_spec_module monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * 8, @@ -295,19 +296,19 @@ def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: in assert spec.num_chunks == 8 +@pytest.mark.parametrize("rocm", [False, True], ids=["non-cuda-alike", "rocm"]) @pytest.mark.parametrize("world_size", [2, 4, 8]) def test_cpu_spec_replicated_disabled_without_shared_region( - monkeypatch, world_size: int + monkeypatch, world_size: int, rocm: bool ): - # Data-loss guard: non-CUDA-alike platforms keep a per-rank private pinned + # Data-loss guard: platforms without shared regions keep a private pinned # tensor (no shared medium), so replicated layout MUST stay off. Otherwise # the rank-0 writer gate would ack rank>0 stores without writing, leaving # those private buffers empty and corrupting subsequent loads. import vllm.v1.kv_offload.cpu.spec as cpu_spec_module - monkeypatch.setattr( - cpu_spec_module.current_platform, "is_cuda_alike", lambda: False - ) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: rocm) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: rocm) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * world_size * 2, @@ -324,17 +325,26 @@ def test_cpu_spec_replicated_disabled_without_shared_region( @pytest.mark.parametrize("config_replicated", [True, False]) -@pytest.mark.parametrize("cuda_alike", [True, False]) +@pytest.mark.parametrize( + ("cuda_alike", "rocm", "shared_region"), + [(True, False, True), (True, True, False), (False, False, False)], + ids=["cuda", "rocm", "non-cuda-alike"], +) def test_cpu_spec_replicated_layout_truth_matrix( - monkeypatch, cuda_alike: bool, config_replicated: bool + monkeypatch, + cuda_alike: bool, + rocm: bool, + shared_region: bool, + config_replicated: bool, ): # replicated_layout is enabled iff the config gate passes AND the deployment - # actually allocates on the shared region (CUDA-alike). + # actually allocates on the shared region. import vllm.v1.kv_offload.cpu.spec as cpu_spec_module monkeypatch.setattr( cpu_spec_module.current_platform, "is_cuda_alike", lambda: cuda_alike ) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: rocm) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * 8, @@ -344,10 +354,10 @@ def test_cpu_spec_replicated_layout_truth_matrix( ) assert isinstance(spec, CPUOffloadingSpec) - assert spec.replicated_layout is (cuda_alike and config_replicated) + assert spec.replicated_layout is (shared_region and config_replicated) -def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch): +def test_cpu_spec_create_worker_uses_mmap_on_cuda(monkeypatch): import vllm.v1.kv_offload.cpu.spec as cpu_spec_module worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT @@ -371,6 +381,7 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch): return MagicMock() monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) monkeypatch.setattr(cpu_spec_module, "SharedOffloadRegion", fake_region_ctor) monkeypatch.setattr(cpu_spec_module, "CPUOffloadingWorker", fake_worker_ctor) monkeypatch.setattr( @@ -386,7 +397,10 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch): assert worker_calls[0]["mmap_region"] is region -def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch): +@pytest.mark.parametrize("rocm", [False, True], ids=["non-cuda-alike", "rocm"]) +def test_cpu_spec_create_worker_uses_tensor_path_without_shared_region( + monkeypatch, rocm +): import vllm.v1.kv_offload.cpu.spec as cpu_spec_module spec = _create_spec(worker_kv_bytes_per_block=4096, world_size=4) @@ -403,15 +417,14 @@ def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch): worker_calls.append(kwargs) return MagicMock() - monkeypatch.setattr( - cpu_spec_module.current_platform, "is_cuda_alike", lambda: False - ) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: rocm) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: rocm) monkeypatch.setattr(cpu_spec_module, "SharedOffloadRegion", fake_region_ctor) monkeypatch.setattr(cpu_spec_module, "CPUOffloadingWorker", fake_worker_ctor) spec.create_worker(MagicMock()) - # Non-CUDA-alike platforms keep the per-rank pinned-tensor path. + # ROCm and non-CUDA-alike platforms use the per-rank pinned-tensor path. assert region_calls == [] assert worker_calls[0]["mmap_region"] is None @@ -420,7 +433,7 @@ def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch): import vllm.v1.kv_offload.cpu.spec as cpu_spec_module # worker_kv_bytes_per_block=0 yields num_chunks=0; a zero-byte region cannot - # be mmap'd, so even on CUDA-alike this must fall back to the tensor path. + # be mmap'd, so even on CUDA this must fall back to the tensor path. spec = _create_spec(worker_kv_bytes_per_block=0, world_size=4) assert isinstance(spec, CPUOffloadingSpec) assert spec.num_chunks == 0 @@ -429,6 +442,7 @@ def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch): worker_calls: list[dict[str, Any]] = [] monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) monkeypatch.setattr( cpu_spec_module, "SharedOffloadRegion", @@ -461,6 +475,7 @@ def test_cpu_spec_create_worker_rank_assignment( import vllm.v1.kv_offload.cpu.spec as cpu_spec_module monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * 8, diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index 00b36e863a..fddf566085 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -162,7 +162,7 @@ class CPUOffloadingSpec(OffloadingSpec): def _uses_shared_region(self) -> bool: """Whether the worker CPU buffer is the shared mmap region (vs a private per-rank tensor); replicated-layout dedup is gated on this being True.""" - return current_platform.is_cuda_alike() + return current_platform.is_cuda_alike() and not current_platform.is_rocm() def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker: mmap_region: SharedOffloadRegion | None = None