From 2bea22002d66a6c89d208fa682b40a89538f023c Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Sat, 10 Oct 2026 20:25:11 +0800 Subject: [PATCH] [BUG] Fix KV offload shared-memory slot selection using worker rank (#58497) Signed-off-by: Tony Lin --- tests/v1/kv_offload/test_factory.py | 33 +++++++++++++---------------- vllm/v1/kv_offload/cpu/spec.py | 9 ++------ vllm/v1/kv_offload/tiering/spec.py | 9 +------- 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/tests/v1/kv_offload/test_factory.py b/tests/v1/kv_offload/test_factory.py index 84b9da36e0..dbc586bcab 100644 --- a/tests/v1/kv_offload/test_factory.py +++ b/tests/v1/kv_offload/test_factory.py @@ -6,6 +6,7 @@ from typing import Any from unittest.mock import MagicMock import pytest +import torch from vllm.v1.kv_offload.base import ( CanonicalKVCaches, @@ -231,9 +232,7 @@ def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monke monkeypatch.setattr(tiering_spec_module, "SharedOffloadRegion", fake_region_ctor) monkeypatch.setattr(tiering_spec_module, "CPUOffloadingWorker", fake_worker_ctor) - monkeypatch.setattr( - tiering_spec_module.torch.accelerator, "current_device_index", lambda: 5 - ) + monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 5) kv_caches = MagicMock() spec.create_worker(kv_caches) @@ -244,12 +243,13 @@ def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monke assert worker_calls[0]["mmap_region"] is region -def test_tiering_spec_create_worker_folds_device_index_for_sharded_layout(monkeypatch): +def test_tiering_spec_create_worker_uses_worker_rank_for_sharded_layout(monkeypatch): import vllm.v1.kv_offload.tiering.spec as tiering_spec_module spec = _create_spec( spec_name="TieringOffloadingSpec", worker_kv_bytes_per_block=4096, + rank=1, world_size=4, ) assert isinstance(spec, TieringOffloadingSpec) @@ -263,9 +263,9 @@ def test_tiering_spec_create_worker_folds_device_index_for_sharded_layout(monkey monkeypatch.setattr(tiering_spec_module, "SharedOffloadRegion", fake_region_ctor) monkeypatch.setattr(tiering_spec_module, "CPUOffloadingWorker", MagicMock()) monkeypatch.setattr( - tiering_spec_module.torch.accelerator, + torch.accelerator, "current_device_index", - lambda: 5, + lambda: 6, ) spec.create_worker(MagicMock()) @@ -386,9 +386,7 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda(monkeypatch): monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) monkeypatch.setattr(cpu_spec_module, "SharedOffloadRegion", fake_region_ctor) monkeypatch.setattr(cpu_spec_module, "CPUOffloadingWorker", fake_worker_ctor) - monkeypatch.setattr( - cpu_spec_module.torch.accelerator, "current_device_index", lambda: 5 - ) + monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 5) kv_caches = MagicMock() spec.create_worker(kv_caches) @@ -464,16 +462,16 @@ def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch): @pytest.mark.parametrize( - ("replicated_layout", "device_index", "world_size", "expected_rank"), + ("replicated_layout", "rank", "device_index", "world_size", "expected_rank"), [ - (True, 5, 4, 0), # replicated: always slot 0 - (True, 0, 4, 0), # replicated: slot 0 regardless of device - (False, 5, 4, 1), # non-replicated: 5 % 4 == 1 - (False, 7, 4, 3), # non-replicated: 7 % 4 == 3 + (True, 1, 5, 4, 0), + (True, 0, 0, 4, 0), + (False, 1, 5, 4, 1), + (False, 3, 6, 4, 3), ], ) def test_cpu_spec_create_worker_rank_assignment( - monkeypatch, replicated_layout, device_index, world_size, expected_rank + monkeypatch, replicated_layout, rank, device_index, world_size, expected_rank ): import vllm.v1.kv_offload.cpu.spec as cpu_spec_module @@ -483,6 +481,7 @@ def test_cpu_spec_create_worker_rank_assignment( spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * 8, worker_kv_bytes_per_block=worker_kv_bytes_per_block, + rank=rank, world_size=world_size, replicated_layout=replicated_layout, ) @@ -495,9 +494,7 @@ def test_cpu_spec_create_worker_rank_assignment( monkeypatch.setattr(cpu_spec_module, "SharedOffloadRegion", fake_region_ctor) monkeypatch.setattr(cpu_spec_module, "CPUOffloadingWorker", MagicMock()) - monkeypatch.setattr( - cpu_spec_module.torch.accelerator, "current_device_index", lambda: device_index - ) + monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: device_index) spec.create_worker(MagicMock()) diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index bf289d37a6..c26cd32f97 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -import torch from typing_extensions import override from vllm.platforms import current_platform @@ -172,12 +171,8 @@ class CPUOffloadingSpec(OffloadingSpec): # mmap'd; fall back to the tensor path (empty tensors) as before. if self._uses_shared_region() and self.num_chunks > 0: # Replicated layout puts all ranks on slot 0 (single MLA copy); - # otherwise each rank takes its own slot by physical device index. - if self.replicated_layout: - rank = 0 - else: - world_size = self.config.parallel.world_size - rank = torch.accelerator.current_device_index() % world_size + # otherwise each worker uses its own rank-indexed slot. + rank = 0 if self.replicated_layout else self.config.parallel.rank mmap_region = SharedOffloadRegion( engine_id=self.config.engine_id, num_chunks=self.num_chunks, diff --git a/vllm/v1/kv_offload/tiering/spec.py b/vllm/v1/kv_offload/tiering/spec.py index 93c6cbabcb..508361d392 100644 --- a/vllm/v1/kv_offload/tiering/spec.py +++ b/vllm/v1/kv_offload/tiering/spec.py @@ -54,7 +54,6 @@ Example out-of-tree tier configuration: from typing import Any -import torch from typing_extensions import override from vllm.logger import init_logger @@ -436,13 +435,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec): @override def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker: - world_size = self.config.parallel.world_size - if self.replicated_layout: - rank = 0 - else: - # Fold the global physical device index into the replica-local - # [0, world_size) slot range. - rank = torch.accelerator.current_device_index() % world_size + rank = 0 if self.replicated_layout else self.config.parallel.rank worker_mmap = SharedOffloadRegion( engine_id=self._engine_id, num_chunks=self.num_chunks,