[BUG] Fix KV offload shared-memory slot selection using worker rank (#58497)

Signed-off-by: Tony Lin <tony.lin@intel.com>
This commit is contained in:
Tony Lin
2026-10-10 20:25:11 +08:00
committed by GitHub
parent c41b2639e2
commit 2bea22002d
3 changed files with 18 additions and 33 deletions
+15 -18
View File
@@ -6,6 +6,7 @@ from typing import Any
from unittest.mock import MagicMock
import pytest
import torch
from vllm.v1.kv_offload.base import (
CanonicalKVCaches,
@@ -231,9 +232,7 @@ def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monke
monkeypatch.setattr(tiering_spec_module, "SharedOffloadRegion", fake_region_ctor)
monkeypatch.setattr(tiering_spec_module, "CPUOffloadingWorker", fake_worker_ctor)
monkeypatch.setattr(
tiering_spec_module.torch.accelerator, "current_device_index", lambda: 5
)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 5)
kv_caches = MagicMock()
spec.create_worker(kv_caches)
@@ -244,12 +243,13 @@ def test_tiering_spec_create_worker_uses_single_slot_for_replicated_layout(monke
assert worker_calls[0]["mmap_region"] is region
def test_tiering_spec_create_worker_folds_device_index_for_sharded_layout(monkeypatch):
def test_tiering_spec_create_worker_uses_worker_rank_for_sharded_layout(monkeypatch):
import vllm.v1.kv_offload.tiering.spec as tiering_spec_module
spec = _create_spec(
spec_name="TieringOffloadingSpec",
worker_kv_bytes_per_block=4096,
rank=1,
world_size=4,
)
assert isinstance(spec, TieringOffloadingSpec)
@@ -263,9 +263,9 @@ def test_tiering_spec_create_worker_folds_device_index_for_sharded_layout(monkey
monkeypatch.setattr(tiering_spec_module, "SharedOffloadRegion", fake_region_ctor)
monkeypatch.setattr(tiering_spec_module, "CPUOffloadingWorker", MagicMock())
monkeypatch.setattr(
tiering_spec_module.torch.accelerator,
torch.accelerator,
"current_device_index",
lambda: 5,
lambda: 6,
)
spec.create_worker(MagicMock())
@@ -386,9 +386,7 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda(monkeypatch):
monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False)
monkeypatch.setattr(cpu_spec_module, "SharedOffloadRegion", fake_region_ctor)
monkeypatch.setattr(cpu_spec_module, "CPUOffloadingWorker", fake_worker_ctor)
monkeypatch.setattr(
cpu_spec_module.torch.accelerator, "current_device_index", lambda: 5
)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 5)
kv_caches = MagicMock()
spec.create_worker(kv_caches)
@@ -464,16 +462,16 @@ def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch):
@pytest.mark.parametrize(
("replicated_layout", "device_index", "world_size", "expected_rank"),
("replicated_layout", "rank", "device_index", "world_size", "expected_rank"),
[
(True, 5, 4, 0), # replicated: always slot 0
(True, 0, 4, 0), # replicated: slot 0 regardless of device
(False, 5, 4, 1), # non-replicated: 5 % 4 == 1
(False, 7, 4, 3), # non-replicated: 7 % 4 == 3
(True, 1, 5, 4, 0),
(True, 0, 0, 4, 0),
(False, 1, 5, 4, 1),
(False, 3, 6, 4, 3),
],
)
def test_cpu_spec_create_worker_rank_assignment(
monkeypatch, replicated_layout, device_index, world_size, expected_rank
monkeypatch, replicated_layout, rank, device_index, world_size, expected_rank
):
import vllm.v1.kv_offload.cpu.spec as cpu_spec_module
@@ -483,6 +481,7 @@ def test_cpu_spec_create_worker_rank_assignment(
spec = _create_spec(
cpu_bytes_to_use=worker_kv_bytes_per_block * 8,
worker_kv_bytes_per_block=worker_kv_bytes_per_block,
rank=rank,
world_size=world_size,
replicated_layout=replicated_layout,
)
@@ -495,9 +494,7 @@ def test_cpu_spec_create_worker_rank_assignment(
monkeypatch.setattr(cpu_spec_module, "SharedOffloadRegion", fake_region_ctor)
monkeypatch.setattr(cpu_spec_module, "CPUOffloadingWorker", MagicMock())
monkeypatch.setattr(
cpu_spec_module.torch.accelerator, "current_device_index", lambda: device_index
)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: device_index)
spec.create_worker(MagicMock())
+2 -7
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
from typing_extensions import override
from vllm.platforms import current_platform
@@ -172,12 +171,8 @@ class CPUOffloadingSpec(OffloadingSpec):
# mmap'd; fall back to the tensor path (empty tensors) as before.
if self._uses_shared_region() and self.num_chunks > 0:
# Replicated layout puts all ranks on slot 0 (single MLA copy);
# otherwise each rank takes its own slot by physical device index.
if self.replicated_layout:
rank = 0
else:
world_size = self.config.parallel.world_size
rank = torch.accelerator.current_device_index() % world_size
# otherwise each worker uses its own rank-indexed slot.
rank = 0 if self.replicated_layout else self.config.parallel.rank
mmap_region = SharedOffloadRegion(
engine_id=self.config.engine_id,
num_chunks=self.num_chunks,
+1 -8
View File
@@ -54,7 +54,6 @@ Example out-of-tree tier configuration:
from typing import Any
import torch
from typing_extensions import override
from vllm.logger import init_logger
@@ -436,13 +435,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
@override
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
world_size = self.config.parallel.world_size
if self.replicated_layout:
rank = 0
else:
# Fold the global physical device index into the replica-local
# [0, world_size) slot range.
rank = torch.accelerator.current_device_index() % world_size
rank = 0 if self.replicated_layout else self.config.parallel.rank
worker_mmap = SharedOffloadRegion(
engine_id=self._engine_id,
num_chunks=self.num_chunks,