[Bugfix][Offloader] Preserve prefetch static-buffer slot ownership (#54975)
Signed-off-by: Big2Wheel <1404207590@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
de69e821b7
commit
1f778486fc
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.offloader.prefetch import _get_next_prefetch_index
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module_count", "prefetch_step", "expected_targets"),
|
||||
[
|
||||
(3, 1, [1, 2, 0]),
|
||||
(3, 2, [2, 1, 0]),
|
||||
(4, 2, [2, 3, 0, 1]),
|
||||
(5, 2, [2, 3, 4, 1, 0]),
|
||||
(3, 3, [0, 1, 2]),
|
||||
(3, 4, [1, 2, 0]),
|
||||
],
|
||||
)
|
||||
def test_next_prefetch_index_preserves_slot_ownership(
|
||||
module_count: int,
|
||||
prefetch_step: int,
|
||||
expected_targets: list[int],
|
||||
) -> None:
|
||||
targets = [
|
||||
_get_next_prefetch_index(index, prefetch_step, module_count)
|
||||
for index in range(module_count)
|
||||
]
|
||||
|
||||
assert targets == expected_targets
|
||||
if prefetch_step < module_count:
|
||||
assert all(
|
||||
target % prefetch_step == index % prefetch_step
|
||||
for index, target in enumerate(targets)
|
||||
)
|
||||
@@ -124,6 +124,21 @@ class StaticBufferPool:
|
||||
return self._buffers[key][slot_idx % self.slot_capacity]
|
||||
|
||||
|
||||
def _get_next_prefetch_index(
|
||||
index: int,
|
||||
prefetch_step: int,
|
||||
module_count: int,
|
||||
) -> int:
|
||||
"""Return a refill target that preserves static-buffer slot ownership."""
|
||||
next_index = (index + prefetch_step) % module_count
|
||||
if (
|
||||
prefetch_step < module_count
|
||||
and next_index % prefetch_step != index % prefetch_step
|
||||
):
|
||||
next_index = index % prefetch_step
|
||||
return next_index
|
||||
|
||||
|
||||
class PrefetchOffloader(BaseOffloader):
|
||||
"""Prefetching-based offloader with group-based layer selection.
|
||||
|
||||
@@ -231,7 +246,11 @@ class PrefetchOffloader(BaseOffloader):
|
||||
|
||||
# Start prefetch for next layer (circular)
|
||||
# mutates_args on output_tensor creates ordering dependency
|
||||
next_index = (index + self.prefetch_step) % len(self.module_offloaders)
|
||||
next_index = _get_next_prefetch_index(
|
||||
index,
|
||||
self.prefetch_step,
|
||||
len(self.module_offloaders),
|
||||
)
|
||||
# Handle tuple output (e.g., (hidden_states, residual))
|
||||
if isinstance(output, tuple):
|
||||
torch.ops.vllm.start_prefetch(output[0], next_index)
|
||||
|
||||
Reference in New Issue
Block a user