[Bugfix][Offloader] Preserve prefetch static-buffer slot ownership (#54975)

Signed-off-by: Big2Wheel <1404207590@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
yulun
2026-09-07 10:58:44 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent de69e821b7
commit 1f778486fc
2 changed files with 57 additions and 1 deletions
+20 -1
View File
@@ -124,6 +124,21 @@ class StaticBufferPool:
return self._buffers[key][slot_idx % self.slot_capacity]
def _get_next_prefetch_index(
index: int,
prefetch_step: int,
module_count: int,
) -> int:
"""Return a refill target that preserves static-buffer slot ownership."""
next_index = (index + prefetch_step) % module_count
if (
prefetch_step < module_count
and next_index % prefetch_step != index % prefetch_step
):
next_index = index % prefetch_step
return next_index
class PrefetchOffloader(BaseOffloader):
"""Prefetching-based offloader with group-based layer selection.
@@ -231,7 +246,11 @@ class PrefetchOffloader(BaseOffloader):
# Start prefetch for next layer (circular)
# mutates_args on output_tensor creates ordering dependency
next_index = (index + self.prefetch_step) % len(self.module_offloaders)
next_index = _get_next_prefetch_index(
index,
self.prefetch_step,
len(self.module_offloaders),
)
# Handle tuple output (e.g., (hidden_states, residual))
if isinstance(output, tuple):
torch.ops.vllm.start_prefetch(output[0], next_index)