[Bugfix][Core] Wait for the previous PP tensor sends before the next forward pass (#54962)

Signed-off-by: Derek Weitzel <djw8605@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Derek Weitzel
2026-09-03 08:01:36 +00:00
committed by GitHub
co-authored by Claude Fable 5.1
parent ee17d0d869
commit 859dd39512
3 changed files with 85 additions and 7 deletions
+1 -1
View File
@@ -43,7 +43,7 @@ steps:
- label: ":nvidia: (H200 MIG 35GB) V1 Core + KV + Metrics"
device: h200_35gb
key: v1-core-kv-metrics
timeout_in_minutes: 80
timeout_in_minutes: 100
env:
# GitHub intermittently rejects HTTP/2 upload-pack requests from the H200
# CI image. Force Git's smart protocol to use HTTP/1.1 for source installs.
+69
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import patch
@@ -152,3 +153,71 @@ def test_profiling_fallback_declines_off_rocm(rocm):
result = _profile_result(consumed=-RELEASED_BY_OTHERS)
assert maybe_rocm_profiling_fallback(result) is None
class _OrderedHandle:
"""Send handle that logs when it is waited."""
def __init__(self, log: list[str], name: str):
self.log = log
self.name = name
def is_completed(self) -> bool:
return True
def wait(self) -> None:
self.log.append(f"wait:{self.name}")
def test_execute_model_waits_previous_pp_send_before_forward(
monkeypatch: pytest.MonkeyPatch,
):
"""Previous device handles are waited before the forward pass; the
metadata handle is left to the GroupCoordinator's reaper."""
import torch
from vllm.sequence import IntermediateTensors
log: list[str] = []
previous_tensor_send = _OrderedHandle(log, "prev-tensor")
metadata_handle = _OrderedHandle(log, "meta")
tensor_handle = _OrderedHandle(log, "tensor")
def isend_tensor_dict(tensors, all_gather_group=None, all_gather_tensors=None):
log.append("isend")
return [metadata_handle, tensor_handle]
pp_group = SimpleNamespace(
is_first_rank=True,
is_last_rank=False,
isend_tensor_dict=isend_tensor_dict,
)
monkeypatch.setattr(gpu_worker, "get_pp_group", lambda: pp_group)
monkeypatch.setattr(gpu_worker, "get_tp_group", lambda: SimpleNamespace())
def run_model(scheduler_output, intermediate_tensors):
log.append("forward")
return IntermediateTensors({"hidden_states": torch.zeros(1)})
worker = SimpleNamespace(
vllm_config=SimpleNamespace(
compilation_config=SimpleNamespace(
pass_config=SimpleNamespace(enable_sp=False)
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=2, distributed_executor_backend="mp"
),
),
use_v2_model_runner=False,
model_runner=SimpleNamespace(execute_model=run_model),
annotate_profile=lambda scheduler_output: nullcontext(),
_pp_send_work=[previous_tensor_send],
)
scheduler_output = SimpleNamespace(
total_num_scheduled_tokens=4, num_scheduled_tokens={"r0": 4}
)
assert gpu_worker.Worker.execute_model(worker, scheduler_output) is None
assert log == ["wait:prev-tensor", "forward", "isend"]
assert worker._pp_send_work == [tensor_handle]
+15 -6
View File
@@ -219,6 +219,9 @@ class Worker(WorkerBase):
self.use_v2_model_runner = vllm_config.use_v2_model_runner
# Device handles of the previous step's PP intermediate-tensor send.
self._pp_send_work: list[Handle] = []
# Resolved lazily on first sleep/wake; persists worker-process state.
self._sleep_mode_backend: SleepModeBackend | None = None
@@ -1113,6 +1116,13 @@ class Worker(WorkerBase):
def execute_model(
self, scheduler_output: "SchedulerOutput"
) -> ModelRunnerOutput | AsyncModelRunnerOutput | None:
# Wait for the previous step's sends so this forward pass cannot
# overwrite buffers they are still reading.
if self._pp_send_work:
for handle in self._pp_send_work:
handle.wait()
self._pp_send_work = []
intermediate_tensors = None
forward_pass = scheduler_output.total_num_scheduled_tokens > 0
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
@@ -1185,16 +1195,15 @@ class Worker(WorkerBase):
and not get_pp_group().is_last_rank
)
# launch non-blocking send of intermediate tensors; the
# GroupCoordinator self-retains the source tensors and the
# metadata handle in ``_pending_isends`` and reaps them lazily on
# the next ``isend_tensor_dict(is_async=True)`` call, so this
# step returns without waiting for the receiver to drain.
get_pp_group().isend_tensor_dict(
# Non-blocking send of the intermediate tensors. The metadata handle
# is reaped lazily by the GroupCoordinator; the device handles are
# waited at the top of the next step.
handles = get_pp_group().isend_tensor_dict(
output.tensors,
all_gather_group=get_tp_group(),
all_gather_tensors=all_gather_tensors,
)
self._pp_send_work = handles[1:]
return None