[Bugfix][Core] Wait for the previous PP tensor sends before the next forward pass (#54962)
Signed-off-by: Derek Weitzel <djw8605@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
ee17d0d869
commit
859dd39512
@@ -43,7 +43,7 @@ steps:
|
||||
- label: ":nvidia: (H200 MIG 35GB) V1 Core + KV + Metrics"
|
||||
device: h200_35gb
|
||||
key: v1-core-kv-metrics
|
||||
timeout_in_minutes: 80
|
||||
timeout_in_minutes: 100
|
||||
env:
|
||||
# GitHub intermittently rejects HTTP/2 upload-pack requests from the H200
|
||||
# CI image. Force Git's smart protocol to use HTTP/1.1 for source installs.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -152,3 +153,71 @@ def test_profiling_fallback_declines_off_rocm(rocm):
|
||||
result = _profile_result(consumed=-RELEASED_BY_OTHERS)
|
||||
|
||||
assert maybe_rocm_profiling_fallback(result) is None
|
||||
|
||||
|
||||
class _OrderedHandle:
|
||||
"""Send handle that logs when it is waited."""
|
||||
|
||||
def __init__(self, log: list[str], name: str):
|
||||
self.log = log
|
||||
self.name = name
|
||||
|
||||
def is_completed(self) -> bool:
|
||||
return True
|
||||
|
||||
def wait(self) -> None:
|
||||
self.log.append(f"wait:{self.name}")
|
||||
|
||||
|
||||
def test_execute_model_waits_previous_pp_send_before_forward(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Previous device handles are waited before the forward pass; the
|
||||
metadata handle is left to the GroupCoordinator's reaper."""
|
||||
import torch
|
||||
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
log: list[str] = []
|
||||
previous_tensor_send = _OrderedHandle(log, "prev-tensor")
|
||||
metadata_handle = _OrderedHandle(log, "meta")
|
||||
tensor_handle = _OrderedHandle(log, "tensor")
|
||||
|
||||
def isend_tensor_dict(tensors, all_gather_group=None, all_gather_tensors=None):
|
||||
log.append("isend")
|
||||
return [metadata_handle, tensor_handle]
|
||||
|
||||
pp_group = SimpleNamespace(
|
||||
is_first_rank=True,
|
||||
is_last_rank=False,
|
||||
isend_tensor_dict=isend_tensor_dict,
|
||||
)
|
||||
monkeypatch.setattr(gpu_worker, "get_pp_group", lambda: pp_group)
|
||||
monkeypatch.setattr(gpu_worker, "get_tp_group", lambda: SimpleNamespace())
|
||||
|
||||
def run_model(scheduler_output, intermediate_tensors):
|
||||
log.append("forward")
|
||||
return IntermediateTensors({"hidden_states": torch.zeros(1)})
|
||||
|
||||
worker = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(
|
||||
pass_config=SimpleNamespace(enable_sp=False)
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=2, distributed_executor_backend="mp"
|
||||
),
|
||||
),
|
||||
use_v2_model_runner=False,
|
||||
model_runner=SimpleNamespace(execute_model=run_model),
|
||||
annotate_profile=lambda scheduler_output: nullcontext(),
|
||||
_pp_send_work=[previous_tensor_send],
|
||||
)
|
||||
scheduler_output = SimpleNamespace(
|
||||
total_num_scheduled_tokens=4, num_scheduled_tokens={"r0": 4}
|
||||
)
|
||||
|
||||
assert gpu_worker.Worker.execute_model(worker, scheduler_output) is None
|
||||
|
||||
assert log == ["wait:prev-tensor", "forward", "isend"]
|
||||
assert worker._pp_send_work == [tensor_handle]
|
||||
|
||||
@@ -219,6 +219,9 @@ class Worker(WorkerBase):
|
||||
|
||||
self.use_v2_model_runner = vllm_config.use_v2_model_runner
|
||||
|
||||
# Device handles of the previous step's PP intermediate-tensor send.
|
||||
self._pp_send_work: list[Handle] = []
|
||||
|
||||
# Resolved lazily on first sleep/wake; persists worker-process state.
|
||||
self._sleep_mode_backend: SleepModeBackend | None = None
|
||||
|
||||
@@ -1113,6 +1116,13 @@ class Worker(WorkerBase):
|
||||
def execute_model(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> ModelRunnerOutput | AsyncModelRunnerOutput | None:
|
||||
# Wait for the previous step's sends so this forward pass cannot
|
||||
# overwrite buffers they are still reading.
|
||||
if self._pp_send_work:
|
||||
for handle in self._pp_send_work:
|
||||
handle.wait()
|
||||
self._pp_send_work = []
|
||||
|
||||
intermediate_tensors = None
|
||||
forward_pass = scheduler_output.total_num_scheduled_tokens > 0
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
@@ -1185,16 +1195,15 @@ class Worker(WorkerBase):
|
||||
and not get_pp_group().is_last_rank
|
||||
)
|
||||
|
||||
# launch non-blocking send of intermediate tensors; the
|
||||
# GroupCoordinator self-retains the source tensors and the
|
||||
# metadata handle in ``_pending_isends`` and reaps them lazily on
|
||||
# the next ``isend_tensor_dict(is_async=True)`` call, so this
|
||||
# step returns without waiting for the receiver to drain.
|
||||
get_pp_group().isend_tensor_dict(
|
||||
# Non-blocking send of the intermediate tensors. The metadata handle
|
||||
# is reaped lazily by the GroupCoordinator; the device handles are
|
||||
# waited at the top of the next step.
|
||||
handles = get_pp_group().isend_tensor_dict(
|
||||
output.tensors,
|
||||
all_gather_group=get_tp_group(),
|
||||
all_gather_tensors=all_gather_tensors,
|
||||
)
|
||||
self._pp_send_work = handles[1:]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user