diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 5b40298f58..71e2923702 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -43,7 +43,7 @@ steps: - label: ":nvidia: (H200 MIG 35GB) V1 Core + KV + Metrics" device: h200_35gb key: v1-core-kv-metrics - timeout_in_minutes: 80 + timeout_in_minutes: 100 env: # GitHub intermittently rejects HTTP/2 upload-pack requests from the H200 # CI image. Force Git's smart protocol to use HTTP/1.1 for source installs. diff --git a/tests/v1/worker/test_gpu_worker.py b/tests/v1/worker/test_gpu_worker.py index 0b9cc5a79c..88cbee6802 100644 --- a/tests/v1/worker/test_gpu_worker.py +++ b/tests/v1/worker/test_gpu_worker.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import patch @@ -152,3 +153,71 @@ def test_profiling_fallback_declines_off_rocm(rocm): result = _profile_result(consumed=-RELEASED_BY_OTHERS) assert maybe_rocm_profiling_fallback(result) is None + + +class _OrderedHandle: + """Send handle that logs when it is waited.""" + + def __init__(self, log: list[str], name: str): + self.log = log + self.name = name + + def is_completed(self) -> bool: + return True + + def wait(self) -> None: + self.log.append(f"wait:{self.name}") + + +def test_execute_model_waits_previous_pp_send_before_forward( + monkeypatch: pytest.MonkeyPatch, +): + """Previous device handles are waited before the forward pass; the + metadata handle is left to the GroupCoordinator's reaper.""" + import torch + + from vllm.sequence import IntermediateTensors + + log: list[str] = [] + previous_tensor_send = _OrderedHandle(log, "prev-tensor") + metadata_handle = _OrderedHandle(log, "meta") + tensor_handle = _OrderedHandle(log, "tensor") + + def isend_tensor_dict(tensors, all_gather_group=None, all_gather_tensors=None): + log.append("isend") + return [metadata_handle, tensor_handle] + + pp_group = SimpleNamespace( + is_first_rank=True, + is_last_rank=False, + isend_tensor_dict=isend_tensor_dict, + ) + monkeypatch.setattr(gpu_worker, "get_pp_group", lambda: pp_group) + monkeypatch.setattr(gpu_worker, "get_tp_group", lambda: SimpleNamespace()) + + def run_model(scheduler_output, intermediate_tensors): + log.append("forward") + return IntermediateTensors({"hidden_states": torch.zeros(1)}) + + worker = SimpleNamespace( + vllm_config=SimpleNamespace( + compilation_config=SimpleNamespace( + pass_config=SimpleNamespace(enable_sp=False) + ), + parallel_config=SimpleNamespace( + pipeline_parallel_size=2, distributed_executor_backend="mp" + ), + ), + use_v2_model_runner=False, + model_runner=SimpleNamespace(execute_model=run_model), + annotate_profile=lambda scheduler_output: nullcontext(), + _pp_send_work=[previous_tensor_send], + ) + scheduler_output = SimpleNamespace( + total_num_scheduled_tokens=4, num_scheduled_tokens={"r0": 4} + ) + + assert gpu_worker.Worker.execute_model(worker, scheduler_output) is None + + assert log == ["wait:prev-tensor", "forward", "isend"] + assert worker._pp_send_work == [tensor_handle] diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 9ee217936c..70ab3ab95f 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -219,6 +219,9 @@ class Worker(WorkerBase): self.use_v2_model_runner = vllm_config.use_v2_model_runner + # Device handles of the previous step's PP intermediate-tensor send. + self._pp_send_work: list[Handle] = [] + # Resolved lazily on first sleep/wake; persists worker-process state. self._sleep_mode_backend: SleepModeBackend | None = None @@ -1113,6 +1116,13 @@ class Worker(WorkerBase): def execute_model( self, scheduler_output: "SchedulerOutput" ) -> ModelRunnerOutput | AsyncModelRunnerOutput | None: + # Wait for the previous step's sends so this forward pass cannot + # overwrite buffers they are still reading. + if self._pp_send_work: + for handle in self._pp_send_work: + handle.wait() + self._pp_send_work = [] + intermediate_tensors = None forward_pass = scheduler_output.total_num_scheduled_tokens > 0 num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens @@ -1185,16 +1195,15 @@ class Worker(WorkerBase): and not get_pp_group().is_last_rank ) - # launch non-blocking send of intermediate tensors; the - # GroupCoordinator self-retains the source tensors and the - # metadata handle in ``_pending_isends`` and reaps them lazily on - # the next ``isend_tensor_dict(is_async=True)`` call, so this - # step returns without waiting for the receiver to drain. - get_pp_group().isend_tensor_dict( + # Non-blocking send of the intermediate tensors. The metadata handle + # is reaped lazily by the GroupCoordinator; the device handles are + # waited at the top of the next step. + handles = get_pp_group().isend_tensor_dict( output.tensors, all_gather_group=get_tp_group(), all_gather_tensors=all_gather_tensors, ) + self._pp_send_work = handles[1:] return None