[Bugfix] Skip PP sampled-token receive on last rank during async scheduling (#40749)
Signed-off-by: Adam Winstanley <adam@winstanley.industries>
This commit is contained in:
@@ -7,6 +7,7 @@ import numpy as np
|
|||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
import vllm.v1.worker.gpu_model_runner as gpu_model_runner_module
|
||||||
from vllm.config import (
|
from vllm.config import (
|
||||||
AttentionConfig,
|
AttentionConfig,
|
||||||
CacheConfig,
|
CacheConfig,
|
||||||
@@ -219,6 +220,58 @@ def test_select_common_block_size_uses_largest_shared_int():
|
|||||||
assert selected_size == 64
|
assert selected_size == 64
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("world_size", "is_last_rank", "expected_calls"),
|
||||||
|
[(1, True, 0), (2, True, 0), (2, False, 1)],
|
||||||
|
)
|
||||||
|
def test_sample_tokens_receives_pp_sampled_ids_only_on_non_last_rank(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
world_size: int,
|
||||||
|
is_last_rank: bool,
|
||||||
|
expected_calls: int,
|
||||||
|
):
|
||||||
|
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||||
|
runner.execute_model_state = None
|
||||||
|
runner.kv_connector_output = None
|
||||||
|
runner.use_async_scheduling = True
|
||||||
|
receive_calls = 0
|
||||||
|
|
||||||
|
def receive_prev_sampled_token_ids():
|
||||||
|
nonlocal receive_calls
|
||||||
|
receive_calls += 1
|
||||||
|
|
||||||
|
runner._pp_receive_prev_sampled_token_ids_to_input_batch = (
|
||||||
|
receive_prev_sampled_token_ids
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
gpu_model_runner_module,
|
||||||
|
"get_pp_group",
|
||||||
|
lambda: SimpleNamespace(world_size=world_size, is_last_rank=is_last_rank),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert GPUModelRunner.sample_tokens(runner, None) is None
|
||||||
|
assert receive_calls == expected_calls
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
def test_sample_tokens_skips_pp_group_lookup_without_async_scheduling(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
):
|
||||||
|
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||||
|
runner.execute_model_state = None
|
||||||
|
runner.kv_connector_output = None
|
||||||
|
runner.use_async_scheduling = False
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
gpu_model_runner_module,
|
||||||
|
"get_pp_group",
|
||||||
|
pytest.fail,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert GPUModelRunner.sample_tokens(runner, None) is None
|
||||||
|
|
||||||
|
|
||||||
def test_select_common_block_size_no_valid_option():
|
def test_select_common_block_size_no_valid_option():
|
||||||
backend_a = _make_mock_backend_for_kernel_block_size([64])
|
backend_a = _make_mock_backend_for_kernel_block_size([64])
|
||||||
backend_b = _make_mock_backend_for_kernel_block_size([MultipleOf(16)])
|
backend_b = _make_mock_backend_for_kernel_block_size([MultipleOf(16)])
|
||||||
|
|||||||
@@ -4182,7 +4182,7 @@ class GPUModelRunner(
|
|||||||
kv_connector_output = self.kv_connector_output
|
kv_connector_output = self.kv_connector_output
|
||||||
self.kv_connector_output = None
|
self.kv_connector_output = None
|
||||||
# receive sampled token ids from the last PP rank.
|
# receive sampled token ids from the last PP rank.
|
||||||
if self.use_async_scheduling and get_pp_group().world_size > 1:
|
if self.use_async_scheduling and not get_pp_group().is_last_rank:
|
||||||
self._pp_receive_prev_sampled_token_ids_to_input_batch()
|
self._pp_receive_prev_sampled_token_ids_to_input_batch()
|
||||||
if not kv_connector_output:
|
if not kv_connector_output:
|
||||||
return None # type: ignore[return-value]
|
return None # type: ignore[return-value]
|
||||||
|
|||||||
Reference in New Issue
Block a user