[Core][Distributed] use cpu group to broadcast metadata in cpu (#4444)

This commit is contained in:
youkaichao
2024-04-29 13:52:22 -07:00
committed by GitHub
parent ac5ccf0156
commit f4f921b7f1
3 changed files with 63 additions and 35 deletions
+12 -11
View File
@@ -2,8 +2,10 @@ import pytest
import torch
from vllm.config import ModelConfig, SchedulerConfig
from vllm.distributed.parallel_state import init_distributed_environment
from vllm.model_executor.sampling_metadata import SamplingMetadata
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
from vllm.utils import get_open_port
from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
@@ -249,19 +251,18 @@ def test_empty_seq_group():
assert len(return_prompt_lens) == 0
@pytest.fixture
def distributed_init():
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
local_rank=0)
@pytest.mark.parametrize("batch_size", list(range(2, 128)))
@pytest.mark.parametrize("enforce_eager", [True, False])
def test_hybrid_batches(batch_size, enforce_eager, monkeypatch):
def get_world_size(group=None):
return 1
def mock_get_process_group_ranks(group=None):
return [0]
monkeypatch.setattr(torch.distributed, "get_world_size", get_world_size)
monkeypatch.setattr(torch.distributed, "get_process_group_ranks",
mock_get_process_group_ranks)
def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
model_config = ModelConfig(
"facebook/opt-125m",