[Core][Distributed] use cpu group to broadcast metadata in cpu (#4444)
This commit is contained in:
@@ -6,14 +6,14 @@ import uuid
|
||||
from functools import partial
|
||||
from typing import Type
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from tensorizer import (DecryptionParams, EncryptionParams, TensorDeserializer,
|
||||
TensorSerializer, stream_io)
|
||||
from tensorizer.utils import convert_bytes, get_mem_usage, no_init_or_tensor
|
||||
from transformers import AutoConfig, PretrainedConfig
|
||||
|
||||
from vllm.distributed import initialize_model_parallel
|
||||
from vllm.distributed import (init_distributed_environment,
|
||||
initialize_model_parallel)
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.engine.llm_engine import LLMEngine
|
||||
from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
|
||||
@@ -226,7 +226,7 @@ model_name = model_ref.split("/")[1]
|
||||
os.environ["MASTER_ADDR"] = "127.0.0.1"
|
||||
os.environ["MASTER_PORT"] = "8080"
|
||||
|
||||
torch.distributed.init_process_group(world_size=1, rank=0)
|
||||
init_distributed_environment(world_size=1, rank=0, local_rank=0)
|
||||
initialize_model_parallel()
|
||||
|
||||
keyfile = args.keyfile if args.keyfile else None
|
||||
|
||||
@@ -2,8 +2,10 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig, SchedulerConfig
|
||||
from vllm.distributed.parallel_state import init_distributed_environment
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
|
||||
from vllm.utils import get_open_port
|
||||
from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
|
||||
|
||||
|
||||
@@ -249,19 +251,18 @@ def test_empty_seq_group():
|
||||
assert len(return_prompt_lens) == 0
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def distributed_init():
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
|
||||
local_rank=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size", list(range(2, 128)))
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
def test_hybrid_batches(batch_size, enforce_eager, monkeypatch):
|
||||
|
||||
def get_world_size(group=None):
|
||||
return 1
|
||||
|
||||
def mock_get_process_group_ranks(group=None):
|
||||
return [0]
|
||||
|
||||
monkeypatch.setattr(torch.distributed, "get_world_size", get_world_size)
|
||||
monkeypatch.setattr(torch.distributed, "get_process_group_ranks",
|
||||
mock_get_process_group_ranks)
|
||||
def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
|
||||
|
||||
model_config = ModelConfig(
|
||||
"facebook/opt-125m",
|
||||
|
||||
Reference in New Issue
Block a user