[CI/Build] Use file rendezvous for local distributed tests (#51652)
Signed-off-by: yu-xin-c <2182712990@qq.com>
This commit is contained in:
@@ -29,7 +29,7 @@ from vllm.distributed.parallel_state import (
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.network_utils import get_file_store_init_method
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -266,7 +266,7 @@ def test_async_tp_pass_replace(
|
||||
dynamic: bool,
|
||||
):
|
||||
num_processes = 2
|
||||
master_port = str(get_open_port())
|
||||
distributed_init_method = get_file_store_init_method()
|
||||
|
||||
def run_torch_spawn(fn, nprocs):
|
||||
# need to use torch.mp.spawn otherwise will have problems with
|
||||
@@ -281,7 +281,7 @@ def test_async_tp_pass_replace(
|
||||
hidden_size,
|
||||
dtype,
|
||||
dynamic,
|
||||
master_port,
|
||||
distributed_init_method,
|
||||
),
|
||||
nprocs=nprocs,
|
||||
)
|
||||
@@ -314,7 +314,7 @@ def async_tp_pass_on_test_model(
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
dynamic: bool,
|
||||
master_port: str = "0",
|
||||
distributed_init_method: str,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
@@ -328,13 +328,16 @@ def async_tp_pass_on_test_model(
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": master_port,
|
||||
}
|
||||
)
|
||||
|
||||
# initialize distributed
|
||||
init_distributed_environment()
|
||||
init_distributed_environment(
|
||||
world_size=world_size,
|
||||
rank=local_rank,
|
||||
distributed_init_method=distributed_init_method,
|
||||
local_rank=local_rank,
|
||||
)
|
||||
|
||||
# configure vllm config for SequenceParallelismPass
|
||||
vllm_config = VllmConfig()
|
||||
|
||||
@@ -17,7 +17,7 @@ import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.network_utils import get_file_store_init_method
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
mp.set_start_method("spawn", force=True)
|
||||
@@ -76,7 +76,7 @@ def _assert_packed_a2a_close(
|
||||
|
||||
|
||||
def _distributed_run(fn, world_size: int, extra_env: dict[str, str]) -> None:
|
||||
port = str(get_open_port())
|
||||
distributed_init_method = get_file_store_init_method()
|
||||
processes: list[mp.Process] = []
|
||||
for rank in range(world_size):
|
||||
env = {
|
||||
@@ -84,8 +84,7 @@ def _distributed_run(fn, world_size: int, extra_env: dict[str, str]) -> None:
|
||||
"LOCAL_RANK": str(rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"LOCAL_WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": port,
|
||||
"DISTRIBUTED_INIT_METHOD": distributed_init_method,
|
||||
**extra_env,
|
||||
}
|
||||
process = mp.Process(target=fn, args=(env,))
|
||||
@@ -483,14 +482,23 @@ class TestPackedA2AKernels:
|
||||
def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
|
||||
update_environment_variables(env)
|
||||
local_rank = int(env["LOCAL_RANK"])
|
||||
world_size = int(env["WORLD_SIZE"])
|
||||
torch.accelerator.set_device_index(local_rank)
|
||||
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
|
||||
dist.init_process_group(
|
||||
backend="cpu:gloo,cuda:nccl",
|
||||
device_id=torch.device(f"cuda:{local_rank}"),
|
||||
init_method=env["DISTRIBUTED_INIT_METHOD"],
|
||||
rank=local_rank,
|
||||
world_size=world_size,
|
||||
)
|
||||
else:
|
||||
dist.init_process_group(backend="nccl")
|
||||
dist.init_process_group(
|
||||
backend="nccl",
|
||||
init_method=env["DISTRIBUTED_INIT_METHOD"],
|
||||
rank=local_rank,
|
||||
world_size=world_size,
|
||||
)
|
||||
use_workspace = env.get("USE_WORKSPACE") == "1"
|
||||
if use_workspace:
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
@@ -8,6 +8,7 @@ import zmq
|
||||
|
||||
from vllm.utils import network_utils
|
||||
from vllm.utils.network_utils import (
|
||||
get_file_store_init_method,
|
||||
get_open_port,
|
||||
get_open_ports_list,
|
||||
get_tcp_uri,
|
||||
@@ -19,6 +20,13 @@ from vllm.utils.network_utils import (
|
||||
)
|
||||
|
||||
|
||||
def test_get_file_store_init_method_is_unique():
|
||||
init_methods = {get_file_store_init_method() for _ in range(2)}
|
||||
|
||||
assert len(init_methods) == 2
|
||||
assert all(method.startswith("file://") for method in init_methods)
|
||||
|
||||
|
||||
def _call_with_timeout(func, timeout: float = 10.0):
|
||||
"""Run func in a daemon thread so a livelock regression fails the test
|
||||
quickly instead of hanging the suite."""
|
||||
|
||||
Reference in New Issue
Block a user