[ROCm][CI] Extend ROCm quick reduce coverage (#40990)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
+26
-31
@@ -1363,43 +1363,38 @@ def multi_process_parallel(
|
||||
) -> None:
|
||||
import ray
|
||||
|
||||
# Using ray helps debugging the error when it failed
|
||||
# as compared to multiprocessing.
|
||||
# NOTE: We need to set working_dir for distributed tests,
|
||||
# otherwise we may get import errors on ray workers
|
||||
# NOTE: Force ray not to use gitignore file as excluding, otherwise
|
||||
# it will not move .so files to working dir.
|
||||
# So we have to manually add some of large directories
|
||||
os.environ["RAY_RUNTIME_ENV_IGNORE_GITIGNORE"] = "1"
|
||||
# Using ray helps debugging the error when it failed as compared to
|
||||
# multiprocessing. For local Ray workers, putting the repo root on
|
||||
# PYTHONPATH is enough and avoids uploading the full source tree, which
|
||||
# exceeds Ray's working_dir package size limit on CI.
|
||||
env_vars = {
|
||||
"PYTHONPATH": os.pathsep.join(
|
||||
filter(None, [str(VLLM_PATH), os.environ.get("PYTHONPATH")])
|
||||
),
|
||||
**{env_var: "1" for env_var in current_platform.ray_noset_device_env_vars},
|
||||
}
|
||||
ray.init(
|
||||
runtime_env={
|
||||
"working_dir": VLLM_PATH,
|
||||
"excludes": [
|
||||
"build",
|
||||
".git",
|
||||
"cmake-build-*",
|
||||
"shellcheck",
|
||||
"dist",
|
||||
"ep_kernels_workspace",
|
||||
],
|
||||
"env_vars": env_vars,
|
||||
}
|
||||
)
|
||||
|
||||
distributed_init_port = get_open_port()
|
||||
refs = []
|
||||
for rank in range(tp_size * pp_size):
|
||||
refs.append(
|
||||
test_target.remote(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
),
|
||||
)
|
||||
ray.get(refs)
|
||||
|
||||
ray.shutdown()
|
||||
try:
|
||||
refs = []
|
||||
for rank in range(tp_size * pp_size):
|
||||
refs.append(
|
||||
test_target.remote(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
),
|
||||
)
|
||||
ray.get(refs)
|
||||
finally:
|
||||
ray.shutdown()
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
||||
Reference in New Issue
Block a user