[ROCm][Perf][Bugfix] Multi-stream perf improvements; rocprofiler fixes (#55099)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
@@ -6,7 +6,6 @@ DeepEP test utilities
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from typing import Concatenate
|
||||
@@ -16,7 +15,6 @@ from torch.distributed import ProcessGroup
|
||||
from torch.multiprocessing import spawn # pyright: ignore[reportPrivateImportUsage]
|
||||
from typing_extensions import ParamSpec
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
@@ -74,7 +72,6 @@ def _worker_parallel_launch(
|
||||
barrier = torch.tensor([rank], device=device)
|
||||
torch.distributed.all_reduce(barrier)
|
||||
|
||||
exit_code = 0
|
||||
try:
|
||||
worker(
|
||||
ProcessGroupInfo(
|
||||
@@ -91,18 +88,9 @@ def _worker_parallel_launch(
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
traceback.print_exc()
|
||||
exit_code = 1
|
||||
raise
|
||||
finally:
|
||||
torch.distributed.destroy_process_group()
|
||||
if current_platform.is_rocm():
|
||||
# Bypass a ROCm teardown use-after-free (SIGSEGV in HIP's atexit
|
||||
# handler; fixed by https://github.com/ROCm/rocm-systems/pull/6942).
|
||||
# Flush first since os._exit skips it, to keep a failing traceback.
|
||||
# TODO (Rohan138): Remove after ROCm 10.0 lands
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
os._exit(exit_code)
|
||||
|
||||
|
||||
def parallel_launch(
|
||||
|
||||
@@ -488,17 +488,10 @@ def test_deep_ep_moe(
|
||||
world_dp_size: tuple[int, int],
|
||||
per_act_token_quant: bool,
|
||||
workspace_init,
|
||||
monkeypatch,
|
||||
):
|
||||
low_latency_mode = False
|
||||
use_fp8_dispatch = False
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# The cooperative kernel launches on this path segfault inside the ROCm
|
||||
# HSA runtime at process exit while rocprofiler-sdk is attached, and
|
||||
# torch attaches it implicitly. Workers are spawned, so they inherit it.
|
||||
monkeypatch.setenv("ROCPROFILER_REGISTER_ENABLED", "0")
|
||||
|
||||
set_random_seed(7)
|
||||
world_size, dp_size = world_dp_size
|
||||
config = TestConfig(dtype=dtype, topk=topk, m=m, k=k, n=n, num_experts=num_experts)
|
||||
|
||||
Reference in New Issue
Block a user