[ROCm][Perf][Bugfix] Multi-stream perf improvements; rocprofiler fixes (#55099)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
Matt
2026-09-08 17:42:34 -07:00
committed by GitHub
parent d98c8c030b
commit bc8587f829
4 changed files with 161 additions and 104 deletions
-12
View File
@@ -6,7 +6,6 @@ DeepEP test utilities
import dataclasses
import os
import sys
import traceback
from collections.abc import Callable
from typing import Concatenate
@@ -16,7 +15,6 @@ from torch.distributed import ProcessGroup
from torch.multiprocessing import spawn # pyright: ignore[reportPrivateImportUsage]
from typing_extensions import ParamSpec
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2
from vllm.utils.network_utils import get_open_port
@@ -74,7 +72,6 @@ def _worker_parallel_launch(
barrier = torch.tensor([rank], device=device)
torch.distributed.all_reduce(barrier)
exit_code = 0
try:
worker(
ProcessGroupInfo(
@@ -91,18 +88,9 @@ def _worker_parallel_launch(
except Exception as ex:
print(ex)
traceback.print_exc()
exit_code = 1
raise
finally:
torch.distributed.destroy_process_group()
if current_platform.is_rocm():
# Bypass a ROCm teardown use-after-free (SIGSEGV in HIP's atexit
# handler; fixed by https://github.com/ROCm/rocm-systems/pull/6942).
# Flush first since os._exit skips it, to keep a failing traceback.
# TODO (Rohan138): Remove after ROCm 10.0 lands
sys.stdout.flush()
sys.stderr.flush()
os._exit(exit_code)
def parallel_launch(
-7
View File
@@ -488,17 +488,10 @@ def test_deep_ep_moe(
world_dp_size: tuple[int, int],
per_act_token_quant: bool,
workspace_init,
monkeypatch,
):
low_latency_mode = False
use_fp8_dispatch = False
if current_platform.is_rocm():
# The cooperative kernel launches on this path segfault inside the ROCm
# HSA runtime at process exit while rocprofiler-sdk is attached, and
# torch attaches it implicitly. Workers are spawned, so they inherit it.
monkeypatch.setenv("ROCPROFILER_REGISTER_ENABLED", "0")
set_random_seed(7)
world_size, dp_size = world_dp_size
config = TestConfig(dtype=dtype, topk=topk, m=m, k=k, n=n, num_experts=num_experts)