diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base index 639ddd01f5..1b1d089881 100644 --- a/docker/Dockerfile.rock_base +++ b/docker/Dockerfile.rock_base @@ -10,6 +10,8 @@ ARG AITER_BRANCH="v0.1.21.post1" ARG AITER_REPO="https://github.com/ROCm/aiter.git" ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0 ARG MORI_REPO="https://github.com/ROCm/mori.git" +ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git" +ARG ROCM_RUNTIME_COMMIT="25e14349f2606a6b95c4ccd45eb92a1835646cf2" # TheRock configuration @@ -246,6 +248,61 @@ RUN pip install pyyaml && cd aiter \ RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install +### +### ROCr + CLR build +### +# Brings in multi-stream perf improvements; drop after these arrive in an official release +FROM base AS build_rocm_runtime +ARG ROCM_SYSTEMS_REPO +ARG ROCM_RUNTIME_COMMIT +# libgl-dev: rocclr does find_package(OpenGL REQUIRED) unconditionally and the wheel image has +# no GL headers. The link must not pick up a libGL DT_NEEDED, which is asserted after the build. +RUN apt-get update \ + && apt-get install -y --no-install-recommends g++ libdw-dev xxd libgl-dev libglx-dev \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir CppHeaderParser +RUN git init -q /src && cd /src \ + && git remote add origin ${ROCM_SYSTEMS_REPO} \ + && git config core.sparseCheckout true \ + && git config remote.origin.promisor true \ + && git config remote.origin.partialclonefilter blob:none \ + && git sparse-checkout init --cone \ + && git sparse-checkout set projects/rocr-runtime projects/clr projects/hip shared cmake \ + && git fetch --filter=blob:none --depth 1 origin ${ROCM_RUNTIME_COMMIT} \ + && git checkout -q FETCH_HEAD +# libelf is taken from the SDK's own vendored sysdeps rather than apt -- the wheel image has no +# /usr/include/libelf.h -- so ROCr links against the copy the wheel itself was built against. +ENV SYSDEPS="$SDK_DEV/lib/rocm_sysdeps" +# ROCr first, and installed into $SDK_DEV, because the clr build below needs its headers. +RUN cd /src/projects/rocr-runtime \ + && cmake -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="$SDK_DEV;$SYSDEPS" \ + -DCMAKE_INSTALL_PREFIX="$SDK_DEV" \ + -DLIBELF_INCLUDE_DIRS="$SYSDEPS/include" \ + -DLIBELF_LIBRARIES="$SYSDEPS/lib/libelf.so" \ + && cmake --build build --parallel 16 \ + && cmake --install build \ + && cmake --install build --prefix /rocr-install --strip +RUN cd /src/projects/clr \ + && cmake -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCLR_BUILD_HIP=ON \ + -DCLR_BUILD_OCL=OFF \ + -DROCM_KPACK_ENABLED=ON \ + -DHIP_COMMON_DIR=/src/projects/hip \ + -DROCM_PATH="$SDK_DEV" \ + -DCMAKE_PREFIX_PATH="$SDK_DEV;$SYSDEPS" \ + -DCMAKE_INSTALL_PREFIX="$SDK_DEV" \ + -DHIP_LLVM_ROOT="$SDK_DEV/lib/llvm" \ + && cmake --build build --parallel 16 \ + && cmake --install build --prefix /clr-install --strip +RUN mkdir -p /staging/lib \ + && cp -P /rocr-install/lib/libhsa-runtime64.so* /staging/lib/ \ + && cp -P /clr-install/lib/libamdhip64.so* /staging/lib/ \ + && ls -l /staging/lib + + ### ### Final Build ### @@ -274,6 +331,25 @@ FROM base AS final # teardown, so disable it. TODO: drop once ROCm's stock rocprofiler carries the fix. ENV ROCPROFILER_QUEUE_INTERPOSITION=0 +COPY --from=build_rocm_runtime /staging/lib/ ${SDK_DEV}/lib/ +# Hard link the patched library onto every stock file name, in both SDK trees, so that the +# SONAME lookup and the hard coded paths all resolve to it. Hard links rather than symlinks +# because ldconfig rewrites SONAME symlinks but cannot re-point an inode. $SDK_CORE keeps its +# own libhsa-runtime64 copy, which is on LD_LIBRARY_PATH and in ld.so.conf; relinking it is +# belt and braces rather than strictly required, but it leaves no stock ROCr in the image. +RUN set -eux; \ + cd "$SDK_DEV/lib"; \ + ours_hsa="$(readlink -f libhsa-runtime64.so.1)"; \ + ours_hip="$(readlink -f libamdhip64.so.7)"; \ + for f in "$SDK_DEV"/lib/libhsa-runtime64.so.1* "$SDK_CORE"/lib/libhsa-runtime64.so.1* \ + "$SDK_DEV"/lib/libamdhip64.so.7* "$SDK_CORE"/lib/libamdhip64.so.7*; do \ + [ -f "$f" ] && [ ! -L "$f" ] || continue; \ + case "${f##*/}" in libhsa*) ours="$ours_hsa";; *) ours="$ours_hip";; esac; \ + [ "$f" -ef "$ours" ] || ln -f "$ours" "$f"; \ + done + +RUN ldconfig + ARG TRITON_VERSION RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \ pip install /install/*.whl \ @@ -304,6 +380,8 @@ ARG AITER_BRANCH ARG AITER_REPO ARG MORI_BRANCH ARG MORI_REPO +ARG ROCM_SYSTEMS_REPO +ARG ROCM_RUNTIME_COMMIT # Record hashes for Triton and pybind11 so we can verify later. RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ && echo "PYTHON_VERSION: ${PYTHON_VERSION}" >> /app/versions.txt \ @@ -319,6 +397,8 @@ RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \ && echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \ && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt \ + && echo "ROCM_SYSTEMS_REPO: ${ROCM_SYSTEMS_REPO}" >> /app/versions.txt \ + && echo "ROCM_RUNTIME_COMMIT: ${ROCM_RUNTIME_COMMIT}" >> /app/versions.txt \ # Fingerprint our triton so Dockerfile.rocm can prove it survived the build. && echo "TRITON_VERSION: $(pip show triton | awk '/^Version:/ {print $2}')" >> /app/versions.txt \ && echo "TRITON_SHA256: $(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \ diff --git a/docker/Dockerfile.rocm_base b/docker/Dockerfile.rocm_base index 1634c1f13b..c7c5c7b4ea 100644 --- a/docker/Dockerfile.rocm_base +++ b/docker/Dockerfile.rocm_base @@ -13,10 +13,9 @@ ARG AITER_BRANCH="v0.1.21.post1" ARG AITER_REPO="https://github.com/ROCm/aiter.git" ARG MORI_BRANCH="v1.1.0" ARG MORI_REPO="https://github.com/ROCm/mori.git" -ARG ROCPROFILER_SDK_REPO="https://github.com/ROCm/rocm-systems.git" +ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git" ARG ROCPROFILER_SDK_COMMIT="2b22ab0195cc1461cd9abf3b969e9dd7c10af350" -ARG ROCM_RUNTIME_REPO="https://github.com/mawong-amd/rocm-systems.git" -ARG ROCM_RUNTIME_COMMIT="c7e40e16bca10311af72aee09cfa0cd1726f70a7" +ARG ROCM_RUNTIME_COMMIT="b539bf7eebfd99ad0a69668caa1f4037034d501f" # Sccache configuration (only used in release pipeline) ARG USE_SCCACHE @@ -27,85 +26,12 @@ ARG SCCACHE_REGION_NAME=us-west-2 ARG SCCACHE_S3_NO_CREDENTIALS=0 -### -### ROCr + CLR build -### -# Pull in updates to ROCr + clr from ROCm 7.2.4 and beyond. Happens before base so they -# are present in each downstream stage -FROM ${BASE_IMAGE} AS build_rocm_runtime -ARG ROCM_RUNTIME_REPO -ARG ROCM_RUNTIME_COMMIT -RUN apt-get update \ - && apt-get install -y --no-install-recommends git cmake ninja-build g++ pkg-config \ - python3-pip libelf-dev libdrm-dev libnuma-dev libdw-dev xxd \ - && rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir CppHeaderParser -RUN git init -q /src && cd /src \ - && git remote add origin ${ROCM_RUNTIME_REPO} \ - && git config core.sparseCheckout true \ - && git config remote.origin.promisor true \ - && git config remote.origin.partialclonefilter blob:none \ - && git sparse-checkout init --cone \ - && git sparse-checkout set projects/rocr-runtime projects/clr projects/hip shared cmake \ - && git fetch --filter=blob:none --depth 1 origin ${ROCM_RUNTIME_COMMIT} \ - && git checkout -q FETCH_HEAD -# ROCr's trap_handler and blit_shaders call find_package(Clang/LLVM REQUIRED) only to get -# IMPORTED executables. The ROCm dev image ships the binaries but no CMake package config, so -# supply one pointing at them. -RUN mkdir -p /opt/rocm-cmake-shim \ - && printf 'if(NOT TARGET clang)\n add_executable(clang IMPORTED GLOBAL)\n set_target_properties(clang PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/clang")\nendif()\nset(Clang_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/ClangConfig.cmake \ - && printf 'if(NOT TARGET llvm-objcopy)\n add_executable(llvm-objcopy IMPORTED GLOBAL)\n set_target_properties(llvm-objcopy PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/llvm-objcopy")\nendif()\nset(LLVM_FOUND TRUE)\nset(LLVM_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/LLVMConfig.cmake -# ROCr first, and installed into /opt/rocm, because the clr build below needs its headers -RUN cd /src/projects/rocr-runtime \ - && cmake -B build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=/opt/rocm \ - -DCMAKE_INSTALL_PREFIX=/opt/rocm \ - -DClang_DIR=/opt/rocm-cmake-shim \ - -DLLVM_DIR=/opt/rocm-cmake-shim \ - && cmake --build build --parallel 16 \ - && cmake --install build \ - && cmake --install build --prefix /rocr-install --strip -RUN cd /src/projects/clr \ - && cmake -B build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCLR_BUILD_HIP=ON \ - -DCLR_BUILD_OCL=OFF \ - -DHIP_COMMON_DIR=/src/projects/hip \ - -DROCM_PATH=/opt/rocm \ - -DCMAKE_PREFIX_PATH=/opt/rocm \ - -DCMAKE_INSTALL_PREFIX=/opt/rocm \ - -DLLVM_DIR=/opt/rocm-cmake-shim \ - -DHIP_LLVM_ROOT=/opt/rocm/llvm \ - && cmake --build build --parallel 16 \ - && cmake --install build --prefix /clr-install --strip -RUN mkdir -p /staging/lib \ - && cp -P /rocr-install/lib/libhsa-runtime64.so* /staging/lib/ \ - && cp -P /clr-install/lib/libamdhip64.so* /staging/lib/ \ - && ls -l /staging/lib - - FROM ${BASE_IMAGE} AS base ENV PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ENV ROCM_PATH=/opt/rocm ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib: -COPY --from=build_rocm_runtime /staging/ /opt/rocm/ -# Hard link the patched library onto every stock file name so that both the -# SONAME lookup and the hard coded paths resolve to it. -RUN set -eux; \ - cd /opt/rocm/lib; \ - ours_hsa="$(readlink -f libhsa-runtime64.so.1)"; \ - ours_hip="$(readlink -f libamdhip64.so.7)"; \ - for f in libhsa-runtime64.so.1.*; do \ - [ "$f" = "${ours_hsa##*/}" ] || ln -f "$ours_hsa" "$f"; \ - done; \ - for f in libamdhip64.so.7.*; do \ - [ "$f" = "${ours_hip##*/}" ] || ln -f "$ours_hip" "$f"; \ - done; \ - ldconfig - ARG PYTORCH_ROCM_ARCH=gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151 ENV PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} ENV AITER_ROCM_ARCH=gfx942;gfx950 @@ -195,7 +121,7 @@ RUN cd triton \ && git checkout ${TRITON_BRANCH} \ && git config --global user.email "you@example.com" && git config --global user.name "Your Name" \ && if [ ! -f setup.py ]; then cd python; fi \ - && python3 setup.py bdist_wheel --dist-dir=dist \ + && MAX_JOBS=16 python3 setup.py bdist_wheel --dist-dir=dist \ && mkdir -p /app/install && cp dist/*.whl /app/install @@ -392,16 +318,72 @@ RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \ cp /install/*.whl /app/debs +### +### ROCr + CLR build +### +# Pull in updates to ROCr + clr from ROCm 7.2.4 and beyond. +FROM base AS build_rocm_runtime +ARG ROCM_SYSTEMS_REPO +ARG ROCM_RUNTIME_COMMIT +RUN apt-get update \ + && apt-get install -y --no-install-recommends g++ \ + libelf-dev libdrm-dev libnuma-dev libdw-dev xxd \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir CppHeaderParser +RUN git init -q /src && cd /src \ + && git remote add origin ${ROCM_SYSTEMS_REPO} \ + && git config core.sparseCheckout true \ + && git config remote.origin.promisor true \ + && git config remote.origin.partialclonefilter blob:none \ + && git sparse-checkout init --cone \ + && git sparse-checkout set projects/rocr-runtime projects/clr projects/hip shared cmake \ + && git fetch --filter=blob:none --depth 1 origin ${ROCM_RUNTIME_COMMIT} \ + && git checkout -q FETCH_HEAD +# ROCr's trap_handler and blit_shaders call find_package(Clang/LLVM REQUIRED) only to get +# IMPORTED executables. The ROCm dev image ships the binaries but no CMake package config, so +# supply one pointing at them. +RUN mkdir -p /opt/rocm-cmake-shim \ + && printf 'if(NOT TARGET clang)\n add_executable(clang IMPORTED GLOBAL)\n set_target_properties(clang PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/clang")\nendif()\nset(Clang_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/ClangConfig.cmake \ + && printf 'if(NOT TARGET llvm-objcopy)\n add_executable(llvm-objcopy IMPORTED GLOBAL)\n set_target_properties(llvm-objcopy PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/llvm-objcopy")\nendif()\nset(LLVM_FOUND TRUE)\nset(LLVM_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/LLVMConfig.cmake +# ROCr first, and installed into /opt/rocm, because the clr build below needs its headers +RUN cd /src/projects/rocr-runtime \ + && cmake -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=/opt/rocm \ + -DCMAKE_INSTALL_PREFIX=/opt/rocm \ + -DClang_DIR=/opt/rocm-cmake-shim \ + -DLLVM_DIR=/opt/rocm-cmake-shim \ + && cmake --build build --parallel 16 \ + && cmake --install build \ + && cmake --install build --prefix /rocr-install --strip +RUN cd /src/projects/clr \ + && cmake -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCLR_BUILD_HIP=ON \ + -DCLR_BUILD_OCL=OFF \ + -DHIP_COMMON_DIR=/src/projects/hip \ + -DROCM_PATH=/opt/rocm \ + -DCMAKE_PREFIX_PATH=/opt/rocm \ + -DCMAKE_INSTALL_PREFIX=/opt/rocm \ + -DLLVM_DIR=/opt/rocm-cmake-shim \ + -DHIP_LLVM_ROOT=/opt/rocm/llvm \ + && cmake --build build --parallel 16 \ + && cmake --install build --prefix /clr-install --strip +RUN mkdir -p /staging/lib \ + && cp -P /rocr-install/lib/libhsa-runtime64.so* /staging/lib/ \ + && cp -P /clr-install/lib/libamdhip64.so* /staging/lib/ \ + && ls -l /staging/lib + # rocprofiler-sdk 1.3.2 fixes torch 2.12 profiler on ROCm 7.2.3 FROM base AS build_rocprofiler_sdk -ARG ROCPROFILER_SDK_REPO +ARG ROCM_SYSTEMS_REPO ARG ROCPROFILER_SDK_COMMIT ARG USE_SCCACHE RUN apt-get update \ && apt-get install -y --no-install-recommends libdw-dev libelf-dev libsqlite3-dev libzstd-dev \ && rm -rf /var/lib/apt/lists/* RUN git init -q /src && cd /src \ - && git remote add origin ${ROCPROFILER_SDK_REPO} \ + && git remote add origin ${ROCM_SYSTEMS_REPO} \ && git config core.sparseCheckout true \ && git config remote.origin.promisor true \ && git config remote.origin.partialclonefilter blob:none \ @@ -417,8 +399,8 @@ ARG ROCPROFILER_SDK_PR_7796=fae7fd57ffb8a76e811d6410b8a0cee1e94d710a ARG ROCPROFILER_SDK_PR_7924=f8db22adbe17686a9b05ae03367dd1ec73011653 RUN cd /src \ && HSA_PATHS='projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/*' \ - && curl -fsSL -o /tmp/qi-7796.patch "${ROCPROFILER_SDK_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7796}.patch" \ - && curl -fsSL -o /tmp/qi-7924.patch "${ROCPROFILER_SDK_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7924}.patch" \ + && curl -fsSL -o /tmp/qi-7796.patch "${ROCM_SYSTEMS_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7796}.patch" \ + && curl -fsSL -o /tmp/qi-7924.patch "${ROCM_SYSTEMS_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7924}.patch" \ && git apply -p1 --include="$HSA_PATHS" --verbose /tmp/qi-7796.patch \ && git apply -p1 --include="$HSA_PATHS" --verbose /tmp/qi-7924.patch RUN cd /src/projects/rocprofiler-sdk \ @@ -441,6 +423,22 @@ RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \ pip install /install/*.whl COPY --from=build_rocprofiler_sdk /staging/ /opt/rocm/ +ENV ROCPROFILER_QUEUE_INTERPOSITION=0 + +COPY --from=build_rocm_runtime /staging/ /opt/rocm/ +# Hard link the patched library onto every stock file name so that both the +# SONAME lookup and the hard coded paths resolve to it. +RUN set -eux; \ + cd /opt/rocm/lib; \ + ours_hsa="$(readlink -f libhsa-runtime64.so.1)"; \ + ours_hip="$(readlink -f libamdhip64.so.7)"; \ + for f in libhsa-runtime64.so.1.*; do \ + [ "$f" = "${ours_hsa##*/}" ] || ln -f "$ours_hsa" "$f"; \ + done; \ + for f in libamdhip64.so.7.*; do \ + [ "$f" = "${ours_hip##*/}" ] || ln -f "$ours_hip" "$f"; \ + done + RUN ldconfig ARG BASE_IMAGE @@ -458,14 +456,12 @@ ARG AITER_BRANCH ARG AITER_REPO ARG MORI_BRANCH ARG MORI_REPO -ARG ROCPROFILER_SDK_REPO +ARG ROCM_SYSTEMS_REPO ARG ROCPROFILER_SDK_COMMIT -ARG ROCM_RUNTIME_REPO ARG ROCM_RUNTIME_COMMIT RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ - && echo "ROCPROFILER_SDK_REPO: ${ROCPROFILER_SDK_REPO}" >> /app/versions.txt \ + && echo "ROCM_SYSTEMS_REPO: ${ROCM_SYSTEMS_REPO}" >> /app/versions.txt \ && echo "ROCPROFILER_SDK_COMMIT: ${ROCPROFILER_SDK_COMMIT}" >> /app/versions.txt \ - && echo "ROCM_RUNTIME_REPO: ${ROCM_RUNTIME_REPO}" >> /app/versions.txt \ && echo "ROCM_RUNTIME_COMMIT: ${ROCM_RUNTIME_COMMIT}" >> /app/versions.txt \ && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \ && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \ diff --git a/tests/kernels/moe/parallel_utils.py b/tests/kernels/moe/parallel_utils.py index 439e624252..f926b034b3 100644 --- a/tests/kernels/moe/parallel_utils.py +++ b/tests/kernels/moe/parallel_utils.py @@ -6,7 +6,6 @@ DeepEP test utilities import dataclasses import os -import sys import traceback from collections.abc import Callable from typing import Concatenate @@ -16,7 +15,6 @@ from torch.distributed import ProcessGroup from torch.multiprocessing import spawn # pyright: ignore[reportPrivateImportUsage] from typing_extensions import ParamSpec -from vllm.platforms import current_platform from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2 from vllm.utils.network_utils import get_open_port @@ -74,7 +72,6 @@ def _worker_parallel_launch( barrier = torch.tensor([rank], device=device) torch.distributed.all_reduce(barrier) - exit_code = 0 try: worker( ProcessGroupInfo( @@ -91,18 +88,9 @@ def _worker_parallel_launch( except Exception as ex: print(ex) traceback.print_exc() - exit_code = 1 raise finally: torch.distributed.destroy_process_group() - if current_platform.is_rocm(): - # Bypass a ROCm teardown use-after-free (SIGSEGV in HIP's atexit - # handler; fixed by https://github.com/ROCm/rocm-systems/pull/6942). - # Flush first since os._exit skips it, to keep a failing traceback. - # TODO (Rohan138): Remove after ROCm 10.0 lands - sys.stdout.flush() - sys.stderr.flush() - os._exit(exit_code) def parallel_launch( diff --git a/tests/kernels/moe/test_deepep_moe.py b/tests/kernels/moe/test_deepep_moe.py index 8aa1be8423..ce9e4f343d 100644 --- a/tests/kernels/moe/test_deepep_moe.py +++ b/tests/kernels/moe/test_deepep_moe.py @@ -488,17 +488,10 @@ def test_deep_ep_moe( world_dp_size: tuple[int, int], per_act_token_quant: bool, workspace_init, - monkeypatch, ): low_latency_mode = False use_fp8_dispatch = False - if current_platform.is_rocm(): - # The cooperative kernel launches on this path segfault inside the ROCm - # HSA runtime at process exit while rocprofiler-sdk is attached, and - # torch attaches it implicitly. Workers are spawned, so they inherit it. - monkeypatch.setenv("ROCPROFILER_REGISTER_ENABLED", "0") - set_random_seed(7) world_size, dp_size = world_dp_size config = TestConfig(dtype=dtype, topk=topk, m=m, k=k, n=n, num_experts=num_experts)