[ROCm][Bugfix] Fix profiler in TheRock image (#56190)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
Matt
2026-09-10 13:39:33 -05:00
committed by GitHub
parent 7cdd9304ae
commit 48cb12c184
+74 -11
View File
@@ -5,21 +5,28 @@ import importlib.util
import os
def _get_torch_cuda_version():
"""Peripheral function to _maybe_set_cuda_compatibility_path().
def _get_torch_root():
"""Locate the installed torch package without importing it."""
spec = importlib.util.find_spec("torch")
if not spec:
return None
if spec.origin:
return os.path.dirname(spec.origin)
if spec.submodule_search_locations:
return spec.submodule_search_locations[0]
return None
def _get_torch_version_attr(attr):
"""Read an attribute of torch.version without importing torch.
PyTorch version must not be determined by importing directly
because it will trigger the CUDA initialization, losing the
chance to set the LD_LIBRARY_PATH beforehand.
"""
try:
spec = importlib.util.find_spec("torch")
if not spec:
return None
if spec.origin:
torch_root = os.path.dirname(spec.origin)
elif spec.submodule_search_locations:
torch_root = spec.submodule_search_locations[0]
else:
torch_root = _get_torch_root()
if not torch_root:
return None
version_path = os.path.join(torch_root, "version.py")
if not os.path.exists(version_path):
@@ -31,11 +38,16 @@ def _get_torch_cuda_version():
module = importlib.util.module_from_spec(ver_spec)
# Avoid registering in sys.modules to not confuse future imports
ver_spec.loader.exec_module(module)
return getattr(module, "cuda", None)
return getattr(module, attr, None)
except Exception:
return None
def _get_torch_cuda_version():
"""Peripheral function to _maybe_set_cuda_compatibility_path()."""
return _get_torch_version_attr("cuda")
def _maybe_set_cuda_compatibility_path():
"""Set LD_LIBRARY_PATH for CUDA forward compatibility if enabled.
@@ -82,7 +94,58 @@ def _maybe_set_cuda_compatibility_path():
os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(new_paths)
def _maybe_promote_torch_symbols_for_rocm():
"""Put libtorch_cpu.so in the global symbol scope on ROCm, for GPU profiling.
Must run before 'import torch'. libkineto advertises itself to
rocprofiler-sdk by exporting rocprofiler_configure from libtorch_cpu.so,
and rocprofiler-sdk looks for that symbol exactly once, from a lazy
initializer that runs *during* 'import torch'. CPython loads extension
modules RTLD_LOCAL, so unless something has already placed the symbol in
the global scope the lookup misses and no client is ever registered.
The failure is silent and total: no queue interception, so roctracer
yields no GPU records, while torch.profiler still writes a complete
CPU-only trace and reports success.
Best effort. If the library cannot be loaded early we leave the process
exactly as it would have been, losing only GPU tracing.
"""
if _get_torch_version_attr("hip") is None:
return
try:
import ctypes
import glob
torch_root = _get_torch_root()
if not torch_root:
return
lib = os.path.join(torch_root, "lib", "libtorch_cpu.so")
if not os.path.exists(lib):
return
try:
ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL)
except OSError:
# Some ROCm wheel layouts leave a stale RUNPATH in libtorch_cpu.so,
# so a dependency that resolves once torch has set things up does
# not resolve for a standalone load. Satisfy it by search, then
# retry; give up quietly if it is not where we expect.
site_root = os.path.dirname(torch_root)
deps = glob.glob(
os.path.join(
site_root, "_rocm_sdk_*", "lib", "*", "lib", "librocm-openblas.so.*"
)
)
if not deps:
return
ctypes.CDLL(deps[0], mode=ctypes.RTLD_LOCAL)
ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL)
except Exception:
return
_maybe_set_cuda_compatibility_path()
_maybe_promote_torch_symbols_for_rocm()
import torch