[ROCm][Bugfix] Fix profiler in TheRock image (#56190)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
+74
-11
@@ -5,21 +5,28 @@ import importlib.util
|
||||
import os
|
||||
|
||||
|
||||
def _get_torch_cuda_version():
|
||||
"""Peripheral function to _maybe_set_cuda_compatibility_path().
|
||||
def _get_torch_root():
|
||||
"""Locate the installed torch package without importing it."""
|
||||
spec = importlib.util.find_spec("torch")
|
||||
if not spec:
|
||||
return None
|
||||
if spec.origin:
|
||||
return os.path.dirname(spec.origin)
|
||||
if spec.submodule_search_locations:
|
||||
return spec.submodule_search_locations[0]
|
||||
return None
|
||||
|
||||
|
||||
def _get_torch_version_attr(attr):
|
||||
"""Read an attribute of torch.version without importing torch.
|
||||
|
||||
PyTorch version must not be determined by importing directly
|
||||
because it will trigger the CUDA initialization, losing the
|
||||
chance to set the LD_LIBRARY_PATH beforehand.
|
||||
"""
|
||||
try:
|
||||
spec = importlib.util.find_spec("torch")
|
||||
if not spec:
|
||||
return None
|
||||
if spec.origin:
|
||||
torch_root = os.path.dirname(spec.origin)
|
||||
elif spec.submodule_search_locations:
|
||||
torch_root = spec.submodule_search_locations[0]
|
||||
else:
|
||||
torch_root = _get_torch_root()
|
||||
if not torch_root:
|
||||
return None
|
||||
version_path = os.path.join(torch_root, "version.py")
|
||||
if not os.path.exists(version_path):
|
||||
@@ -31,11 +38,16 @@ def _get_torch_cuda_version():
|
||||
module = importlib.util.module_from_spec(ver_spec)
|
||||
# Avoid registering in sys.modules to not confuse future imports
|
||||
ver_spec.loader.exec_module(module)
|
||||
return getattr(module, "cuda", None)
|
||||
return getattr(module, attr, None)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _get_torch_cuda_version():
|
||||
"""Peripheral function to _maybe_set_cuda_compatibility_path()."""
|
||||
return _get_torch_version_attr("cuda")
|
||||
|
||||
|
||||
def _maybe_set_cuda_compatibility_path():
|
||||
"""Set LD_LIBRARY_PATH for CUDA forward compatibility if enabled.
|
||||
|
||||
@@ -82,7 +94,58 @@ def _maybe_set_cuda_compatibility_path():
|
||||
os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(new_paths)
|
||||
|
||||
|
||||
def _maybe_promote_torch_symbols_for_rocm():
|
||||
"""Put libtorch_cpu.so in the global symbol scope on ROCm, for GPU profiling.
|
||||
|
||||
Must run before 'import torch'. libkineto advertises itself to
|
||||
rocprofiler-sdk by exporting rocprofiler_configure from libtorch_cpu.so,
|
||||
and rocprofiler-sdk looks for that symbol exactly once, from a lazy
|
||||
initializer that runs *during* 'import torch'. CPython loads extension
|
||||
modules RTLD_LOCAL, so unless something has already placed the symbol in
|
||||
the global scope the lookup misses and no client is ever registered.
|
||||
|
||||
The failure is silent and total: no queue interception, so roctracer
|
||||
yields no GPU records, while torch.profiler still writes a complete
|
||||
CPU-only trace and reports success.
|
||||
|
||||
Best effort. If the library cannot be loaded early we leave the process
|
||||
exactly as it would have been, losing only GPU tracing.
|
||||
"""
|
||||
if _get_torch_version_attr("hip") is None:
|
||||
return
|
||||
try:
|
||||
import ctypes
|
||||
import glob
|
||||
|
||||
torch_root = _get_torch_root()
|
||||
if not torch_root:
|
||||
return
|
||||
lib = os.path.join(torch_root, "lib", "libtorch_cpu.so")
|
||||
if not os.path.exists(lib):
|
||||
return
|
||||
try:
|
||||
ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL)
|
||||
except OSError:
|
||||
# Some ROCm wheel layouts leave a stale RUNPATH in libtorch_cpu.so,
|
||||
# so a dependency that resolves once torch has set things up does
|
||||
# not resolve for a standalone load. Satisfy it by search, then
|
||||
# retry; give up quietly if it is not where we expect.
|
||||
site_root = os.path.dirname(torch_root)
|
||||
deps = glob.glob(
|
||||
os.path.join(
|
||||
site_root, "_rocm_sdk_*", "lib", "*", "lib", "librocm-openblas.so.*"
|
||||
)
|
||||
)
|
||||
if not deps:
|
||||
return
|
||||
ctypes.CDLL(deps[0], mode=ctypes.RTLD_LOCAL)
|
||||
ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
_maybe_set_cuda_compatibility_path()
|
||||
_maybe_promote_torch_symbols_for_rocm()
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
Reference in New Issue
Block a user