[Hardware][AMD][CI] Toggle test coredumps on ROCm debug agent (#47222)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
Matt
2026-06-30 23:30:10 -05:00
committed by GitHub
parent 9969466a59
commit 3c1396bab6
2 changed files with 16 additions and 1 deletions
@@ -534,6 +534,20 @@ else
echo "--- Single-node job"
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
ulimit_core_hard=$(ulimit -H -c)
if [[ "$ulimit_core_hard" == "unlimited" ]]; then
# docker run can't pass "unlimited" to --ulimit
ulimit_core_hard="-1"
fi
# Disable core dumps in the ROCm test container unless the ROCm debug agent is enabled
coredump_flags="--ulimit core=0:$ulimit_core_hard"
if [[ "$commands" == *"ROCm debug agent enabled"* ]]; then
# Works around https://github.com/rocm/rocm-systems/issues/6206
coredump_flags='-e HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"'
else
echo "ROCm debug agent not enabled, coredumps are disabled in the test container."
fi
docker run \
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
$RDMA_FLAGS \
@@ -541,6 +555,7 @@ else
--shm-size=16gb \
--group-add "$render_gid" \
--rm \
$coredump_flags \
-e HF_TOKEN \
-e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
-e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \
+1 -1
View File
@@ -62,7 +62,7 @@ def _get_ray_assigned_device() -> torch.device:
def _set_ray_assigned_device() -> torch.device:
device = _get_ray_assigned_device()
torch.accelerator.set_device(device)
torch.accelerator.set_device_index(device)
return device