diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 4a2a55f407..c04c107815 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -534,6 +534,20 @@ else echo "--- Single-node job" echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES" + ulimit_core_hard=$(ulimit -H -c) + if [[ "$ulimit_core_hard" == "unlimited" ]]; then + # docker run can't pass "unlimited" to --ulimit + ulimit_core_hard="-1" + fi + # Disable core dumps in the ROCm test container unless the ROCm debug agent is enabled + coredump_flags="--ulimit core=0:$ulimit_core_hard" + if [[ "$commands" == *"ROCm debug agent enabled"* ]]; then + # Works around https://github.com/rocm/rocm-systems/issues/6206 + coredump_flags='-e HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"' + else + echo "ROCm debug agent not enabled, coredumps are disabled in the test container." + fi + docker run \ --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \ $RDMA_FLAGS \ @@ -541,6 +555,7 @@ else --shm-size=16gb \ --group-add "$render_gid" \ --rm \ + $coredump_flags \ -e HF_TOKEN \ -e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \ -e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \ diff --git a/tests/distributed/test_weight_transfer.py b/tests/distributed/test_weight_transfer.py index 697be7b407..ec56d24a40 100644 --- a/tests/distributed/test_weight_transfer.py +++ b/tests/distributed/test_weight_transfer.py @@ -62,7 +62,7 @@ def _get_ray_assigned_device() -> torch.device: def _set_ray_assigned_device() -> torch.device: device = _get_ray_assigned_device() - torch.accelerator.set_device(device) + torch.accelerator.set_device_index(device) return device