[ROCm] Fix CI runtime and tests for MI355 DPX (#58244)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Mahesh Kunreddi <mahesh.kunreddi@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Mahesh Kunreddi
Claude
Andreas Karatzas
Codex
Andreas Karatzas
parent
44257c2e35
commit
4013f3ad6c
@@ -605,6 +605,33 @@ initialize_native_environment() {
|
||||
fi
|
||||
}
|
||||
|
||||
check_dpx_gpu_exclusivity() {
|
||||
local devices=(/dev/dri/renderD*)
|
||||
local device_id lock_status
|
||||
|
||||
if [[ "${#devices[@]}" -ne 1 || ! -c "${devices[0]}" ]]; then
|
||||
echo "DPX guard requires exactly one render device; refusing to start tests." >&2
|
||||
return 1
|
||||
fi
|
||||
# This queue mounts the same host-local HF cache into every pod on the node.
|
||||
if ! mountpoint -q "${HF_HOME}"; then
|
||||
echo "DPX guard requires the shared HF cache mount; refusing to start tests." >&2
|
||||
return 1
|
||||
fi
|
||||
device_id=$(stat -Lc '%t-%T' "${devices[0]}") || return 1
|
||||
mkdir -p "${HF_HOME}/.vllm-dpx-locks" || return 1
|
||||
# Hold the descriptor through workload and teardown; never delete the file.
|
||||
exec {dpx_gpu_lock_fd}>>"${HF_HOME}/.vllm-dpx-locks/${device_id}.lock" || return 1
|
||||
flock -n -E 75 "${dpx_gpu_lock_fd}" && return 0
|
||||
lock_status=$?
|
||||
if [[ "${lock_status}" -eq 75 ]]; then
|
||||
echo "DPX GPU collision: ${devices[0]} (${device_id}) is already locked by another CI job; refusing to start tests." >&2
|
||||
else
|
||||
echo "DPX GPU lock failed (status ${lock_status}); refusing to start tests." >&2
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
run_native_preflight() {
|
||||
local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
|
||||
|
||||
@@ -1166,15 +1193,15 @@ collect_rocm_failure_diagnostics() {
|
||||
# Bus data identifies a card within the public node without publishing its
|
||||
# persistent UUID, serial number, or process list.
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi static -b -g all
|
||||
amd-smi static --bus --gpu all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi metric -e -k -P -x -g all
|
||||
amd-smi metric --ecc --ecc-blocks --pcie --gpu all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi bad-pages -p -r -u -g all
|
||||
amd-smi static --ras --gpu all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi metric -p -t -u -m -v -g all
|
||||
amd-smi metric --power --temperature --usage --mem-usage --gpu all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi xgmi -l -g all
|
||||
amd-smi xgmi --link-status --gpu all
|
||||
elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \
|
||||
&& command -v rocm-smi >/dev/null 2>&1; then
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \
|
||||
@@ -1485,6 +1512,10 @@ if is_native_runtime; then
|
||||
echo "Failed to initialize the native test environment"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${BUILDKITE_AGENT_META_DATA_QUEUE:-}" == *dpx* \
|
||||
&& "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" != "0" ]]; then
|
||||
check_dpx_gpu_exclusivity || exit 1
|
||||
fi
|
||||
if [[ "${commands}" == *python_only_compile.sh* ]]; then
|
||||
# This no-GPU job validates the ROCm precompiled/editable install path,
|
||||
# rather than CPU runtime platform selection.
|
||||
|
||||
Reference in New Issue
Block a user