[CI][AMD] Improve Kubernetes failure diagnostics (#52264)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-17 18:09:12 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent 455edc022b
commit 60c3a31b12
+447 -109
View File
@@ -44,11 +44,14 @@ amd_diagnostics_dir="${VLLM_CI_DIAGNOSTICS_DIR:-artifacts/amd-gpu-diagnostics}"
amd_diagnostics_checkout_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd -P)}"
amd_diagnostics_execution_mode="${VLLM_CI_EXECUTION_MODE:-single-node}"
amd_diagnostics_test_group="${VLLM_TEST_GROUP_NAME:-${BUILDKITE_STEP_KEY:-unknown}}"
amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-}"
amd_diagnostics_workspace="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
amd_diagnostics_pod_name="${VLLM_CI_K8S_POD_NAME:-${POD_NAME:-${HOSTNAME:-}}}"
amd_diagnostics_k8s_namespace="${VLLM_CI_K8S_NAMESPACE:-${POD_NAMESPACE:-unknown}}"
amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-${NODE_NAME:-}}"
amd_diagnostics_collected=0
amd_diagnostics_memory_events_path=""
amd_diagnostics_probe_budget_seconds=25
amd_diagnostics_command_timeout_seconds=5
amd_diagnostics_upload_timeout_seconds=20
if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
fi
@@ -101,6 +104,8 @@ clear_ci_orchestration_env() {
VLLM_CI_ARTIFACT_GLOB \
VLLM_CI_ARTIFACT_CHECKSUM_GLOB \
VLLM_CI_EXPECTED_GPU_COUNT \
VLLM_CI_K8S_POD_NAME \
VLLM_CI_K8S_NAMESPACE \
VLLM_CI_K8S_NODE_NAME \
VLLM_CI_USE_ARTIFACTS \
VLLM_CI_RESULTS_ROOT \
@@ -577,7 +582,24 @@ is_multi_node() {
return 1
}
run_amd_diagnostic() {
append_failure_diagnostic_section() {
local log_file=$1
local title=$2
printf '\n===============================================================================\n' \
>> "${log_file}"
printf '%s\n' "${title}" >> "${log_file}"
printf '===============================================================================\n' \
>> "${log_file}"
}
append_failure_diagnostic_note() {
local log_file=$1
shift
printf '%s\n' "$@" >> "${log_file}"
}
run_failure_diagnostic() {
local log_file=$1
local probe_deadline=$2
shift 2
@@ -589,23 +611,285 @@ run_amd_diagnostic() {
printf '\n$'
printf ' %q' "$@"
printf '\n'
} | tee -a "${log_file}"
} >> "${log_file}"
if ! command -v timeout >/dev/null 2>&1; then
echo "[timeout unavailable; bounded diagnostic command skipped]" \
>> "${log_file}"
return 0
fi
if [[ "${remaining_seconds}" -le 0 ]]; then
echo "[diagnostic probe budget exhausted; command skipped]" \
| tee -a "${log_file}"
>> "${log_file}"
return 0
fi
if [[ "${remaining_seconds}" -lt "${command_timeout_seconds}" ]]; then
command_timeout_seconds=${remaining_seconds}
fi
timeout --kill-after=1s "${command_timeout_seconds}s" "$@" 2>&1 \
| tee -a "${log_file}"
command_status=${PIPESTATUS[0]}
timeout --kill-after=1s "${command_timeout_seconds}s" "$@" \
>> "${log_file}" 2>&1
command_status=$?
if [[ "${command_status}" -ne 0 ]]; then
echo "[diagnostic command exited with status ${command_status}]" \
| tee -a "${log_file}"
>> "${log_file}"
fi
return 0
}
append_failure_diagnostic_file() {
local log_file=$1
local label=$2
local path=$3
local command_status=0
printf '\n%s:\n' "${label}" >> "${log_file}"
printf '$ cat %q\n' "${path}" >> "${log_file}"
if [[ ! -r "${path}" ]]; then
echo "[file unavailable]" >> "${log_file}"
return 0
fi
cat "${path}" >> "${log_file}" 2>&1
command_status=$?
if [[ "${command_status}" -ne 0 ]]; then
echo "[diagnostic file read exited with status ${command_status}]" \
>> "${log_file}"
fi
return 0
}
decode_mountinfo_path() {
local value=$1
value=${value//\\040/ }
value=${value//\\011/$'\t'}
value=${value//\\012/$'\n'}
value=${value//\\134/\\}
printf '%s\n' "${value}"
}
resolve_current_cgroup_v2_dir() {
local cgroup_path=""
local mount_root=""
local mount_point=""
local relative_path=""
local candidate=""
cgroup_path=$(awk -F: '$2 == "" { print $3; exit }' \
/proc/self/cgroup 2>/dev/null)
[[ -n "${cgroup_path}" ]] || return 1
while IFS=$'\t' read -r mount_root mount_point; do
mount_root=$(decode_mountinfo_path "${mount_root}")
mount_point=$(decode_mountinfo_path "${mount_point}")
if [[ "${cgroup_path}" == "/" ]]; then
relative_path=""
elif [[ "${mount_root}" == "/" ]]; then
relative_path="${cgroup_path}"
elif [[ "${cgroup_path}" == "${mount_root}" ]]; then
relative_path=""
elif [[ "${cgroup_path}" == "${mount_root}/"* ]]; then
relative_path="/${cgroup_path#"${mount_root}/"}"
else
continue
fi
candidate="${mount_point%/}${relative_path}"
if [[ -d "${candidate}" ]]; then
printf '%s\n' "${candidate}"
return 0
fi
done < <(
awk '
{
for (i = 6; i <= NF; i++) {
if ($i == "-" && $(i + 1) == "cgroup2") {
print $4 "\t" $5
break
}
}
}
' /proc/self/mountinfo 2>/dev/null
)
return 1
}
collect_cgroup_files() {
local log_file=$1
local title=$2
local cgroup_dir=$3
shift 3
local cgroup_file=""
local files_collected=0
append_failure_diagnostic_section "${log_file}" "${title}"
if [[ -z "${cgroup_dir}" || ! -d "${cgroup_dir}" ]]; then
append_failure_diagnostic_note "${log_file}" \
"resolved_cgroup_path=unavailable"
return 0
fi
append_failure_diagnostic_note "${log_file}" \
"resolved_cgroup_path=${cgroup_dir}"
for cgroup_file in "$@"; do
if [[ -r "${cgroup_dir}/${cgroup_file}" ]]; then
append_failure_diagnostic_file "${log_file}" "${cgroup_file}" \
"${cgroup_dir}/${cgroup_file}"
files_collected=$((files_collected + 1))
fi
done
if [[ "${files_collected}" -eq 0 ]]; then
append_failure_diagnostic_note "${log_file}" \
"[no requested cgroup files were readable]"
fi
}
collect_cgroup_diagnostics() {
local log_file=$1
local cgroup_dir=""
append_failure_diagnostic_section "${log_file}" "Cgroup membership"
append_failure_diagnostic_file "${log_file}" "current_process" \
/proc/self/cgroup
append_failure_diagnostic_file "${log_file}" "pid_namespace_init_process" \
/proc/1/cgroup
cgroup_dir=$(resolve_current_cgroup_v2_dir 2>/dev/null || true)
collect_cgroup_files "${log_file}" \
"Cgroup v2 current-process resource state" "${cgroup_dir}" \
cgroup.type cgroup.events \
memory.current memory.peak memory.max memory.high memory.low memory.min \
memory.oom.group memory.swap.current memory.swap.max \
memory.events memory.events.local memory.swap.events memory.stat \
memory.pressure \
cpu.max cpu.max.burst cpu.weight cpu.stat cpu.pressure \
cpuset.cpus cpuset.cpus.effective cpuset.mems cpuset.mems.effective \
pids.current pids.max pids.events pids.events.local \
io.stat io.max io.weight io.pressure
if [[ -n "${cgroup_dir}" && -r "${cgroup_dir}/memory.events" ]]; then
amd_diagnostics_memory_events_path="${cgroup_dir}/memory.events"
fi
}
collect_process_diagnostics() {
local log_file=$1
local probe_deadline=$2
local shell_proc_dir="/proc/$$"
append_failure_diagnostic_section "${log_file}" \
"Runner process constraints and PID-namespace-visible processes"
append_failure_diagnostic_file "${log_file}" "diagnostic_shell_limits" \
"${shell_proc_dir}/limits"
if command -v awk >/dev/null 2>&1; then
# shellcheck disable=SC2016 # The expression is evaluated by awk.
run_failure_diagnostic "${log_file}" "${probe_deadline}" awk '
$1 ~ /^(Pid:|PPid:|Threads:|VmPeak:|VmSize:|VmHWM:|VmRSS:|RssAnon:|RssFile:|RssShmem:|Cpus_allowed_list:|Mems_allowed_list:|voluntary_ctxt_switches:|nonvoluntary_ctxt_switches:)$/ {
print
}
' "${shell_proc_dir}/status"
fi
if command -v nproc >/dev/null 2>&1; then
run_failure_diagnostic "${log_file}" "${probe_deadline}" nproc
fi
run_failure_diagnostic "${log_file}" "${probe_deadline}" \
/bin/bash -c 'ulimit -a'
if command -v ps >/dev/null 2>&1 && command -v head >/dev/null 2>&1; then
append_failure_diagnostic_note "${log_file}" \
"process_snapshot=top_100_by_rss"
run_failure_diagnostic "${log_file}" "${probe_deadline}" \
/bin/bash -c \
'ps -eo pid,ppid,stat,etimes,nlwp,pcpu,rss,vsz,comm --sort=-rss | head -n 101'
fi
return 0
}
collect_mount_diagnostic() {
local log_file=$1
local probe_deadline=$2
local label=$3
local path=$4
printf '\n%s (%s):\n' "${label}" "${path}" >> "${log_file}"
if [[ ! -e "${path}" ]]; then
echo "[path unavailable]" >> "${log_file}"
return 0
fi
if command -v findmnt >/dev/null 2>&1; then
run_failure_diagnostic "${log_file}" "${probe_deadline}" findmnt \
-n -T "${path}" -o TARGET,FSTYPE,VFS-OPTIONS
fi
if command -v df >/dev/null 2>&1; then
run_failure_diagnostic "${log_file}" "${probe_deadline}" df -h \
--output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent,target \
-- "${path}"
fi
return 0
}
collect_execution_resource_diagnostics() {
local log_file=$1
local probe_deadline=$2
# free and /proc/meminfo generally describe the node from a pod. Cgroups are
# the live source for this container's resource limits, usage, and events.
collect_cgroup_diagnostics "${log_file}"
collect_process_diagnostics "${log_file}" "${probe_deadline}"
append_failure_diagnostic_section "${log_file}" \
"Mount-visible filesystem capacity"
append_failure_diagnostic_note "${log_file}" \
"These values are filesystem views, not Kubernetes ephemeral-storage quotas."
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
"checkout" "${amd_diagnostics_checkout_root}"
if is_native_runtime; then
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
"native workspace emptyDir" "${amd_diagnostics_workspace}"
else
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
"test workspace (if mounted)" "${amd_diagnostics_workspace}"
fi
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
"temporary directory" "${TMPDIR:-/tmp}"
if [[ "${TMPDIR:-/tmp}" != "/tmp" ]]; then
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
"system temporary directory" /tmp
fi
if is_native_runtime; then
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
"shared-memory emptyDir" /dev/shm
else
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
"shared-memory mount" /dev/shm
fi
if [[ -n "${HF_HOME:-}" ]]; then
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
"Hugging Face cache" "${HF_HOME}"
fi
return 0
}
collect_amd_device_nodes() {
local log_file=$1
local probe_deadline=$2
local device=""
local device_count=0
for device in /dev/kfd /dev/dri/renderD*; do
if [[ ! -e "${device}" ]]; then
continue
fi
device_count=$((device_count + 1))
if command -v stat >/dev/null 2>&1; then
run_failure_diagnostic "${log_file}" "${probe_deadline}" stat -Lc \
'%n type=%F mode=%a owner=%u:%g device=%t:%T' -- "${device}"
else
append_failure_diagnostic_note "${log_file}" "device=${device}"
fi
done
if [[ "${device_count}" -eq 0 ]]; then
append_failure_diagnostic_note "${log_file}" \
"No /dev/kfd or /dev/dri/renderD* device nodes are visible."
fi
return 0
}
@@ -617,30 +901,55 @@ collect_rocm_failure_diagnostics() {
local parallel_job="${BUILDKITE_PARALLEL_JOB:-0}"
local diagnostics_relative_path=""
local diagnostics_path=""
local upload_root="${amd_diagnostics_checkout_root}"
local upload_path=""
local diagnostics_parent=""
local checkout_real=""
local diagnostics_parent_real=""
local exit_signal=""
local probe_deadline=0
local runtime="single-node-docker"
local diagnostics_scope="outer-runner-after-test-container-exit"
local identity_label="runner"
local identity_value="${HOSTNAME:-unknown}"
local k8s_pod="unknown"
local k8s_namespace="${amd_diagnostics_k8s_namespace}"
local oom_kill_count=""
local summary_identity_label="Runner"
local -a summary_rows=()
if [[ "${amd_diagnostics_collected}" == "1" ]]; then
return 0
fi
amd_diagnostics_collected=1
if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then
echo "Skipping AMD GPU diagnostics for a CPU-only job."
return 0
if is_native_runtime; then
runtime="native-kubernetes"
diagnostics_scope="current-container-cgroup-and-namespaces"
if [[ -n "${amd_diagnostics_pod_name}" ]]; then
identity_label="pod"
identity_value="${amd_diagnostics_pod_name}"
k8s_pod="${amd_diagnostics_pod_name}"
else
identity_label="container_hostname"
fi
elif [[ "${amd_diagnostics_execution_mode}" == "multi-node" ]]; then
runtime="multi-node-docker"
fi
if ! command -v timeout >/dev/null 2>&1; then
echo "WARNING: timeout is unavailable; skipping failure diagnostics rather than risking a hung job."
return 0
if [[ "${k8s_namespace}" == "unknown" \
&& -r /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then
k8s_namespace=$(
tr -d '\r\n' \
< /var/run/secrets/kubernetes.io/serviceaccount/namespace 2>/dev/null
)
fi
job_id="${job_id//[^A-Za-z0-9_.-]/_}"
retry_count="${retry_count//[^A-Za-z0-9_.-]/_}"
parallel_job="${parallel_job//[^A-Za-z0-9_.-]/_}"
if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then
exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true)
fi
# Buildkite keeps the supplied artifact path. Restrict the configurable
# directory to a checkout-relative path so the UI shows a clean artifact key.
if [[ -z "${amd_diagnostics_dir}" \
@@ -649,26 +958,32 @@ collect_rocm_failure_diagnostics() {
echo "WARNING: ignoring unsafe VLLM_CI_DIAGNOSTICS_DIR"
amd_diagnostics_dir="artifacts/amd-gpu-diagnostics"
fi
diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/amd-gpu-diagnostics.log"
diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/diagnostics.log"
diagnostics_path="${amd_diagnostics_checkout_root}/${diagnostics_relative_path}"
upload_path="${diagnostics_relative_path}"
if ! mkdir -p "$(dirname "${diagnostics_path}")" \
|| ! : > "${diagnostics_path}"; then
diagnostics_path=$(mktemp -t amd-gpu-diagnostics.XXXXXX.log) || {
echo "WARNING: unable to create an AMD GPU diagnostics log."
return 0
}
upload_root=$(dirname "${diagnostics_path}")
upload_path=$(basename "${diagnostics_path}")
fi
if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then
exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true)
diagnostics_parent=$(dirname "${diagnostics_path}")
checkout_real=$(readlink -m "${amd_diagnostics_checkout_root}" 2>/dev/null || true)
diagnostics_parent_real=$(readlink -m "${diagnostics_parent}" 2>/dev/null || true)
if [[ -z "${checkout_real}" || -z "${diagnostics_parent_real}" \
|| ("${diagnostics_parent_real}" != "${checkout_real}" \
&& "${diagnostics_parent_real}" != "${checkout_real}/"*) ]] \
|| ! mkdir -p "${diagnostics_parent}" \
|| [[ -L "${diagnostics_path}" ]] \
|| ! (set -o noclobber; : > "${diagnostics_path}") 2>/dev/null; then
echo "WARNING: unable to create AMD CI diagnostics at ${diagnostics_relative_path}."
printf '\nAMD CI failure summary\n'
printf '%-22s | %s\n' \
"Field" "Value" \
"----------------------" "-----" \
"Exit code" "${exit_code}" \
"Signal" "${exit_signal:-none}" \
"Runtime" "${runtime}" \
"Pod/container" "${identity_value}" \
"Diagnostics artifact" "unavailable: ${diagnostics_relative_path}"
return 0
fi
{
echo "AMD GPU failure diagnostics"
echo "AMD CI failure diagnostics"
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "exit_code=${exit_code}"
echo "exit_signal=${exit_signal:-none}"
@@ -680,105 +995,130 @@ collect_rocm_failure_diagnostics() {
echo "retry_count=${retry_count}"
echo "parallel_job=${parallel_job}"
echo "execution_mode=${amd_diagnostics_execution_mode}"
echo "runtime=${runtime}"
echo "diagnostics_scope=${diagnostics_scope}"
echo "expected_gpu_count=${amd_diagnostics_expected_gpu_count}"
echo "probe_budget_seconds=${amd_diagnostics_probe_budget_seconds}"
echo "command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}"
echo "upload_timeout_seconds=${amd_diagnostics_upload_timeout_seconds}"
echo "agent_name=${BUILDKITE_AGENT_NAME:-unknown}"
echo "container_hostname=${HOSTNAME:-unknown}"
echo "k8s_pod=${k8s_pod}"
echo "k8s_namespace=${k8s_namespace:-unknown}"
echo "k8s_node=${amd_diagnostics_k8s_node_name:-unknown}"
echo "kernel=$(uname -srmo 2>/dev/null || echo unknown)"
} > "${diagnostics_path}"
echo "--- :rotating_light: AMD GPU failure diagnostics"
cat "${diagnostics_path}"
probe_deadline=$((SECONDS + amd_diagnostics_probe_budget_seconds))
# Collect the most useful evidence first so auxiliary probes cannot consume
# the shared deadline before GPU health is captured.
echo "AMD GPU diagnostics" | tee -a "${diagnostics_path}"
if command -v amd-smi >/dev/null 2>&1; then
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi version
# Cgroup, process, and mount state is fast and most useful for explaining OOM,
# throttling, PID exhaustion, and emptyDir pressure. Capture it before external
# probes can consume the shared deadline.
collect_execution_resource_diagnostics \
"${diagnostics_path}" "${probe_deadline}"
append_failure_diagnostic_section "${diagnostics_path}" \
"AMD GPU diagnostics (tool-visible devices)"
if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then
append_failure_diagnostic_note "${diagnostics_path}" \
"CPU-only job; AMD GPU probes skipped."
else
if [[ "${runtime}" == "native-kubernetes" ]]; then
append_failure_diagnostic_note "${diagnostics_path}" \
"SMI visibility may be broader than the Kubernetes device allocation."
else
append_failure_diagnostic_note "${diagnostics_path}" \
"Outer-runner SMI visibility may be broader than the test container."
fi
collect_amd_device_nodes "${diagnostics_path}" "${probe_deadline}"
fi
if [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \
&& command -v amd-smi >/dev/null 2>&1; then
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi version
# Bus data identifies a card within the public node without publishing its
# persistent UUID, serial number, or process list.
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi static -b -g all
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -e -k -P -x -g all
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi bad-pages -p -r -u -g all
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -p -t -u -m -v -g all
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi xgmi -l -g all
elif command -v rocm-smi >/dev/null 2>&1; then
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi static -b -g all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi metric -e -k -P -x -g all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi bad-pages -p -r -u -g all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi metric -p -t -u -m -v -g all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi xgmi -l -g all
elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \
&& command -v rocm-smi >/dev/null 2>&1; then
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \
--showbus --showreplaycount --showrasinfo --showpagesinfo
elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]]; then
append_failure_diagnostic_note "${diagnostics_path}" \
"Neither amd-smi nor rocm-smi is available."
fi
if [[ "${runtime}" == "native-kubernetes" ]]; then
append_failure_diagnostic_section "${diagnostics_path}" \
"Unauthenticated pod network reachability"
else
echo "Neither amd-smi nor rocm-smi is available." \
| tee -a "${diagnostics_path}"
append_failure_diagnostic_section "${diagnostics_path}" \
"Unauthenticated outer-runner network reachability"
fi
echo "Node resource diagnostics (runner-visible scope)" \
| tee -a "${diagnostics_path}"
if command -v df >/dev/null 2>&1; then
echo "checkout_filesystem:" | tee -a "${diagnostics_path}"
if [[ -d "${amd_diagnostics_checkout_root}" ]]; then
(
cd "${amd_diagnostics_checkout_root}" || exit 0
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \
--output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \
-- .
)
else
echo "[checkout directory unavailable]" | tee -a "${diagnostics_path}"
fi
echo "tmp_filesystem:" | tee -a "${diagnostics_path}"
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \
--output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \
-- /tmp
fi
if command -v free >/dev/null 2>&1; then
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" free -h
fi
if [[ -r /sys/fs/cgroup/memory.events ]]; then
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" cat \
/sys/fs/cgroup/memory.events
fi
echo "Unauthenticated outer-runner network reachability diagnostics" \
| tee -a "${diagnostics_path}"
if command -v curl >/dev/null 2>&1; then
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
--silent --location --proto '=https' --proto-redir '=https' \
--max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \
--write-out 'target=huggingface http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \
https://huggingface.co/api/models/gpt2
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
--silent --location --proto '=https' --proto-redir '=https' \
--max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \
--write-out 'target=github_git_smart_http http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \
'https://github.com/vllm-project/ci-infra.git/info/refs?service=git-upload-pack'
else
echo "curl unavailable; reachability probes skipped." \
| tee -a "${diagnostics_path}"
append_failure_diagnostic_note "${diagnostics_path}" \
"curl unavailable; reachability probes skipped."
fi
if command -v buildkite-agent >/dev/null 2>&1 \
&& [[ -n "${BUILDKITE_JOB_ID:-}" ]]; then
if (
cd "${upload_root}" \
&& BUILDKITE_AGENT_DEBUG=false \
BUILDKITE_AGENT_DEBUG_HTTP=false \
BUILDKITE_AGENT_TRACE_HTTP=false \
BUILDKITE_AGENT_LOG_LEVEL=error \
timeout --kill-after=2s "${amd_diagnostics_upload_timeout_seconds}s" \
buildkite-agent artifact upload "${upload_path}" \
>/dev/null 2>&1
); then
echo "Uploaded AMD GPU diagnostics artifact: ${upload_path}"
else
echo "WARNING: failed to upload AMD GPU diagnostics artifact: ${upload_path}"
if [[ "${runtime}" == "native-kubernetes" \
&& -n "${amd_diagnostics_memory_events_path}" \
&& -r "${amd_diagnostics_memory_events_path}" ]]; then
oom_kill_count=$(
awk '$1 == "oom_kill" { print $2; exit }' \
"${amd_diagnostics_memory_events_path}" 2>/dev/null
)
if [[ ! "${oom_kill_count}" =~ ^[0-9]+$ ]]; then
oom_kill_count=""
fi
else
echo "Buildkite agent unavailable; diagnostics artifact was not uploaded."
fi
case "${identity_label}" in
pod)
summary_identity_label="Kubernetes pod"
;;
container_hostname)
summary_identity_label="Container hostname"
;;
esac
summary_rows=(
"Field" "Value"
"----------------------" "-----"
"Exit code" "${exit_code}"
"Signal" "${exit_signal:-none}"
"Runtime" "${runtime}"
"${summary_identity_label}" "${identity_value}"
)
if [[ -n "${amd_diagnostics_k8s_node_name}" ]]; then
summary_rows+=("Kubernetes node" "${amd_diagnostics_k8s_node_name}")
fi
if [[ -n "${oom_kill_count}" ]]; then
summary_rows+=("Cgroup OOM kills" "${oom_kill_count}")
fi
summary_rows+=("Diagnostics artifact" "${diagnostics_relative_path}")
printf '\nAMD CI failure summary\n'
printf '%-22s | %s\n' "${summary_rows[@]}"
return 0
}
@@ -788,9 +1128,6 @@ handle_pytest_exit() {
echo "Pytest exit code 5 (no tests collected) - treating as success."
exit 0
fi
if [[ "${exit_code}" -ne 0 ]]; then
collect_rocm_failure_diagnostics "${exit_code}"
fi
exit "$exit_code"
}
@@ -959,7 +1296,7 @@ handle_amd_runner_exit() {
}
# Catch both test failures and wrapper/setup failures. Runtime-specific cleanup
# traps below replace this trap but call the same handler before cleaning up.
# traps below replace this trap and call the same handler after cleanup.
trap handle_amd_runner_exit EXIT
###############################################################################
@@ -973,10 +1310,10 @@ if is_native_runtime; then
# shellcheck disable=SC2317 # Called indirectly by the EXIT trap.
cleanup_native_workspace() {
local exit_code=$?
handle_amd_runner_exit "${exit_code}"
if [[ -n "${artifact_work_dir}" ]]; then
rm -rf "${artifact_work_dir}"
fi
handle_amd_runner_exit "${exit_code}"
}
trap cleanup_native_workspace EXIT
@@ -1020,6 +1357,7 @@ if is_native_runtime; then
echo "Native test commands: $commands"
run_native_preflight || exit 1
echo "--- Test log"
# Keep AMD CI orchestration variables out of vLLM's runtime environment.
clear_ci_orchestration_env
/bin/bash -o pipefail -c "${commands}"
@@ -1042,7 +1380,6 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea
# shellcheck disable=SC2317 # Called indirectly by the EXIT trap.
remove_docker_container() {
local exit_code=$?
handle_amd_runner_exit "${exit_code}"
if docker container inspect "${container_name}" >/dev/null 2>&1; then
docker rm -f "${container_name}" || true
fi
@@ -1055,6 +1392,7 @@ remove_docker_container() {
if [[ -n "${artifact_work_dir}" ]]; then
rm -rf "${artifact_work_dir}"
fi
handle_amd_runner_exit "${exit_code}"
}
trap remove_docker_container EXIT