[CI][AMD] Improve Kubernetes failure diagnostics (#52264)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
455edc022b
commit
60c3a31b12
@@ -44,11 +44,14 @@ amd_diagnostics_dir="${VLLM_CI_DIAGNOSTICS_DIR:-artifacts/amd-gpu-diagnostics}"
|
||||
amd_diagnostics_checkout_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd -P)}"
|
||||
amd_diagnostics_execution_mode="${VLLM_CI_EXECUTION_MODE:-single-node}"
|
||||
amd_diagnostics_test_group="${VLLM_TEST_GROUP_NAME:-${BUILDKITE_STEP_KEY:-unknown}}"
|
||||
amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-}"
|
||||
amd_diagnostics_workspace="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
|
||||
amd_diagnostics_pod_name="${VLLM_CI_K8S_POD_NAME:-${POD_NAME:-${HOSTNAME:-}}}"
|
||||
amd_diagnostics_k8s_namespace="${VLLM_CI_K8S_NAMESPACE:-${POD_NAMESPACE:-unknown}}"
|
||||
amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-${NODE_NAME:-}}"
|
||||
amd_diagnostics_collected=0
|
||||
amd_diagnostics_memory_events_path=""
|
||||
amd_diagnostics_probe_budget_seconds=25
|
||||
amd_diagnostics_command_timeout_seconds=5
|
||||
amd_diagnostics_upload_timeout_seconds=20
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
|
||||
fi
|
||||
@@ -101,6 +104,8 @@ clear_ci_orchestration_env() {
|
||||
VLLM_CI_ARTIFACT_GLOB \
|
||||
VLLM_CI_ARTIFACT_CHECKSUM_GLOB \
|
||||
VLLM_CI_EXPECTED_GPU_COUNT \
|
||||
VLLM_CI_K8S_POD_NAME \
|
||||
VLLM_CI_K8S_NAMESPACE \
|
||||
VLLM_CI_K8S_NODE_NAME \
|
||||
VLLM_CI_USE_ARTIFACTS \
|
||||
VLLM_CI_RESULTS_ROOT \
|
||||
@@ -577,7 +582,24 @@ is_multi_node() {
|
||||
return 1
|
||||
}
|
||||
|
||||
run_amd_diagnostic() {
|
||||
append_failure_diagnostic_section() {
|
||||
local log_file=$1
|
||||
local title=$2
|
||||
|
||||
printf '\n===============================================================================\n' \
|
||||
>> "${log_file}"
|
||||
printf '%s\n' "${title}" >> "${log_file}"
|
||||
printf '===============================================================================\n' \
|
||||
>> "${log_file}"
|
||||
}
|
||||
|
||||
append_failure_diagnostic_note() {
|
||||
local log_file=$1
|
||||
shift
|
||||
printf '%s\n' "$@" >> "${log_file}"
|
||||
}
|
||||
|
||||
run_failure_diagnostic() {
|
||||
local log_file=$1
|
||||
local probe_deadline=$2
|
||||
shift 2
|
||||
@@ -589,23 +611,285 @@ run_amd_diagnostic() {
|
||||
printf '\n$'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
} | tee -a "${log_file}"
|
||||
} >> "${log_file}"
|
||||
|
||||
if ! command -v timeout >/dev/null 2>&1; then
|
||||
echo "[timeout unavailable; bounded diagnostic command skipped]" \
|
||||
>> "${log_file}"
|
||||
return 0
|
||||
fi
|
||||
if [[ "${remaining_seconds}" -le 0 ]]; then
|
||||
echo "[diagnostic probe budget exhausted; command skipped]" \
|
||||
| tee -a "${log_file}"
|
||||
>> "${log_file}"
|
||||
return 0
|
||||
fi
|
||||
if [[ "${remaining_seconds}" -lt "${command_timeout_seconds}" ]]; then
|
||||
command_timeout_seconds=${remaining_seconds}
|
||||
fi
|
||||
|
||||
timeout --kill-after=1s "${command_timeout_seconds}s" "$@" 2>&1 \
|
||||
| tee -a "${log_file}"
|
||||
command_status=${PIPESTATUS[0]}
|
||||
timeout --kill-after=1s "${command_timeout_seconds}s" "$@" \
|
||||
>> "${log_file}" 2>&1
|
||||
command_status=$?
|
||||
if [[ "${command_status}" -ne 0 ]]; then
|
||||
echo "[diagnostic command exited with status ${command_status}]" \
|
||||
| tee -a "${log_file}"
|
||||
>> "${log_file}"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
append_failure_diagnostic_file() {
|
||||
local log_file=$1
|
||||
local label=$2
|
||||
local path=$3
|
||||
local command_status=0
|
||||
|
||||
printf '\n%s:\n' "${label}" >> "${log_file}"
|
||||
printf '$ cat %q\n' "${path}" >> "${log_file}"
|
||||
if [[ ! -r "${path}" ]]; then
|
||||
echo "[file unavailable]" >> "${log_file}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
cat "${path}" >> "${log_file}" 2>&1
|
||||
command_status=$?
|
||||
if [[ "${command_status}" -ne 0 ]]; then
|
||||
echo "[diagnostic file read exited with status ${command_status}]" \
|
||||
>> "${log_file}"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
decode_mountinfo_path() {
|
||||
local value=$1
|
||||
|
||||
value=${value//\\040/ }
|
||||
value=${value//\\011/$'\t'}
|
||||
value=${value//\\012/$'\n'}
|
||||
value=${value//\\134/\\}
|
||||
printf '%s\n' "${value}"
|
||||
}
|
||||
|
||||
resolve_current_cgroup_v2_dir() {
|
||||
local cgroup_path=""
|
||||
local mount_root=""
|
||||
local mount_point=""
|
||||
local relative_path=""
|
||||
local candidate=""
|
||||
|
||||
cgroup_path=$(awk -F: '$2 == "" { print $3; exit }' \
|
||||
/proc/self/cgroup 2>/dev/null)
|
||||
[[ -n "${cgroup_path}" ]] || return 1
|
||||
|
||||
while IFS=$'\t' read -r mount_root mount_point; do
|
||||
mount_root=$(decode_mountinfo_path "${mount_root}")
|
||||
mount_point=$(decode_mountinfo_path "${mount_point}")
|
||||
if [[ "${cgroup_path}" == "/" ]]; then
|
||||
relative_path=""
|
||||
elif [[ "${mount_root}" == "/" ]]; then
|
||||
relative_path="${cgroup_path}"
|
||||
elif [[ "${cgroup_path}" == "${mount_root}" ]]; then
|
||||
relative_path=""
|
||||
elif [[ "${cgroup_path}" == "${mount_root}/"* ]]; then
|
||||
relative_path="/${cgroup_path#"${mount_root}/"}"
|
||||
else
|
||||
continue
|
||||
fi
|
||||
|
||||
candidate="${mount_point%/}${relative_path}"
|
||||
if [[ -d "${candidate}" ]]; then
|
||||
printf '%s\n' "${candidate}"
|
||||
return 0
|
||||
fi
|
||||
done < <(
|
||||
awk '
|
||||
{
|
||||
for (i = 6; i <= NF; i++) {
|
||||
if ($i == "-" && $(i + 1) == "cgroup2") {
|
||||
print $4 "\t" $5
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
' /proc/self/mountinfo 2>/dev/null
|
||||
)
|
||||
return 1
|
||||
}
|
||||
|
||||
collect_cgroup_files() {
|
||||
local log_file=$1
|
||||
local title=$2
|
||||
local cgroup_dir=$3
|
||||
shift 3
|
||||
local cgroup_file=""
|
||||
local files_collected=0
|
||||
|
||||
append_failure_diagnostic_section "${log_file}" "${title}"
|
||||
if [[ -z "${cgroup_dir}" || ! -d "${cgroup_dir}" ]]; then
|
||||
append_failure_diagnostic_note "${log_file}" \
|
||||
"resolved_cgroup_path=unavailable"
|
||||
return 0
|
||||
fi
|
||||
|
||||
append_failure_diagnostic_note "${log_file}" \
|
||||
"resolved_cgroup_path=${cgroup_dir}"
|
||||
for cgroup_file in "$@"; do
|
||||
if [[ -r "${cgroup_dir}/${cgroup_file}" ]]; then
|
||||
append_failure_diagnostic_file "${log_file}" "${cgroup_file}" \
|
||||
"${cgroup_dir}/${cgroup_file}"
|
||||
files_collected=$((files_collected + 1))
|
||||
fi
|
||||
done
|
||||
if [[ "${files_collected}" -eq 0 ]]; then
|
||||
append_failure_diagnostic_note "${log_file}" \
|
||||
"[no requested cgroup files were readable]"
|
||||
fi
|
||||
}
|
||||
|
||||
collect_cgroup_diagnostics() {
|
||||
local log_file=$1
|
||||
local cgroup_dir=""
|
||||
|
||||
append_failure_diagnostic_section "${log_file}" "Cgroup membership"
|
||||
append_failure_diagnostic_file "${log_file}" "current_process" \
|
||||
/proc/self/cgroup
|
||||
append_failure_diagnostic_file "${log_file}" "pid_namespace_init_process" \
|
||||
/proc/1/cgroup
|
||||
|
||||
cgroup_dir=$(resolve_current_cgroup_v2_dir 2>/dev/null || true)
|
||||
collect_cgroup_files "${log_file}" \
|
||||
"Cgroup v2 current-process resource state" "${cgroup_dir}" \
|
||||
cgroup.type cgroup.events \
|
||||
memory.current memory.peak memory.max memory.high memory.low memory.min \
|
||||
memory.oom.group memory.swap.current memory.swap.max \
|
||||
memory.events memory.events.local memory.swap.events memory.stat \
|
||||
memory.pressure \
|
||||
cpu.max cpu.max.burst cpu.weight cpu.stat cpu.pressure \
|
||||
cpuset.cpus cpuset.cpus.effective cpuset.mems cpuset.mems.effective \
|
||||
pids.current pids.max pids.events pids.events.local \
|
||||
io.stat io.max io.weight io.pressure
|
||||
if [[ -n "${cgroup_dir}" && -r "${cgroup_dir}/memory.events" ]]; then
|
||||
amd_diagnostics_memory_events_path="${cgroup_dir}/memory.events"
|
||||
fi
|
||||
}
|
||||
|
||||
collect_process_diagnostics() {
|
||||
local log_file=$1
|
||||
local probe_deadline=$2
|
||||
local shell_proc_dir="/proc/$$"
|
||||
|
||||
append_failure_diagnostic_section "${log_file}" \
|
||||
"Runner process constraints and PID-namespace-visible processes"
|
||||
append_failure_diagnostic_file "${log_file}" "diagnostic_shell_limits" \
|
||||
"${shell_proc_dir}/limits"
|
||||
if command -v awk >/dev/null 2>&1; then
|
||||
# shellcheck disable=SC2016 # The expression is evaluated by awk.
|
||||
run_failure_diagnostic "${log_file}" "${probe_deadline}" awk '
|
||||
$1 ~ /^(Pid:|PPid:|Threads:|VmPeak:|VmSize:|VmHWM:|VmRSS:|RssAnon:|RssFile:|RssShmem:|Cpus_allowed_list:|Mems_allowed_list:|voluntary_ctxt_switches:|nonvoluntary_ctxt_switches:)$/ {
|
||||
print
|
||||
}
|
||||
' "${shell_proc_dir}/status"
|
||||
fi
|
||||
if command -v nproc >/dev/null 2>&1; then
|
||||
run_failure_diagnostic "${log_file}" "${probe_deadline}" nproc
|
||||
fi
|
||||
run_failure_diagnostic "${log_file}" "${probe_deadline}" \
|
||||
/bin/bash -c 'ulimit -a'
|
||||
if command -v ps >/dev/null 2>&1 && command -v head >/dev/null 2>&1; then
|
||||
append_failure_diagnostic_note "${log_file}" \
|
||||
"process_snapshot=top_100_by_rss"
|
||||
run_failure_diagnostic "${log_file}" "${probe_deadline}" \
|
||||
/bin/bash -c \
|
||||
'ps -eo pid,ppid,stat,etimes,nlwp,pcpu,rss,vsz,comm --sort=-rss | head -n 101'
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
collect_mount_diagnostic() {
|
||||
local log_file=$1
|
||||
local probe_deadline=$2
|
||||
local label=$3
|
||||
local path=$4
|
||||
|
||||
printf '\n%s (%s):\n' "${label}" "${path}" >> "${log_file}"
|
||||
if [[ ! -e "${path}" ]]; then
|
||||
echo "[path unavailable]" >> "${log_file}"
|
||||
return 0
|
||||
fi
|
||||
if command -v findmnt >/dev/null 2>&1; then
|
||||
run_failure_diagnostic "${log_file}" "${probe_deadline}" findmnt \
|
||||
-n -T "${path}" -o TARGET,FSTYPE,VFS-OPTIONS
|
||||
fi
|
||||
if command -v df >/dev/null 2>&1; then
|
||||
run_failure_diagnostic "${log_file}" "${probe_deadline}" df -h \
|
||||
--output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent,target \
|
||||
-- "${path}"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
collect_execution_resource_diagnostics() {
|
||||
local log_file=$1
|
||||
local probe_deadline=$2
|
||||
|
||||
# free and /proc/meminfo generally describe the node from a pod. Cgroups are
|
||||
# the live source for this container's resource limits, usage, and events.
|
||||
collect_cgroup_diagnostics "${log_file}"
|
||||
collect_process_diagnostics "${log_file}" "${probe_deadline}"
|
||||
|
||||
append_failure_diagnostic_section "${log_file}" \
|
||||
"Mount-visible filesystem capacity"
|
||||
append_failure_diagnostic_note "${log_file}" \
|
||||
"These values are filesystem views, not Kubernetes ephemeral-storage quotas."
|
||||
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
|
||||
"checkout" "${amd_diagnostics_checkout_root}"
|
||||
if is_native_runtime; then
|
||||
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
|
||||
"native workspace emptyDir" "${amd_diagnostics_workspace}"
|
||||
else
|
||||
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
|
||||
"test workspace (if mounted)" "${amd_diagnostics_workspace}"
|
||||
fi
|
||||
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
|
||||
"temporary directory" "${TMPDIR:-/tmp}"
|
||||
if [[ "${TMPDIR:-/tmp}" != "/tmp" ]]; then
|
||||
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
|
||||
"system temporary directory" /tmp
|
||||
fi
|
||||
if is_native_runtime; then
|
||||
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
|
||||
"shared-memory emptyDir" /dev/shm
|
||||
else
|
||||
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
|
||||
"shared-memory mount" /dev/shm
|
||||
fi
|
||||
if [[ -n "${HF_HOME:-}" ]]; then
|
||||
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \
|
||||
"Hugging Face cache" "${HF_HOME}"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
collect_amd_device_nodes() {
|
||||
local log_file=$1
|
||||
local probe_deadline=$2
|
||||
local device=""
|
||||
local device_count=0
|
||||
|
||||
for device in /dev/kfd /dev/dri/renderD*; do
|
||||
if [[ ! -e "${device}" ]]; then
|
||||
continue
|
||||
fi
|
||||
device_count=$((device_count + 1))
|
||||
if command -v stat >/dev/null 2>&1; then
|
||||
run_failure_diagnostic "${log_file}" "${probe_deadline}" stat -Lc \
|
||||
'%n type=%F mode=%a owner=%u:%g device=%t:%T' -- "${device}"
|
||||
else
|
||||
append_failure_diagnostic_note "${log_file}" "device=${device}"
|
||||
fi
|
||||
done
|
||||
if [[ "${device_count}" -eq 0 ]]; then
|
||||
append_failure_diagnostic_note "${log_file}" \
|
||||
"No /dev/kfd or /dev/dri/renderD* device nodes are visible."
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
@@ -617,30 +901,55 @@ collect_rocm_failure_diagnostics() {
|
||||
local parallel_job="${BUILDKITE_PARALLEL_JOB:-0}"
|
||||
local diagnostics_relative_path=""
|
||||
local diagnostics_path=""
|
||||
local upload_root="${amd_diagnostics_checkout_root}"
|
||||
local upload_path=""
|
||||
local diagnostics_parent=""
|
||||
local checkout_real=""
|
||||
local diagnostics_parent_real=""
|
||||
local exit_signal=""
|
||||
local probe_deadline=0
|
||||
local runtime="single-node-docker"
|
||||
local diagnostics_scope="outer-runner-after-test-container-exit"
|
||||
local identity_label="runner"
|
||||
local identity_value="${HOSTNAME:-unknown}"
|
||||
local k8s_pod="unknown"
|
||||
local k8s_namespace="${amd_diagnostics_k8s_namespace}"
|
||||
local oom_kill_count=""
|
||||
local summary_identity_label="Runner"
|
||||
local -a summary_rows=()
|
||||
|
||||
if [[ "${amd_diagnostics_collected}" == "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
amd_diagnostics_collected=1
|
||||
|
||||
if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then
|
||||
echo "Skipping AMD GPU diagnostics for a CPU-only job."
|
||||
return 0
|
||||
if is_native_runtime; then
|
||||
runtime="native-kubernetes"
|
||||
diagnostics_scope="current-container-cgroup-and-namespaces"
|
||||
if [[ -n "${amd_diagnostics_pod_name}" ]]; then
|
||||
identity_label="pod"
|
||||
identity_value="${amd_diagnostics_pod_name}"
|
||||
k8s_pod="${amd_diagnostics_pod_name}"
|
||||
else
|
||||
identity_label="container_hostname"
|
||||
fi
|
||||
elif [[ "${amd_diagnostics_execution_mode}" == "multi-node" ]]; then
|
||||
runtime="multi-node-docker"
|
||||
fi
|
||||
|
||||
if ! command -v timeout >/dev/null 2>&1; then
|
||||
echo "WARNING: timeout is unavailable; skipping failure diagnostics rather than risking a hung job."
|
||||
return 0
|
||||
if [[ "${k8s_namespace}" == "unknown" \
|
||||
&& -r /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then
|
||||
k8s_namespace=$(
|
||||
tr -d '\r\n' \
|
||||
< /var/run/secrets/kubernetes.io/serviceaccount/namespace 2>/dev/null
|
||||
)
|
||||
fi
|
||||
|
||||
job_id="${job_id//[^A-Za-z0-9_.-]/_}"
|
||||
retry_count="${retry_count//[^A-Za-z0-9_.-]/_}"
|
||||
parallel_job="${parallel_job//[^A-Za-z0-9_.-]/_}"
|
||||
|
||||
if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then
|
||||
exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
# Buildkite keeps the supplied artifact path. Restrict the configurable
|
||||
# directory to a checkout-relative path so the UI shows a clean artifact key.
|
||||
if [[ -z "${amd_diagnostics_dir}" \
|
||||
@@ -649,26 +958,32 @@ collect_rocm_failure_diagnostics() {
|
||||
echo "WARNING: ignoring unsafe VLLM_CI_DIAGNOSTICS_DIR"
|
||||
amd_diagnostics_dir="artifacts/amd-gpu-diagnostics"
|
||||
fi
|
||||
diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/amd-gpu-diagnostics.log"
|
||||
diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/diagnostics.log"
|
||||
diagnostics_path="${amd_diagnostics_checkout_root}/${diagnostics_relative_path}"
|
||||
upload_path="${diagnostics_relative_path}"
|
||||
|
||||
if ! mkdir -p "$(dirname "${diagnostics_path}")" \
|
||||
|| ! : > "${diagnostics_path}"; then
|
||||
diagnostics_path=$(mktemp -t amd-gpu-diagnostics.XXXXXX.log) || {
|
||||
echo "WARNING: unable to create an AMD GPU diagnostics log."
|
||||
return 0
|
||||
}
|
||||
upload_root=$(dirname "${diagnostics_path}")
|
||||
upload_path=$(basename "${diagnostics_path}")
|
||||
fi
|
||||
|
||||
if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then
|
||||
exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true)
|
||||
diagnostics_parent=$(dirname "${diagnostics_path}")
|
||||
checkout_real=$(readlink -m "${amd_diagnostics_checkout_root}" 2>/dev/null || true)
|
||||
diagnostics_parent_real=$(readlink -m "${diagnostics_parent}" 2>/dev/null || true)
|
||||
if [[ -z "${checkout_real}" || -z "${diagnostics_parent_real}" \
|
||||
|| ("${diagnostics_parent_real}" != "${checkout_real}" \
|
||||
&& "${diagnostics_parent_real}" != "${checkout_real}/"*) ]] \
|
||||
|| ! mkdir -p "${diagnostics_parent}" \
|
||||
|| [[ -L "${diagnostics_path}" ]] \
|
||||
|| ! (set -o noclobber; : > "${diagnostics_path}") 2>/dev/null; then
|
||||
echo "WARNING: unable to create AMD CI diagnostics at ${diagnostics_relative_path}."
|
||||
printf '\nAMD CI failure summary\n'
|
||||
printf '%-22s | %s\n' \
|
||||
"Field" "Value" \
|
||||
"----------------------" "-----" \
|
||||
"Exit code" "${exit_code}" \
|
||||
"Signal" "${exit_signal:-none}" \
|
||||
"Runtime" "${runtime}" \
|
||||
"Pod/container" "${identity_value}" \
|
||||
"Diagnostics artifact" "unavailable: ${diagnostics_relative_path}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
{
|
||||
echo "AMD GPU failure diagnostics"
|
||||
echo "AMD CI failure diagnostics"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "exit_code=${exit_code}"
|
||||
echo "exit_signal=${exit_signal:-none}"
|
||||
@@ -680,105 +995,130 @@ collect_rocm_failure_diagnostics() {
|
||||
echo "retry_count=${retry_count}"
|
||||
echo "parallel_job=${parallel_job}"
|
||||
echo "execution_mode=${amd_diagnostics_execution_mode}"
|
||||
echo "runtime=${runtime}"
|
||||
echo "diagnostics_scope=${diagnostics_scope}"
|
||||
echo "expected_gpu_count=${amd_diagnostics_expected_gpu_count}"
|
||||
echo "probe_budget_seconds=${amd_diagnostics_probe_budget_seconds}"
|
||||
echo "command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}"
|
||||
echo "upload_timeout_seconds=${amd_diagnostics_upload_timeout_seconds}"
|
||||
echo "agent_name=${BUILDKITE_AGENT_NAME:-unknown}"
|
||||
echo "container_hostname=${HOSTNAME:-unknown}"
|
||||
echo "k8s_pod=${k8s_pod}"
|
||||
echo "k8s_namespace=${k8s_namespace:-unknown}"
|
||||
echo "k8s_node=${amd_diagnostics_k8s_node_name:-unknown}"
|
||||
echo "kernel=$(uname -srmo 2>/dev/null || echo unknown)"
|
||||
} > "${diagnostics_path}"
|
||||
|
||||
echo "--- :rotating_light: AMD GPU failure diagnostics"
|
||||
cat "${diagnostics_path}"
|
||||
|
||||
probe_deadline=$((SECONDS + amd_diagnostics_probe_budget_seconds))
|
||||
|
||||
# Collect the most useful evidence first so auxiliary probes cannot consume
|
||||
# the shared deadline before GPU health is captured.
|
||||
echo "AMD GPU diagnostics" | tee -a "${diagnostics_path}"
|
||||
if command -v amd-smi >/dev/null 2>&1; then
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi version
|
||||
# Cgroup, process, and mount state is fast and most useful for explaining OOM,
|
||||
# throttling, PID exhaustion, and emptyDir pressure. Capture it before external
|
||||
# probes can consume the shared deadline.
|
||||
collect_execution_resource_diagnostics \
|
||||
"${diagnostics_path}" "${probe_deadline}"
|
||||
|
||||
append_failure_diagnostic_section "${diagnostics_path}" \
|
||||
"AMD GPU diagnostics (tool-visible devices)"
|
||||
if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then
|
||||
append_failure_diagnostic_note "${diagnostics_path}" \
|
||||
"CPU-only job; AMD GPU probes skipped."
|
||||
else
|
||||
if [[ "${runtime}" == "native-kubernetes" ]]; then
|
||||
append_failure_diagnostic_note "${diagnostics_path}" \
|
||||
"SMI visibility may be broader than the Kubernetes device allocation."
|
||||
else
|
||||
append_failure_diagnostic_note "${diagnostics_path}" \
|
||||
"Outer-runner SMI visibility may be broader than the test container."
|
||||
fi
|
||||
collect_amd_device_nodes "${diagnostics_path}" "${probe_deadline}"
|
||||
fi
|
||||
if [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \
|
||||
&& command -v amd-smi >/dev/null 2>&1; then
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi version
|
||||
# Bus data identifies a card within the public node without publishing its
|
||||
# persistent UUID, serial number, or process list.
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi static -b -g all
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -e -k -P -x -g all
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi bad-pages -p -r -u -g all
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -p -t -u -m -v -g all
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi xgmi -l -g all
|
||||
elif command -v rocm-smi >/dev/null 2>&1; then
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi static -b -g all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi metric -e -k -P -x -g all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi bad-pages -p -r -u -g all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi metric -p -t -u -m -v -g all
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
|
||||
amd-smi xgmi -l -g all
|
||||
elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \
|
||||
&& command -v rocm-smi >/dev/null 2>&1; then
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \
|
||||
--showbus --showreplaycount --showrasinfo --showpagesinfo
|
||||
elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]]; then
|
||||
append_failure_diagnostic_note "${diagnostics_path}" \
|
||||
"Neither amd-smi nor rocm-smi is available."
|
||||
fi
|
||||
|
||||
if [[ "${runtime}" == "native-kubernetes" ]]; then
|
||||
append_failure_diagnostic_section "${diagnostics_path}" \
|
||||
"Unauthenticated pod network reachability"
|
||||
else
|
||||
echo "Neither amd-smi nor rocm-smi is available." \
|
||||
| tee -a "${diagnostics_path}"
|
||||
append_failure_diagnostic_section "${diagnostics_path}" \
|
||||
"Unauthenticated outer-runner network reachability"
|
||||
fi
|
||||
|
||||
echo "Node resource diagnostics (runner-visible scope)" \
|
||||
| tee -a "${diagnostics_path}"
|
||||
if command -v df >/dev/null 2>&1; then
|
||||
echo "checkout_filesystem:" | tee -a "${diagnostics_path}"
|
||||
if [[ -d "${amd_diagnostics_checkout_root}" ]]; then
|
||||
(
|
||||
cd "${amd_diagnostics_checkout_root}" || exit 0
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \
|
||||
--output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \
|
||||
-- .
|
||||
)
|
||||
else
|
||||
echo "[checkout directory unavailable]" | tee -a "${diagnostics_path}"
|
||||
fi
|
||||
echo "tmp_filesystem:" | tee -a "${diagnostics_path}"
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \
|
||||
--output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \
|
||||
-- /tmp
|
||||
fi
|
||||
if command -v free >/dev/null 2>&1; then
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" free -h
|
||||
fi
|
||||
if [[ -r /sys/fs/cgroup/memory.events ]]; then
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" cat \
|
||||
/sys/fs/cgroup/memory.events
|
||||
fi
|
||||
|
||||
echo "Unauthenticated outer-runner network reachability diagnostics" \
|
||||
| tee -a "${diagnostics_path}"
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
|
||||
--silent --location --proto '=https' --proto-redir '=https' \
|
||||
--max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \
|
||||
--write-out 'target=huggingface http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \
|
||||
https://huggingface.co/api/models/gpt2
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
|
||||
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
|
||||
--silent --location --proto '=https' --proto-redir '=https' \
|
||||
--max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \
|
||||
--write-out 'target=github_git_smart_http http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \
|
||||
'https://github.com/vllm-project/ci-infra.git/info/refs?service=git-upload-pack'
|
||||
else
|
||||
echo "curl unavailable; reachability probes skipped." \
|
||||
| tee -a "${diagnostics_path}"
|
||||
append_failure_diagnostic_note "${diagnostics_path}" \
|
||||
"curl unavailable; reachability probes skipped."
|
||||
fi
|
||||
|
||||
if command -v buildkite-agent >/dev/null 2>&1 \
|
||||
&& [[ -n "${BUILDKITE_JOB_ID:-}" ]]; then
|
||||
if (
|
||||
cd "${upload_root}" \
|
||||
&& BUILDKITE_AGENT_DEBUG=false \
|
||||
BUILDKITE_AGENT_DEBUG_HTTP=false \
|
||||
BUILDKITE_AGENT_TRACE_HTTP=false \
|
||||
BUILDKITE_AGENT_LOG_LEVEL=error \
|
||||
timeout --kill-after=2s "${amd_diagnostics_upload_timeout_seconds}s" \
|
||||
buildkite-agent artifact upload "${upload_path}" \
|
||||
>/dev/null 2>&1
|
||||
); then
|
||||
echo "Uploaded AMD GPU diagnostics artifact: ${upload_path}"
|
||||
else
|
||||
echo "WARNING: failed to upload AMD GPU diagnostics artifact: ${upload_path}"
|
||||
if [[ "${runtime}" == "native-kubernetes" \
|
||||
&& -n "${amd_diagnostics_memory_events_path}" \
|
||||
&& -r "${amd_diagnostics_memory_events_path}" ]]; then
|
||||
oom_kill_count=$(
|
||||
awk '$1 == "oom_kill" { print $2; exit }' \
|
||||
"${amd_diagnostics_memory_events_path}" 2>/dev/null
|
||||
)
|
||||
if [[ ! "${oom_kill_count}" =~ ^[0-9]+$ ]]; then
|
||||
oom_kill_count=""
|
||||
fi
|
||||
else
|
||||
echo "Buildkite agent unavailable; diagnostics artifact was not uploaded."
|
||||
fi
|
||||
|
||||
case "${identity_label}" in
|
||||
pod)
|
||||
summary_identity_label="Kubernetes pod"
|
||||
;;
|
||||
container_hostname)
|
||||
summary_identity_label="Container hostname"
|
||||
;;
|
||||
esac
|
||||
|
||||
summary_rows=(
|
||||
"Field" "Value"
|
||||
"----------------------" "-----"
|
||||
"Exit code" "${exit_code}"
|
||||
"Signal" "${exit_signal:-none}"
|
||||
"Runtime" "${runtime}"
|
||||
"${summary_identity_label}" "${identity_value}"
|
||||
)
|
||||
if [[ -n "${amd_diagnostics_k8s_node_name}" ]]; then
|
||||
summary_rows+=("Kubernetes node" "${amd_diagnostics_k8s_node_name}")
|
||||
fi
|
||||
if [[ -n "${oom_kill_count}" ]]; then
|
||||
summary_rows+=("Cgroup OOM kills" "${oom_kill_count}")
|
||||
fi
|
||||
summary_rows+=("Diagnostics artifact" "${diagnostics_relative_path}")
|
||||
|
||||
printf '\nAMD CI failure summary\n'
|
||||
printf '%-22s | %s\n' "${summary_rows[@]}"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -788,9 +1128,6 @@ handle_pytest_exit() {
|
||||
echo "Pytest exit code 5 (no tests collected) - treating as success."
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${exit_code}" -ne 0 ]]; then
|
||||
collect_rocm_failure_diagnostics "${exit_code}"
|
||||
fi
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
@@ -959,7 +1296,7 @@ handle_amd_runner_exit() {
|
||||
}
|
||||
|
||||
# Catch both test failures and wrapper/setup failures. Runtime-specific cleanup
|
||||
# traps below replace this trap but call the same handler before cleaning up.
|
||||
# traps below replace this trap and call the same handler after cleanup.
|
||||
trap handle_amd_runner_exit EXIT
|
||||
|
||||
###############################################################################
|
||||
@@ -973,10 +1310,10 @@ if is_native_runtime; then
|
||||
# shellcheck disable=SC2317 # Called indirectly by the EXIT trap.
|
||||
cleanup_native_workspace() {
|
||||
local exit_code=$?
|
||||
handle_amd_runner_exit "${exit_code}"
|
||||
if [[ -n "${artifact_work_dir}" ]]; then
|
||||
rm -rf "${artifact_work_dir}"
|
||||
fi
|
||||
handle_amd_runner_exit "${exit_code}"
|
||||
}
|
||||
trap cleanup_native_workspace EXIT
|
||||
|
||||
@@ -1020,6 +1357,7 @@ if is_native_runtime; then
|
||||
|
||||
echo "Native test commands: $commands"
|
||||
run_native_preflight || exit 1
|
||||
echo "--- Test log"
|
||||
# Keep AMD CI orchestration variables out of vLLM's runtime environment.
|
||||
clear_ci_orchestration_env
|
||||
/bin/bash -o pipefail -c "${commands}"
|
||||
@@ -1042,7 +1380,6 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea
|
||||
# shellcheck disable=SC2317 # Called indirectly by the EXIT trap.
|
||||
remove_docker_container() {
|
||||
local exit_code=$?
|
||||
handle_amd_runner_exit "${exit_code}"
|
||||
if docker container inspect "${container_name}" >/dev/null 2>&1; then
|
||||
docker rm -f "${container_name}" || true
|
||||
fi
|
||||
@@ -1055,6 +1392,7 @@ remove_docker_container() {
|
||||
if [[ -n "${artifact_work_dir}" ]]; then
|
||||
rm -rf "${artifact_work_dir}"
|
||||
fi
|
||||
handle_amd_runner_exit "${exit_code}"
|
||||
}
|
||||
trap remove_docker_container EXIT
|
||||
|
||||
|
||||
Reference in New Issue
Block a user