diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 86e4b34869..ff1fd0dd83 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -44,11 +44,14 @@ amd_diagnostics_dir="${VLLM_CI_DIAGNOSTICS_DIR:-artifacts/amd-gpu-diagnostics}" amd_diagnostics_checkout_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd -P)}" amd_diagnostics_execution_mode="${VLLM_CI_EXECUTION_MODE:-single-node}" amd_diagnostics_test_group="${VLLM_TEST_GROUP_NAME:-${BUILDKITE_STEP_KEY:-unknown}}" -amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-}" +amd_diagnostics_workspace="${VLLM_CI_WORKSPACE:-/vllm-workspace}" +amd_diagnostics_pod_name="${VLLM_CI_K8S_POD_NAME:-${POD_NAME:-${HOSTNAME:-}}}" +amd_diagnostics_k8s_namespace="${VLLM_CI_K8S_NAMESPACE:-${POD_NAMESPACE:-unknown}}" +amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-${NODE_NAME:-}}" amd_diagnostics_collected=0 +amd_diagnostics_memory_events_path="" amd_diagnostics_probe_budget_seconds=25 amd_diagnostics_command_timeout_seconds=5 -amd_diagnostics_upload_timeout_seconds=20 if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes" fi @@ -101,6 +104,8 @@ clear_ci_orchestration_env() { VLLM_CI_ARTIFACT_GLOB \ VLLM_CI_ARTIFACT_CHECKSUM_GLOB \ VLLM_CI_EXPECTED_GPU_COUNT \ + VLLM_CI_K8S_POD_NAME \ + VLLM_CI_K8S_NAMESPACE \ VLLM_CI_K8S_NODE_NAME \ VLLM_CI_USE_ARTIFACTS \ VLLM_CI_RESULTS_ROOT \ @@ -577,7 +582,24 @@ is_multi_node() { return 1 } -run_amd_diagnostic() { +append_failure_diagnostic_section() { + local log_file=$1 + local title=$2 + + printf '\n===============================================================================\n' \ + >> "${log_file}" + printf '%s\n' "${title}" >> "${log_file}" + printf '===============================================================================\n' \ + >> "${log_file}" +} + +append_failure_diagnostic_note() { + local log_file=$1 + shift + printf '%s\n' "$@" >> "${log_file}" +} + +run_failure_diagnostic() { local log_file=$1 local probe_deadline=$2 shift 2 @@ -589,23 +611,285 @@ run_amd_diagnostic() { printf '\n$' printf ' %q' "$@" printf '\n' - } | tee -a "${log_file}" + } >> "${log_file}" + if ! command -v timeout >/dev/null 2>&1; then + echo "[timeout unavailable; bounded diagnostic command skipped]" \ + >> "${log_file}" + return 0 + fi if [[ "${remaining_seconds}" -le 0 ]]; then echo "[diagnostic probe budget exhausted; command skipped]" \ - | tee -a "${log_file}" + >> "${log_file}" return 0 fi if [[ "${remaining_seconds}" -lt "${command_timeout_seconds}" ]]; then command_timeout_seconds=${remaining_seconds} fi - timeout --kill-after=1s "${command_timeout_seconds}s" "$@" 2>&1 \ - | tee -a "${log_file}" - command_status=${PIPESTATUS[0]} + timeout --kill-after=1s "${command_timeout_seconds}s" "$@" \ + >> "${log_file}" 2>&1 + command_status=$? if [[ "${command_status}" -ne 0 ]]; then echo "[diagnostic command exited with status ${command_status}]" \ - | tee -a "${log_file}" + >> "${log_file}" + fi + return 0 +} + +append_failure_diagnostic_file() { + local log_file=$1 + local label=$2 + local path=$3 + local command_status=0 + + printf '\n%s:\n' "${label}" >> "${log_file}" + printf '$ cat %q\n' "${path}" >> "${log_file}" + if [[ ! -r "${path}" ]]; then + echo "[file unavailable]" >> "${log_file}" + return 0 + fi + + cat "${path}" >> "${log_file}" 2>&1 + command_status=$? + if [[ "${command_status}" -ne 0 ]]; then + echo "[diagnostic file read exited with status ${command_status}]" \ + >> "${log_file}" + fi + return 0 +} + +decode_mountinfo_path() { + local value=$1 + + value=${value//\\040/ } + value=${value//\\011/$'\t'} + value=${value//\\012/$'\n'} + value=${value//\\134/\\} + printf '%s\n' "${value}" +} + +resolve_current_cgroup_v2_dir() { + local cgroup_path="" + local mount_root="" + local mount_point="" + local relative_path="" + local candidate="" + + cgroup_path=$(awk -F: '$2 == "" { print $3; exit }' \ + /proc/self/cgroup 2>/dev/null) + [[ -n "${cgroup_path}" ]] || return 1 + + while IFS=$'\t' read -r mount_root mount_point; do + mount_root=$(decode_mountinfo_path "${mount_root}") + mount_point=$(decode_mountinfo_path "${mount_point}") + if [[ "${cgroup_path}" == "/" ]]; then + relative_path="" + elif [[ "${mount_root}" == "/" ]]; then + relative_path="${cgroup_path}" + elif [[ "${cgroup_path}" == "${mount_root}" ]]; then + relative_path="" + elif [[ "${cgroup_path}" == "${mount_root}/"* ]]; then + relative_path="/${cgroup_path#"${mount_root}/"}" + else + continue + fi + + candidate="${mount_point%/}${relative_path}" + if [[ -d "${candidate}" ]]; then + printf '%s\n' "${candidate}" + return 0 + fi + done < <( + awk ' + { + for (i = 6; i <= NF; i++) { + if ($i == "-" && $(i + 1) == "cgroup2") { + print $4 "\t" $5 + break + } + } + } + ' /proc/self/mountinfo 2>/dev/null + ) + return 1 +} + +collect_cgroup_files() { + local log_file=$1 + local title=$2 + local cgroup_dir=$3 + shift 3 + local cgroup_file="" + local files_collected=0 + + append_failure_diagnostic_section "${log_file}" "${title}" + if [[ -z "${cgroup_dir}" || ! -d "${cgroup_dir}" ]]; then + append_failure_diagnostic_note "${log_file}" \ + "resolved_cgroup_path=unavailable" + return 0 + fi + + append_failure_diagnostic_note "${log_file}" \ + "resolved_cgroup_path=${cgroup_dir}" + for cgroup_file in "$@"; do + if [[ -r "${cgroup_dir}/${cgroup_file}" ]]; then + append_failure_diagnostic_file "${log_file}" "${cgroup_file}" \ + "${cgroup_dir}/${cgroup_file}" + files_collected=$((files_collected + 1)) + fi + done + if [[ "${files_collected}" -eq 0 ]]; then + append_failure_diagnostic_note "${log_file}" \ + "[no requested cgroup files were readable]" + fi +} + +collect_cgroup_diagnostics() { + local log_file=$1 + local cgroup_dir="" + + append_failure_diagnostic_section "${log_file}" "Cgroup membership" + append_failure_diagnostic_file "${log_file}" "current_process" \ + /proc/self/cgroup + append_failure_diagnostic_file "${log_file}" "pid_namespace_init_process" \ + /proc/1/cgroup + + cgroup_dir=$(resolve_current_cgroup_v2_dir 2>/dev/null || true) + collect_cgroup_files "${log_file}" \ + "Cgroup v2 current-process resource state" "${cgroup_dir}" \ + cgroup.type cgroup.events \ + memory.current memory.peak memory.max memory.high memory.low memory.min \ + memory.oom.group memory.swap.current memory.swap.max \ + memory.events memory.events.local memory.swap.events memory.stat \ + memory.pressure \ + cpu.max cpu.max.burst cpu.weight cpu.stat cpu.pressure \ + cpuset.cpus cpuset.cpus.effective cpuset.mems cpuset.mems.effective \ + pids.current pids.max pids.events pids.events.local \ + io.stat io.max io.weight io.pressure + if [[ -n "${cgroup_dir}" && -r "${cgroup_dir}/memory.events" ]]; then + amd_diagnostics_memory_events_path="${cgroup_dir}/memory.events" + fi +} + +collect_process_diagnostics() { + local log_file=$1 + local probe_deadline=$2 + local shell_proc_dir="/proc/$$" + + append_failure_diagnostic_section "${log_file}" \ + "Runner process constraints and PID-namespace-visible processes" + append_failure_diagnostic_file "${log_file}" "diagnostic_shell_limits" \ + "${shell_proc_dir}/limits" + if command -v awk >/dev/null 2>&1; then + # shellcheck disable=SC2016 # The expression is evaluated by awk. + run_failure_diagnostic "${log_file}" "${probe_deadline}" awk ' + $1 ~ /^(Pid:|PPid:|Threads:|VmPeak:|VmSize:|VmHWM:|VmRSS:|RssAnon:|RssFile:|RssShmem:|Cpus_allowed_list:|Mems_allowed_list:|voluntary_ctxt_switches:|nonvoluntary_ctxt_switches:)$/ { + print + } + ' "${shell_proc_dir}/status" + fi + if command -v nproc >/dev/null 2>&1; then + run_failure_diagnostic "${log_file}" "${probe_deadline}" nproc + fi + run_failure_diagnostic "${log_file}" "${probe_deadline}" \ + /bin/bash -c 'ulimit -a' + if command -v ps >/dev/null 2>&1 && command -v head >/dev/null 2>&1; then + append_failure_diagnostic_note "${log_file}" \ + "process_snapshot=top_100_by_rss" + run_failure_diagnostic "${log_file}" "${probe_deadline}" \ + /bin/bash -c \ + 'ps -eo pid,ppid,stat,etimes,nlwp,pcpu,rss,vsz,comm --sort=-rss | head -n 101' + fi + return 0 +} + +collect_mount_diagnostic() { + local log_file=$1 + local probe_deadline=$2 + local label=$3 + local path=$4 + + printf '\n%s (%s):\n' "${label}" "${path}" >> "${log_file}" + if [[ ! -e "${path}" ]]; then + echo "[path unavailable]" >> "${log_file}" + return 0 + fi + if command -v findmnt >/dev/null 2>&1; then + run_failure_diagnostic "${log_file}" "${probe_deadline}" findmnt \ + -n -T "${path}" -o TARGET,FSTYPE,VFS-OPTIONS + fi + if command -v df >/dev/null 2>&1; then + run_failure_diagnostic "${log_file}" "${probe_deadline}" df -h \ + --output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent,target \ + -- "${path}" + fi + return 0 +} + +collect_execution_resource_diagnostics() { + local log_file=$1 + local probe_deadline=$2 + + # free and /proc/meminfo generally describe the node from a pod. Cgroups are + # the live source for this container's resource limits, usage, and events. + collect_cgroup_diagnostics "${log_file}" + collect_process_diagnostics "${log_file}" "${probe_deadline}" + + append_failure_diagnostic_section "${log_file}" \ + "Mount-visible filesystem capacity" + append_failure_diagnostic_note "${log_file}" \ + "These values are filesystem views, not Kubernetes ephemeral-storage quotas." + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ + "checkout" "${amd_diagnostics_checkout_root}" + if is_native_runtime; then + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ + "native workspace emptyDir" "${amd_diagnostics_workspace}" + else + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ + "test workspace (if mounted)" "${amd_diagnostics_workspace}" + fi + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ + "temporary directory" "${TMPDIR:-/tmp}" + if [[ "${TMPDIR:-/tmp}" != "/tmp" ]]; then + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ + "system temporary directory" /tmp + fi + if is_native_runtime; then + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ + "shared-memory emptyDir" /dev/shm + else + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ + "shared-memory mount" /dev/shm + fi + if [[ -n "${HF_HOME:-}" ]]; then + collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ + "Hugging Face cache" "${HF_HOME}" + fi + return 0 +} + +collect_amd_device_nodes() { + local log_file=$1 + local probe_deadline=$2 + local device="" + local device_count=0 + + for device in /dev/kfd /dev/dri/renderD*; do + if [[ ! -e "${device}" ]]; then + continue + fi + device_count=$((device_count + 1)) + if command -v stat >/dev/null 2>&1; then + run_failure_diagnostic "${log_file}" "${probe_deadline}" stat -Lc \ + '%n type=%F mode=%a owner=%u:%g device=%t:%T' -- "${device}" + else + append_failure_diagnostic_note "${log_file}" "device=${device}" + fi + done + if [[ "${device_count}" -eq 0 ]]; then + append_failure_diagnostic_note "${log_file}" \ + "No /dev/kfd or /dev/dri/renderD* device nodes are visible." fi return 0 } @@ -617,30 +901,55 @@ collect_rocm_failure_diagnostics() { local parallel_job="${BUILDKITE_PARALLEL_JOB:-0}" local diagnostics_relative_path="" local diagnostics_path="" - local upload_root="${amd_diagnostics_checkout_root}" - local upload_path="" + local diagnostics_parent="" + local checkout_real="" + local diagnostics_parent_real="" local exit_signal="" local probe_deadline=0 + local runtime="single-node-docker" + local diagnostics_scope="outer-runner-after-test-container-exit" + local identity_label="runner" + local identity_value="${HOSTNAME:-unknown}" + local k8s_pod="unknown" + local k8s_namespace="${amd_diagnostics_k8s_namespace}" + local oom_kill_count="" + local summary_identity_label="Runner" + local -a summary_rows=() if [[ "${amd_diagnostics_collected}" == "1" ]]; then return 0 fi amd_diagnostics_collected=1 - if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then - echo "Skipping AMD GPU diagnostics for a CPU-only job." - return 0 + if is_native_runtime; then + runtime="native-kubernetes" + diagnostics_scope="current-container-cgroup-and-namespaces" + if [[ -n "${amd_diagnostics_pod_name}" ]]; then + identity_label="pod" + identity_value="${amd_diagnostics_pod_name}" + k8s_pod="${amd_diagnostics_pod_name}" + else + identity_label="container_hostname" + fi + elif [[ "${amd_diagnostics_execution_mode}" == "multi-node" ]]; then + runtime="multi-node-docker" fi - - if ! command -v timeout >/dev/null 2>&1; then - echo "WARNING: timeout is unavailable; skipping failure diagnostics rather than risking a hung job." - return 0 + if [[ "${k8s_namespace}" == "unknown" \ + && -r /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then + k8s_namespace=$( + tr -d '\r\n' \ + < /var/run/secrets/kubernetes.io/serviceaccount/namespace 2>/dev/null + ) fi job_id="${job_id//[^A-Za-z0-9_.-]/_}" retry_count="${retry_count//[^A-Za-z0-9_.-]/_}" parallel_job="${parallel_job//[^A-Za-z0-9_.-]/_}" + if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then + exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true) + fi + # Buildkite keeps the supplied artifact path. Restrict the configurable # directory to a checkout-relative path so the UI shows a clean artifact key. if [[ -z "${amd_diagnostics_dir}" \ @@ -649,26 +958,32 @@ collect_rocm_failure_diagnostics() { echo "WARNING: ignoring unsafe VLLM_CI_DIAGNOSTICS_DIR" amd_diagnostics_dir="artifacts/amd-gpu-diagnostics" fi - diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/amd-gpu-diagnostics.log" + diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/diagnostics.log" diagnostics_path="${amd_diagnostics_checkout_root}/${diagnostics_relative_path}" - upload_path="${diagnostics_relative_path}" - - if ! mkdir -p "$(dirname "${diagnostics_path}")" \ - || ! : > "${diagnostics_path}"; then - diagnostics_path=$(mktemp -t amd-gpu-diagnostics.XXXXXX.log) || { - echo "WARNING: unable to create an AMD GPU diagnostics log." - return 0 - } - upload_root=$(dirname "${diagnostics_path}") - upload_path=$(basename "${diagnostics_path}") - fi - - if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then - exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true) + diagnostics_parent=$(dirname "${diagnostics_path}") + checkout_real=$(readlink -m "${amd_diagnostics_checkout_root}" 2>/dev/null || true) + diagnostics_parent_real=$(readlink -m "${diagnostics_parent}" 2>/dev/null || true) + if [[ -z "${checkout_real}" || -z "${diagnostics_parent_real}" \ + || ("${diagnostics_parent_real}" != "${checkout_real}" \ + && "${diagnostics_parent_real}" != "${checkout_real}/"*) ]] \ + || ! mkdir -p "${diagnostics_parent}" \ + || [[ -L "${diagnostics_path}" ]] \ + || ! (set -o noclobber; : > "${diagnostics_path}") 2>/dev/null; then + echo "WARNING: unable to create AMD CI diagnostics at ${diagnostics_relative_path}." + printf '\nAMD CI failure summary\n' + printf '%-22s | %s\n' \ + "Field" "Value" \ + "----------------------" "-----" \ + "Exit code" "${exit_code}" \ + "Signal" "${exit_signal:-none}" \ + "Runtime" "${runtime}" \ + "Pod/container" "${identity_value}" \ + "Diagnostics artifact" "unavailable: ${diagnostics_relative_path}" + return 0 fi { - echo "AMD GPU failure diagnostics" + echo "AMD CI failure diagnostics" echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "exit_code=${exit_code}" echo "exit_signal=${exit_signal:-none}" @@ -680,105 +995,130 @@ collect_rocm_failure_diagnostics() { echo "retry_count=${retry_count}" echo "parallel_job=${parallel_job}" echo "execution_mode=${amd_diagnostics_execution_mode}" + echo "runtime=${runtime}" + echo "diagnostics_scope=${diagnostics_scope}" echo "expected_gpu_count=${amd_diagnostics_expected_gpu_count}" echo "probe_budget_seconds=${amd_diagnostics_probe_budget_seconds}" echo "command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}" - echo "upload_timeout_seconds=${amd_diagnostics_upload_timeout_seconds}" echo "agent_name=${BUILDKITE_AGENT_NAME:-unknown}" + echo "container_hostname=${HOSTNAME:-unknown}" + echo "k8s_pod=${k8s_pod}" + echo "k8s_namespace=${k8s_namespace:-unknown}" echo "k8s_node=${amd_diagnostics_k8s_node_name:-unknown}" echo "kernel=$(uname -srmo 2>/dev/null || echo unknown)" } > "${diagnostics_path}" - echo "--- :rotating_light: AMD GPU failure diagnostics" - cat "${diagnostics_path}" - probe_deadline=$((SECONDS + amd_diagnostics_probe_budget_seconds)) - # Collect the most useful evidence first so auxiliary probes cannot consume - # the shared deadline before GPU health is captured. - echo "AMD GPU diagnostics" | tee -a "${diagnostics_path}" - if command -v amd-smi >/dev/null 2>&1; then - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi version + # Cgroup, process, and mount state is fast and most useful for explaining OOM, + # throttling, PID exhaustion, and emptyDir pressure. Capture it before external + # probes can consume the shared deadline. + collect_execution_resource_diagnostics \ + "${diagnostics_path}" "${probe_deadline}" + + append_failure_diagnostic_section "${diagnostics_path}" \ + "AMD GPU diagnostics (tool-visible devices)" + if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then + append_failure_diagnostic_note "${diagnostics_path}" \ + "CPU-only job; AMD GPU probes skipped." + else + if [[ "${runtime}" == "native-kubernetes" ]]; then + append_failure_diagnostic_note "${diagnostics_path}" \ + "SMI visibility may be broader than the Kubernetes device allocation." + else + append_failure_diagnostic_note "${diagnostics_path}" \ + "Outer-runner SMI visibility may be broader than the test container." + fi + collect_amd_device_nodes "${diagnostics_path}" "${probe_deadline}" + fi + if [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \ + && command -v amd-smi >/dev/null 2>&1; then + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ + amd-smi version # Bus data identifies a card within the public node without publishing its # persistent UUID, serial number, or process list. - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi static -b -g all - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -e -k -P -x -g all - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi bad-pages -p -r -u -g all - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -p -t -u -m -v -g all - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi xgmi -l -g all - elif command -v rocm-smi >/dev/null 2>&1; then - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \ + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ + amd-smi static -b -g all + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ + amd-smi metric -e -k -P -x -g all + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ + amd-smi bad-pages -p -r -u -g all + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ + amd-smi metric -p -t -u -m -v -g all + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ + amd-smi xgmi -l -g all + elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \ + && command -v rocm-smi >/dev/null 2>&1; then + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \ --showbus --showreplaycount --showrasinfo --showpagesinfo + elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]]; then + append_failure_diagnostic_note "${diagnostics_path}" \ + "Neither amd-smi nor rocm-smi is available." + fi + + if [[ "${runtime}" == "native-kubernetes" ]]; then + append_failure_diagnostic_section "${diagnostics_path}" \ + "Unauthenticated pod network reachability" else - echo "Neither amd-smi nor rocm-smi is available." \ - | tee -a "${diagnostics_path}" + append_failure_diagnostic_section "${diagnostics_path}" \ + "Unauthenticated outer-runner network reachability" fi - - echo "Node resource diagnostics (runner-visible scope)" \ - | tee -a "${diagnostics_path}" - if command -v df >/dev/null 2>&1; then - echo "checkout_filesystem:" | tee -a "${diagnostics_path}" - if [[ -d "${amd_diagnostics_checkout_root}" ]]; then - ( - cd "${amd_diagnostics_checkout_root}" || exit 0 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \ - --output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \ - -- . - ) - else - echo "[checkout directory unavailable]" | tee -a "${diagnostics_path}" - fi - echo "tmp_filesystem:" | tee -a "${diagnostics_path}" - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \ - --output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \ - -- /tmp - fi - if command -v free >/dev/null 2>&1; then - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" free -h - fi - if [[ -r /sys/fs/cgroup/memory.events ]]; then - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" cat \ - /sys/fs/cgroup/memory.events - fi - - echo "Unauthenticated outer-runner network reachability diagnostics" \ - | tee -a "${diagnostics_path}" if command -v curl >/dev/null 2>&1; then - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \ + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \ --silent --location --proto '=https' --proto-redir '=https' \ --max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \ --write-out 'target=huggingface http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \ https://huggingface.co/api/models/gpt2 - run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \ + run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \ --silent --location --proto '=https' --proto-redir '=https' \ --max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \ --write-out 'target=github_git_smart_http http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \ 'https://github.com/vllm-project/ci-infra.git/info/refs?service=git-upload-pack' else - echo "curl unavailable; reachability probes skipped." \ - | tee -a "${diagnostics_path}" + append_failure_diagnostic_note "${diagnostics_path}" \ + "curl unavailable; reachability probes skipped." fi - if command -v buildkite-agent >/dev/null 2>&1 \ - && [[ -n "${BUILDKITE_JOB_ID:-}" ]]; then - if ( - cd "${upload_root}" \ - && BUILDKITE_AGENT_DEBUG=false \ - BUILDKITE_AGENT_DEBUG_HTTP=false \ - BUILDKITE_AGENT_TRACE_HTTP=false \ - BUILDKITE_AGENT_LOG_LEVEL=error \ - timeout --kill-after=2s "${amd_diagnostics_upload_timeout_seconds}s" \ - buildkite-agent artifact upload "${upload_path}" \ - >/dev/null 2>&1 - ); then - echo "Uploaded AMD GPU diagnostics artifact: ${upload_path}" - else - echo "WARNING: failed to upload AMD GPU diagnostics artifact: ${upload_path}" + if [[ "${runtime}" == "native-kubernetes" \ + && -n "${amd_diagnostics_memory_events_path}" \ + && -r "${amd_diagnostics_memory_events_path}" ]]; then + oom_kill_count=$( + awk '$1 == "oom_kill" { print $2; exit }' \ + "${amd_diagnostics_memory_events_path}" 2>/dev/null + ) + if [[ ! "${oom_kill_count}" =~ ^[0-9]+$ ]]; then + oom_kill_count="" fi - else - echo "Buildkite agent unavailable; diagnostics artifact was not uploaded." fi + case "${identity_label}" in + pod) + summary_identity_label="Kubernetes pod" + ;; + container_hostname) + summary_identity_label="Container hostname" + ;; + esac + + summary_rows=( + "Field" "Value" + "----------------------" "-----" + "Exit code" "${exit_code}" + "Signal" "${exit_signal:-none}" + "Runtime" "${runtime}" + "${summary_identity_label}" "${identity_value}" + ) + if [[ -n "${amd_diagnostics_k8s_node_name}" ]]; then + summary_rows+=("Kubernetes node" "${amd_diagnostics_k8s_node_name}") + fi + if [[ -n "${oom_kill_count}" ]]; then + summary_rows+=("Cgroup OOM kills" "${oom_kill_count}") + fi + summary_rows+=("Diagnostics artifact" "${diagnostics_relative_path}") + + printf '\nAMD CI failure summary\n' + printf '%-22s | %s\n' "${summary_rows[@]}" + return 0 } @@ -788,9 +1128,6 @@ handle_pytest_exit() { echo "Pytest exit code 5 (no tests collected) - treating as success." exit 0 fi - if [[ "${exit_code}" -ne 0 ]]; then - collect_rocm_failure_diagnostics "${exit_code}" - fi exit "$exit_code" } @@ -959,7 +1296,7 @@ handle_amd_runner_exit() { } # Catch both test failures and wrapper/setup failures. Runtime-specific cleanup -# traps below replace this trap but call the same handler before cleaning up. +# traps below replace this trap and call the same handler after cleanup. trap handle_amd_runner_exit EXIT ############################################################################### @@ -973,10 +1310,10 @@ if is_native_runtime; then # shellcheck disable=SC2317 # Called indirectly by the EXIT trap. cleanup_native_workspace() { local exit_code=$? - handle_amd_runner_exit "${exit_code}" if [[ -n "${artifact_work_dir}" ]]; then rm -rf "${artifact_work_dir}" fi + handle_amd_runner_exit "${exit_code}" } trap cleanup_native_workspace EXIT @@ -1020,6 +1357,7 @@ if is_native_runtime; then echo "Native test commands: $commands" run_native_preflight || exit 1 + echo "--- Test log" # Keep AMD CI orchestration variables out of vLLM's runtime environment. clear_ci_orchestration_env /bin/bash -o pipefail -c "${commands}" @@ -1042,7 +1380,6 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea # shellcheck disable=SC2317 # Called indirectly by the EXIT trap. remove_docker_container() { local exit_code=$? - handle_amd_runner_exit "${exit_code}" if docker container inspect "${container_name}" >/dev/null 2>&1; then docker rm -f "${container_name}" || true fi @@ -1055,6 +1392,7 @@ remove_docker_container() { if [[ -n "${artifact_work_dir}" ]]; then rm -rf "${artifact_work_dir}" fi + handle_amd_runner_exit "${exit_code}" } trap remove_docker_container EXIT