[CI] ci-fetch-log.sh: fetch all failed jobs from a build URL or PR number (#45274)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-12 00:42:18 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent 04cec9e4d8
commit bd59c913bc
4 changed files with 176 additions and 52 deletions
+3
View File
@@ -13,5 +13,8 @@ INPUT_FILE="$1"
# Strip timestamps
sed -i 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE"
# Strip Buildkite inline timestamp markers (ESC _bk;t=<ms> BEL)
sed -i 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE"
# Strip colorization
sed -i -r 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE"
+151 -47
View File
@@ -1,74 +1,178 @@
#!/bin/bash
# Usage: ./ci-fetch-log.sh <buildkite_job_url> [output_file]
# ./ci-fetch-log.sh <build_number> <job_uuid> [output_file]
# Fetch vLLM Buildkite CI logs (public; no login required).
#
# Downloads the raw log for a Buildkite job from the public, unauthenticated
# /organizations/<org>/pipelines/<pipeline>/builds/<n>/jobs/<uuid>/download
# endpoint, then strips ANSI/timestamps via ci-clean-log.sh.
# Usage:
# ci-fetch-log.sh [--soft|--all] --pr [<PR>] failed jobs in the PR's latest
# build (current branch if omitted)
# ci-fetch-log.sh [--soft|--all] <build_url> failed jobs in that build
# ci-fetch-log.sh <job_url> [output] one job; both #<job_uuid> and
# ?sid=<id> URL forms work
# ci-fetch-log.sh <build> <job_uuid> [output]
#
# Find <build_number> and <job_uuid> via:
# gh pr checks <PR> --repo vllm-project/vllm
# Each failing row's URL is .../builds/<build_number>#<job_uuid>.
#
# Default output path: ci-<build>-<uuid_first_13_chars>.log (e.g.
# ci-68478-019e6b07-daae.log). Jobs in the same build share the UUID's
# first 8 chars, so the second segment is needed for uniqueness when
# fetching multiple jobs in parallel. The script refuses to overwrite an
# existing output file; pass an explicit path or set CI_FETCH_LOG_FORCE=1
# to override.
# --soft also fetches soft-failed jobs; --all fetches every finished job.
# Saves each log as ci-<build>-<job-name>.log (ANSI/timestamps stripped) and
# prints "<file>\t<job name>" per job. [output] is single-job only; "-"
# streams to stdout. Existing files are kept; CI_FETCH_LOG_FORCE=1 refetches.
set -euo pipefail
ORG="vllm"
PIPELINE="ci"
UA="vllm-ci-fetch-log"
UUID_RE='[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
usage() {
echo "Usage: $0 <buildkite_job_url> [output_file]"
echo " $0 <build_number> <job_uuid> [output_file]"
sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'
exit 1
}
if [ $# -lt 1 ]; then usage; fi
die() {
echo "$1" >&2
exit 1
}
if [[ "$1" == https://* ]]; then
BUILD="" JOB="" SID="" OUT=""
SCOPE="failed"
while :; do
case "${1:-}" in
--soft) SCOPE="soft" ;;
--all) SCOPE="all" ;;
*) break ;;
esac
shift
done
case "${1:-}" in
--pr)
PR="${2:-}"
# gh pr checks exits non-zero when checks are failing; that is the
# expected case here.
URL=$(gh pr checks ${PR:+"$PR"} --repo vllm-project/vllm 2>/dev/null |
grep -oE "https://buildkite.com/${ORG}/${PIPELINE}/builds/[0-9]+" |
sort -t/ -k7 -n | tail -1 || true)
[ -n "$URL" ] || die "No Buildkite build found via: gh pr checks ${PR:-<current branch>}"
BUILD="${URL##*/}"
;;
https://*)
BUILD=$(echo "$1" | sed -nE 's#.*/builds/([0-9]+).*#\1#p')
JOB=$(echo "$1" | grep -oE '[0-9a-f]{8}-[0-9a-f-]+' | head -n 1)
JOB=$(echo "$1" | grep -oE "#${UUID_RE}" | head -n 1 | cut -c2- || true)
SID=$(echo "$1" | grep -oE "[?&]sid=${UUID_RE}" | head -n 1 | sed 's/.*sid=//' || true)
OUT="${2:-}"
else
if [ $# -lt 2 ]; then usage; fi
[ -n "$BUILD" ] || die "Could not parse build number from: $1"
;;
[0-9]*)
[ $# -ge 2 ] || usage
BUILD="$1"
JOB="$2"
OUT="${3:-}"
fi
if [ -z "$BUILD" ] || [ -z "$JOB" ]; then
echo "Could not parse build number or job UUID from: $1" >&2
;;
*)
usage
fi
# Jobs in the same build share the UUID's first segment, so include the
# second segment (chars 9-13, e.g. "019e6b07-daae") to keep default filenames
# unique when fetching multiple jobs from one build in parallel.
if [ -z "$OUT" ]; then
OUT="ci-${BUILD}-${JOB:0:13}.log"
fi
if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
echo "Refusing to overwrite existing $OUT (set CI_FETCH_LOG_FORCE=1 or pass an explicit output path)." >&2
exit 1
fi
;;
esac
COOKIES=$(mktemp)
trap 'rm -f "$COOKIES"' EXIT
JOBS_TSV=$(mktemp)
trap 'rm -f "$COOKIES" "$JOBS_TSV"' EXIT
# Buildkite issues a session cookie on first hit; subsequent /download needs it.
curl -fsSL -c "$COOKIES" -A "vllm-ci-fetch-log" \
# Buildkite issues a session cookie on first hit; later requests need it.
curl -fsSL -c "$COOKIES" -A "$UA" \
"https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}" -o /dev/null
curl -fsSL -b "$COOKIES" -A "vllm-ci-fetch-log" \
"https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/${JOB}/download" \
-o "$OUT"
# The build's job list (id, step uuid, state, name) is served as JSON from
# the user-facing /data/jobs endpoint. Flatten it to TSV for easy filtering:
# job_id step_uuid failed soft_failed finished slug name
curl -fsSL -b "$COOKIES" -A "$UA" \
"https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}/data/jobs" |
python3 -c '
import json, re, sys
bash "$(dirname "$0")/ci-clean-log.sh" "$OUT"
data = json.load(sys.stdin)
if data.get("has_next_page"):
print("warning: job list is paginated; some jobs not shown", file=sys.stderr)
for r in data["records"]:
if r.get("type") != "script":
continue
name = (r.get("name") or "").replace("\t", " ").replace("\n", " ")
slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")[:60]
print("\t".join([
r["id"],
r.get("step_uuid") or "",
str(r.get("passed") is False),
str(bool(r.get("soft_failed"))),
str(bool(r.get("finished_at"))),
slug,
name,
]))
' >"$JOBS_TSV" || die "Could not list jobs for build ${BUILD}"
echo "$OUT"
if [ -n "$SID" ] && [ -z "$JOB" ]; then
# The ?sid= in builds/<N>/list URLs is the *step* uuid, not the job uuid.
JOB=$(awk -F'\t' -v s="$SID" '$1 == s || $2 == s {print $1; exit}' "$JOBS_TSV")
[ -n "$JOB" ] || die "No job matching sid=${SID} in build ${BUILD}"
fi
fetch_job() { # <job_uuid> <output_file>
curl -fsSL -b "$COOKIES" -A "$UA" \
"https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/$1/download" \
-o "$2"
bash "$(dirname "$0")/ci-clean-log.sh" "$2"
}
if [ -n "$JOB" ]; then
# Single-job mode.
NAME=$(awk -F'\t' -v j="$JOB" '$1 == j {print $7; exit}' "$JOBS_TSV")
SLUG=$(awk -F'\t' -v j="$JOB" '$1 == j {print $6; exit}' "$JOBS_TSV")
[ -n "$OUT" ] || OUT="ci-${BUILD}-${SLUG:-${JOB:0:13}}.log"
if [ "$OUT" = "-" ]; then
TMP=$(mktemp)
fetch_job "$JOB" "$TMP"
cat "$TMP"
rm -f "$TMP"
exit 0
fi
if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
die "Refusing to overwrite existing ${OUT} (set CI_FETCH_LOG_FORCE=1 or pass an output path)."
fi
fetch_job "$JOB" "$OUT"
printf '%s\t%s\n' "$OUT" "${NAME:-$JOB}"
exit 0
fi
# Build-wide mode: fetch finished jobs matching $SCOPE.
[ -z "$OUT" ] || die "[output_file] is only valid when fetching a single job."
case "$SCOPE" in
failed) FILTER='$3 == "True" && $4 == "False" && $5 == "True"' ;;
soft) FILTER='$3 == "True" && $5 == "True"' ;;
all) FILTER='$5 == "True"' ;;
esac
if [ "$SCOPE" = "failed" ]; then
SOFT=$(awk -F'\t' '$3 == "True" && $4 == "True"' "$JOBS_TSV" | wc -l)
[ "$SOFT" -eq 0 ] || echo "Skipping ${SOFT} soft-failed job(s); use --soft to include them." >&2
fi
FOUND=0
EMITTED=" "
while IFS=$'\t' read -r job_id _ _ _ _ slug name; do
FOUND=$((FOUND + 1))
out="ci-${BUILD}-${slug:-${job_id:0:13}}.log"
# Retries share a name with the original job; disambiguate by uuid.
case "$EMITTED" in
*" $out "*) out="ci-${BUILD}-${slug:-job}-${job_id:0:13}.log" ;;
esac
EMITTED="${EMITTED}${out} "
if [ -e "$out" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
echo "Keeping existing ${out} (set CI_FETCH_LOG_FORCE=1 to refetch)." >&2
elif ! fetch_job "$job_id" "$out"; then
echo "Failed to download log for job ${job_id} (${name})." >&2
continue
fi
printf '%s\t%s\n' "$out" "$name"
done < <(awk -F'\t' "$FILTER" "$JOBS_TSV")
if [ "$FOUND" -eq 0 ]; then
echo "No matching jobs in build ${BUILD} (scope: ${SCOPE})." >&2
fi
+11
View File
@@ -105,6 +105,17 @@ The line length limit for Python code is 88 characters. If you are not sure, use
Use [Google-style docstrings](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) (`Args:`/`Returns:`/`Raises:` sections), not reStructuredText/Sphinx fields (`:param:`, `:return:`, `:rtype:`).
### Diagnosing CI failures
Buildkite logs are public; no login needed. Details: [docs/contributing/ci/failures.md](docs/contributing/ci/failures.md).
```bash
# All failed-job logs for a PR's latest build (current branch's PR if omitted):
.buildkite/scripts/ci-fetch-log.sh --pr <PR>
# Any Buildkite build or job URL also works:
.buildkite/scripts/ci-fetch-log.sh "<buildkite_url>"
```
### Commit messages
Add attribution using commit trailers such as `Co-authored-by:` (other projects use `Assisted-by:` or `Generated-by:`). For example:
+11 -5
View File
@@ -60,15 +60,21 @@ the failure?
## Logs Wrangling
Download a job's log (no Buildkite login required):
Logs are public; no Buildkite login needed.
[.buildkite/scripts/ci-fetch-log.sh](../../../.buildkite/scripts/ci-fetch-log.sh)
saves each log as `ci-<build>-<job-name>.log`, stripped of timestamps and
ANSI codes:
```bash
# Find the failing job. Each row's URL is .../builds/<N>#<job_uuid>:
gh pr checks <PR> --repo vllm-project/vllm
# All failed jobs in a PR's latest build (current branch's PR if omitted):
.buildkite/scripts/ci-fetch-log.sh --pr <PR>
# Download + strip timestamps/ANSI in one step:
# All failed jobs in a build (--soft also includes soft-failed jobs;
# --all fetches every finished job):
.buildkite/scripts/ci-fetch-log.sh "https://buildkite.com/vllm/ci/builds/<N>"
# One job — `gh pr checks` URLs (#<job_uuid>) and web UI URLs (?sid=) both
# work; pass "-" as a second argument to stream to stdout:
.buildkite/scripts/ci-fetch-log.sh "https://buildkite.com/vllm/ci/builds/<N>#<job_uuid>"
```