From f5b40fcdfefb66138cb20031f82c86ac0f53722a Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Thu, 24 Sep 2026 19:02:30 -0400 Subject: [PATCH] [CI] Report to CRCR after all jobs finish, gated on the build's long pole (#58628) --- .buildkite/scripts/crcr-report.sh | 74 +++++++++++++++++++++++--- .buildkite/test_areas/crcr_report.yaml | 20 ++++++- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/.buildkite/scripts/crcr-report.sh b/.buildkite/scripts/crcr-report.sh index 3b12d451cd..8812e876ae 100644 --- a/.buildkite/scripts/crcr-report.sh +++ b/.buildkite/scripts/crcr-report.sh @@ -80,13 +80,73 @@ fi BUILD_JSON="$(mktemp)" trap 'rm -f "${BUILD_JSON}"' EXIT -http_code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \ - -H "Authorization: Bearer ${BK_TOKEN}" \ - "https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}")" -if [[ "${http_code}" != "200" ]]; then - echo "buildkite API returned ${http_code} -- skipping report" - exit 0 -fi +BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}" + +fetch_build() { + curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \ + -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}" +} + +# Count jobs that have not reached a terminal state, ignoring this job: the +# report is itself part of the build and cannot wait for itself to finish. +outstanding_jobs() { + python3 - "${BUILD_JSON}" "${BUILDKITE_JOB_ID:-}" <<'PY' +import json, sys +TERMINAL = { + "passed", "failed", "blocked", "canceled", "skipped", "not_run", + "broken", "timed_out", "waiting_failed", "finished", "expired", +} +build = json.load(open(sys.argv[1])) +self_id = sys.argv[2] +print(sum( + 1 for j in build.get("jobs", []) + if j.get("type") == "script" + and j.get("id") != self_id + and j.get("state") not in TERMINAL +)) +PY +} + +# Wait for the rest of the build. The report is a snapshot of the Buildkite +# API's view of this build, so taking it early does not merely omit jobs -- it +# actively reports them as clean. Build 90830 reported with 14 of 347 jobs +# finished and none of the eventual 152 hard failures visible, and HUD recorded +# that nightly as green. +# +# Polling rather than a full `depends_on` barrier: Buildkite groups carry no +# key, so a barrier means naming all ~250 step keys, which silently rots when a +# step is added and fails the pipeline upload when one is removed or excluded +# from this lane (AMD, retired A100). The pipeline does gate this step on the +# long pole so it is scheduled near the end of the build -- that keeps the agent +# from idling for hours, but it is only a hint, and this loop is what actually +# guarantees the snapshot is complete. +POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}" +# Under the step's own timeout, so a stuck build still gets a partial report +# rather than the job being killed with nothing sent. +WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} )) +while :; do + http_code="$(fetch_build)" + if [[ "${http_code}" != "200" ]]; then + echo "buildkite API returned ${http_code} -- skipping report" + exit 0 + fi + remaining="$(outstanding_jobs)" || remaining="" + if [[ -z "${remaining}" ]]; then + echo "could not parse build JSON -- reporting what we have" + break + fi + if (( remaining == 0 )); then + echo "all other jobs have finished -- reporting" + break + fi + if (( $(date +%s) >= WAIT_DEADLINE )); then + echo "still ${remaining} job(s) running at the wait deadline --" \ + "reporting a partial view" + break + fi + echo "waiting for ${remaining} job(s) to finish..." + sleep "${POLL_INTERVAL_S}" +done # One callback per job, matching HUD's per-job crcr_workflow_job schema. # delivery_id is synthetic: the nightly path has no upstream dispatch to borrow diff --git a/.buildkite/test_areas/crcr_report.yaml b/.buildkite/test_areas/crcr_report.yaml index 0dfd6f50e2..f6b6582893 100644 --- a/.buildkite/test_areas/crcr_report.yaml +++ b/.buildkite/test_areas/crcr_report.yaml @@ -1,13 +1,31 @@ group: CRCR Report +# The report waits for the rest of the build (see crcr-report.sh), so gate it on +# the long pole rather than letting it wait on an agent. A torch-nightly build +# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three +# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in +# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours. +# +# Depending on it means Buildkite does not schedule this step until the build is +# nearly over, so the agent polls briefly instead of idling ~13h. It is a +# scheduling hint, not the correctness mechanism: the script still waits for +# every job, so if this stops being the slowest step the report is late to start +# but still correct. depends_on: - image-build-cpu + - lm-eval-large-models-4xh100 steps: - label: ":satellite: Report nightly results to CRCR" key: crcr-report if: build.env("TORCH_NIGHTLY") == "1" soft_fail: true allow_dependency_failure: true - timeout_in_minutes: 15 + # depends_on defers the start to near the end of the build, so in the normal + # case the script polls for minutes. This only has to cover the case where + # the long pole finishes early and something else is still running, so it is + # sized against the whole build rather than the expected wait. The script's + # own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial + # report; this is the backstop that stops the job outliving the build. + timeout_in_minutes: 840 device: cpu-small commands: - bash /vllm-workspace/.buildkite/scripts/crcr-report.sh