From 869278cbb02fc04a5f10d2eaba2bb251939e1eab Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Mon, 28 Sep 2026 20:42:32 -0400 Subject: [PATCH] [CI] Bound the CRCR report by build age, and stop gating it on a job (#59066) --- .buildkite/scripts/crcr-report.sh | 100 ++++++++++++++++++++----- .buildkite/test_areas/crcr_report.yaml | 35 +++++---- 2 files changed, 100 insertions(+), 35 deletions(-) diff --git a/.buildkite/scripts/crcr-report.sh b/.buildkite/scripts/crcr-report.sh index 8812e876ae..3900b80339 100644 --- a/.buildkite/scripts/crcr-report.sh +++ b/.buildkite/scripts/crcr-report.sh @@ -79,12 +79,22 @@ if [[ -z "${OIDC_TOKEN}" ]]; then fi BUILD_JSON="$(mktemp)" -trap 'rm -f "${BUILD_JSON}"' EXIT +BUILD_JSON_STAGING="$(mktemp)" +trap 'rm -f "${BUILD_JSON}" "${BUILD_JSON_STAGING}"' EXIT BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}" +# Staged so a failed request cannot clobber the last good snapshot: curl -o +# writes the error body too, and the deadline path reports from whatever +# snapshot it has. fetch_build() { - curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \ - -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}" + local code + code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON_STAGING}" \ + -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}")" + if [[ "${code}" == "200" ]]; then + mv -f "${BUILD_JSON_STAGING}" "${BUILD_JSON}" + BUILD_JSON_STAGING="$(mktemp)" + fi + echo "${code}" } # Count jobs that have not reached a terminal state, ignoring this job: the @@ -113,22 +123,78 @@ PY # finished and none of the eventual 152 hard failures visible, and HUD recorded # that nightly as green. # -# Polling rather than a full `depends_on` barrier: Buildkite groups carry no -# key, so a barrier means naming all ~250 step keys, which silently rots when a -# step is added and fails the pipeline upload when one is removed or excluded -# from this lane (AMD, retired A100). The pipeline does gate this step on the -# long pole so it is scheduled near the end of the build -- that keeps the agent -# from idling for hours, but it is only a hint, and this loop is what actually -# guarantees the snapshot is complete. -POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}" -# Under the step's own timeout, so a stuck build still gets a partial report -# rather than the job being killed with nothing sent. -WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} )) +# Polling rather than a `depends_on` barrier: Buildkite groups carry no key, so +# a barrier means naming all ~250 step keys, which silently rots when a step is +# added and fails the pipeline upload when one is removed or excluded from this +# lane (AMD, retired A100). Gating on a single step was tried and is worse: a +# dependency that fails or is cancelled takes the report with it despite +# allow_dependency_failure (builds 91191 and 91461 both lost their report that +# way), so this step depends only on the image it runs in. +# 15 min: the wait now spans the whole build, so a 60s poll made ~420 requests +# against an API shared with the rest of CI to learn something that changes on +# the order of minutes. The cost is up to one interval of extra latency after +# the last job lands. +POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-900}" + +# The deadline is measured from when the *build* started, not from when this +# script did, so "report no later than N hours into the build" holds however +# late this step is scheduled. Anchoring it to script start made the bound +# meaningless whenever the step was itself delayed. +# +# Resolved after the first fetch, from the build's own created_at. +MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-25200}" +WAIT_DEADLINE="" +# Used until created_at is known. Without it, an API outage on the very first +# request leaves the deadline unresolved and the loop spins to the step timeout. +FALLBACK_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S )) + +build_deadline() { + python3 - "${BUILD_JSON}" "${MAX_BUILD_AGE_S}" <<'PY' +import datetime, json, sys +build = json.load(open(sys.argv[1])) +created = build.get("created_at") +if not created: + raise SystemExit(1) +started = datetime.datetime.fromisoformat(created.replace("Z", "+00:00")) +print(int(started.timestamp()) + int(sys.argv[2])) +PY +} + while :; do http_code="$(fetch_build)" if [[ "${http_code}" != "200" ]]; then - echo "buildkite API returned ${http_code} -- skipping report" - exit 0 + # A transient 429/5xx must not cost the whole report: treating one + # failure as fatal would drop a nightly for a blip. Keep polling on the + # last good snapshot; only give up if the deadline passes having never + # fetched one. + echo "buildkite API returned ${http_code}" + if [[ ! -s "${BUILD_JSON}" ]]; then + if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then + echo "no build data was ever fetched -- nothing to report" + exit 0 + fi + sleep "${POLL_INTERVAL_S}" + continue + fi + if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then + echo "deadline reached while the API is unavailable --" \ + "reporting from the last good snapshot" + break + fi + sleep "${POLL_INTERVAL_S}" + continue + fi + if [[ -z "${WAIT_DEADLINE}" ]]; then + WAIT_DEADLINE="$(build_deadline)" || WAIT_DEADLINE="" + if [[ -z "${WAIT_DEADLINE}" ]]; then + # No created_at to anchor to; fall back to script start so the loop + # still has a bound rather than running until the step times out. + WAIT_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S )) + echo "build created_at unavailable -- deadline measured from now" + fi + echo "reporting deadline: $(date -u -d "@${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null \ + || date -u -r "${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ')" \ + "(build age limit ${MAX_BUILD_AGE_S}s)" fi remaining="$(outstanding_jobs)" || remaining="" if [[ -z "${remaining}" ]]; then @@ -140,7 +206,7 @@ while :; do break fi if (( $(date +%s) >= WAIT_DEADLINE )); then - echo "still ${remaining} job(s) running at the wait deadline --" \ + echo "still ${remaining} job(s) running at the build-age deadline --" \ "reporting a partial view" break fi diff --git a/.buildkite/test_areas/crcr_report.yaml b/.buildkite/test_areas/crcr_report.yaml index f6b6582893..19e0e064e8 100644 --- a/.buildkite/test_areas/crcr_report.yaml +++ b/.buildkite/test_areas/crcr_report.yaml @@ -1,31 +1,30 @@ group: CRCR Report -# The report waits for the rest of the build (see crcr-report.sh), so gate it on -# the long pole rather than letting it wait on an agent. A torch-nightly build -# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three -# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in -# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours. +# Only image-build-cpu: this step must always run. # -# Depending on it means Buildkite does not schedule this step until the build is -# nearly over, so the agent polls briefly instead of idling ~13h. It is a -# scheduling hint, not the correctness mechanism: the script still waits for -# every job, so if this stops being the slowest step the report is late to start -# but still correct. +# It was briefly gated on the build's long pole (lm-eval-large-models-4xh100) +# so the agent would not idle. That backfired on the first nightly to use it, +# build 91191: the long pole OOMed, hung, and was cancelled -- and a cancelled +# dependency propagates past `allow_dependency_failure`, so the step went +# `waiting_failed` and that nightly got no CRCR row at all. Nothing in this +# pipeline sets timeout_in_minutes (the pipeline default and org maximum are +# both unset), so a hung job has no terminal state to reach either: gating on +# any single job can suppress the report indefinitely. +# +# The bound now lives in the script, measured from the build's created_at, so +# the report lands within CRCR_MAX_BUILD_AGE_S of the build starting whatever +# else happens. depends_on: - image-build-cpu - - lm-eval-large-models-4xh100 steps: - label: ":satellite: Report nightly results to CRCR" key: crcr-report if: build.env("TORCH_NIGHTLY") == "1" soft_fail: true allow_dependency_failure: true - # depends_on defers the start to near the end of the build, so in the normal - # case the script polls for minutes. This only has to cover the case where - # the long pole finishes early and something else is still running, so it is - # sized against the whole build rather than the expected wait. The script's - # own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial - # report; this is the backstop that stops the job outliving the build. - timeout_in_minutes: 840 + # The script polls from the start of the build now, so this covers the whole + # wait. It sits above the script's own build-age deadline (7h) so the script + # sends its partial report rather than being killed with nothing sent. + timeout_in_minutes: 480 device: cpu-small commands: - bash /vllm-workspace/.buildkite/scripts/crcr-report.sh