[CI] Report to CRCR after all jobs finish, gated on the build's long pole (#58628)

.buildkite/scripts/crcr-report.sh

@@ -80,13 +80,73 @@ fi
 
 BUILD_JSON="$(mktemp)"
 trap 'rm -f "${BUILD_JSON}"' EXIT
-http_code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
-    -H "Authorization: Bearer ${BK_TOKEN}" \
-    "https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}")"
-if [[ "${http_code}" != "200" ]]; then
-    echo "buildkite API returned ${http_code} -- skipping report"
-    exit 0
-fi
+BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}"
+
+fetch_build() {
+    curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
+        -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}"
+}
+
+# Count jobs that have not reached a terminal state, ignoring this job: the
+# report is itself part of the build and cannot wait for itself to finish.
+outstanding_jobs() {
+    python3 - "${BUILD_JSON}" "${BUILDKITE_JOB_ID:-}" <<'PY'
+import json, sys
+TERMINAL = {
+    "passed", "failed", "blocked", "canceled", "skipped", "not_run",
+    "broken", "timed_out", "waiting_failed", "finished", "expired",
+}
+build = json.load(open(sys.argv[1]))
+self_id = sys.argv[2]
+print(sum(
+    1 for j in build.get("jobs", [])
+    if j.get("type") == "script"
+    and j.get("id") != self_id
+    and j.get("state") not in TERMINAL
+))
+PY
+}
+
+# Wait for the rest of the build. The report is a snapshot of the Buildkite
+# API's view of this build, so taking it early does not merely omit jobs -- it
+# actively reports them as clean. Build 90830 reported with 14 of 347 jobs
+# finished and none of the eventual 152 hard failures visible, and HUD recorded
+# that nightly as green.
+#
+# Polling rather than a full `depends_on` barrier: Buildkite groups carry no
+# key, so a barrier means naming all ~250 step keys, which silently rots when a
+# step is added and fails the pipeline upload when one is removed or excluded
+# from this lane (AMD, retired A100). The pipeline does gate this step on the
+# long pole so it is scheduled near the end of the build -- that keeps the agent
+# from idling for hours, but it is only a hint, and this loop is what actually
+# guarantees the snapshot is complete.
+POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}"
+# Under the step's own timeout, so a stuck build still gets a partial report
+# rather than the job being killed with nothing sent.
+WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} ))
+while :; do
+    http_code="$(fetch_build)"
+    if [[ "${http_code}" != "200" ]]; then
+        echo "buildkite API returned ${http_code} -- skipping report"
+        exit 0
+    fi
+    remaining="$(outstanding_jobs)" || remaining=""
+    if [[ -z "${remaining}" ]]; then
+        echo "could not parse build JSON -- reporting what we have"
+        break
+    fi
+    if (( remaining == 0 )); then
+        echo "all other jobs have finished -- reporting"
+        break
+    fi
+    if (( $(date +%s) >= WAIT_DEADLINE )); then
+        echo "still ${remaining} job(s) running at the wait deadline --" \
+            "reporting a partial view"
+        break
+    fi
+    echo "waiting for ${remaining} job(s) to finish..."
+    sleep "${POLL_INTERVAL_S}"
+done
 
 # One callback per job, matching HUD's per-job crcr_workflow_job schema.
 # delivery_id is synthetic: the nightly path has no upstream dispatch to borrow

.buildkite/test_areas/crcr_report.yaml

@@ -1,13 +1,31 @@
 group: CRCR Report
+# The report waits for the rest of the build (see crcr-report.sh), so gate it on
+# the long pole rather than letting it wait on an agent. A torch-nightly build
+# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three
+# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in
+# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours.
+#
+# Depending on it means Buildkite does not schedule this step until the build is
+# nearly over, so the agent polls briefly instead of idling ~13h. It is a
+# scheduling hint, not the correctness mechanism: the script still waits for
+# every job, so if this stops being the slowest step the report is late to start
+# but still correct.
 depends_on:
   - image-build-cpu
+  - lm-eval-large-models-4xh100
 steps:
   - label: ":satellite: Report nightly results to CRCR"
     key: crcr-report
     if: build.env("TORCH_NIGHTLY") == "1"
     soft_fail: true
     allow_dependency_failure: true
-    timeout_in_minutes: 15
+    # depends_on defers the start to near the end of the build, so in the normal
+    # case the script polls for minutes. This only has to cover the case where
+    # the long pole finishes early and something else is still running, so it is
+    # sized against the whole build rather than the expected wait. The script's
+    # own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial
+    # report; this is the backstop that stops the job outliving the build.
+    timeout_in_minutes: 840
     device: cpu-small
     commands:
       - bash /vllm-workspace/.buildkite/scripts/crcr-report.sh