[CI] Bound the CRCR report by build age, and stop gating it on a job (#59066)
This commit is contained in:
Andrey Talman
@@ -79,12 +79,22 @@ if [[ -z "${OIDC_TOKEN}" ]]; then
|
||||
fi
|
||||
|
||||
BUILD_JSON="$(mktemp)"
|
||||
trap 'rm -f "${BUILD_JSON}"' EXIT
|
||||
BUILD_JSON_STAGING="$(mktemp)"
|
||||
trap 'rm -f "${BUILD_JSON}" "${BUILD_JSON_STAGING}"' EXIT
|
||||
BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}"
|
||||
|
||||
# Staged so a failed request cannot clobber the last good snapshot: curl -o
|
||||
# writes the error body too, and the deadline path reports from whatever
|
||||
# snapshot it has.
|
||||
fetch_build() {
|
||||
curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
|
||||
-H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}"
|
||||
local code
|
||||
code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON_STAGING}" \
|
||||
-H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}")"
|
||||
if [[ "${code}" == "200" ]]; then
|
||||
mv -f "${BUILD_JSON_STAGING}" "${BUILD_JSON}"
|
||||
BUILD_JSON_STAGING="$(mktemp)"
|
||||
fi
|
||||
echo "${code}"
|
||||
}
|
||||
|
||||
# Count jobs that have not reached a terminal state, ignoring this job: the
|
||||
@@ -113,22 +123,78 @@ PY
|
||||
# finished and none of the eventual 152 hard failures visible, and HUD recorded
|
||||
# that nightly as green.
|
||||
#
|
||||
# Polling rather than a full `depends_on` barrier: Buildkite groups carry no
|
||||
# key, so a barrier means naming all ~250 step keys, which silently rots when a
|
||||
# step is added and fails the pipeline upload when one is removed or excluded
|
||||
# from this lane (AMD, retired A100). The pipeline does gate this step on the
|
||||
# long pole so it is scheduled near the end of the build -- that keeps the agent
|
||||
# from idling for hours, but it is only a hint, and this loop is what actually
|
||||
# guarantees the snapshot is complete.
|
||||
POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}"
|
||||
# Under the step's own timeout, so a stuck build still gets a partial report
|
||||
# rather than the job being killed with nothing sent.
|
||||
WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} ))
|
||||
# Polling rather than a `depends_on` barrier: Buildkite groups carry no key, so
|
||||
# a barrier means naming all ~250 step keys, which silently rots when a step is
|
||||
# added and fails the pipeline upload when one is removed or excluded from this
|
||||
# lane (AMD, retired A100). Gating on a single step was tried and is worse: a
|
||||
# dependency that fails or is cancelled takes the report with it despite
|
||||
# allow_dependency_failure (builds 91191 and 91461 both lost their report that
|
||||
# way), so this step depends only on the image it runs in.
|
||||
# 15 min: the wait now spans the whole build, so a 60s poll made ~420 requests
|
||||
# against an API shared with the rest of CI to learn something that changes on
|
||||
# the order of minutes. The cost is up to one interval of extra latency after
|
||||
# the last job lands.
|
||||
POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-900}"
|
||||
|
||||
# The deadline is measured from when the *build* started, not from when this
|
||||
# script did, so "report no later than N hours into the build" holds however
|
||||
# late this step is scheduled. Anchoring it to script start made the bound
|
||||
# meaningless whenever the step was itself delayed.
|
||||
#
|
||||
# Resolved after the first fetch, from the build's own created_at.
|
||||
MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-25200}"
|
||||
WAIT_DEADLINE=""
|
||||
# Used until created_at is known. Without it, an API outage on the very first
|
||||
# request leaves the deadline unresolved and the loop spins to the step timeout.
|
||||
FALLBACK_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S ))
|
||||
|
||||
build_deadline() {
|
||||
python3 - "${BUILD_JSON}" "${MAX_BUILD_AGE_S}" <<'PY'
|
||||
import datetime, json, sys
|
||||
build = json.load(open(sys.argv[1]))
|
||||
created = build.get("created_at")
|
||||
if not created:
|
||||
raise SystemExit(1)
|
||||
started = datetime.datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
print(int(started.timestamp()) + int(sys.argv[2]))
|
||||
PY
|
||||
}
|
||||
|
||||
while :; do
|
||||
http_code="$(fetch_build)"
|
||||
if [[ "${http_code}" != "200" ]]; then
|
||||
echo "buildkite API returned ${http_code} -- skipping report"
|
||||
exit 0
|
||||
# A transient 429/5xx must not cost the whole report: treating one
|
||||
# failure as fatal would drop a nightly for a blip. Keep polling on the
|
||||
# last good snapshot; only give up if the deadline passes having never
|
||||
# fetched one.
|
||||
echo "buildkite API returned ${http_code}"
|
||||
if [[ ! -s "${BUILD_JSON}" ]]; then
|
||||
if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then
|
||||
echo "no build data was ever fetched -- nothing to report"
|
||||
exit 0
|
||||
fi
|
||||
sleep "${POLL_INTERVAL_S}"
|
||||
continue
|
||||
fi
|
||||
if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then
|
||||
echo "deadline reached while the API is unavailable --" \
|
||||
"reporting from the last good snapshot"
|
||||
break
|
||||
fi
|
||||
sleep "${POLL_INTERVAL_S}"
|
||||
continue
|
||||
fi
|
||||
if [[ -z "${WAIT_DEADLINE}" ]]; then
|
||||
WAIT_DEADLINE="$(build_deadline)" || WAIT_DEADLINE=""
|
||||
if [[ -z "${WAIT_DEADLINE}" ]]; then
|
||||
# No created_at to anchor to; fall back to script start so the loop
|
||||
# still has a bound rather than running until the step times out.
|
||||
WAIT_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S ))
|
||||
echo "build created_at unavailable -- deadline measured from now"
|
||||
fi
|
||||
echo "reporting deadline: $(date -u -d "@${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null \
|
||||
|| date -u -r "${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ')" \
|
||||
"(build age limit ${MAX_BUILD_AGE_S}s)"
|
||||
fi
|
||||
remaining="$(outstanding_jobs)" || remaining=""
|
||||
if [[ -z "${remaining}" ]]; then
|
||||
@@ -140,7 +206,7 @@ while :; do
|
||||
break
|
||||
fi
|
||||
if (( $(date +%s) >= WAIT_DEADLINE )); then
|
||||
echo "still ${remaining} job(s) running at the wait deadline --" \
|
||||
echo "still ${remaining} job(s) running at the build-age deadline --" \
|
||||
"reporting a partial view"
|
||||
break
|
||||
fi
|
||||
|
||||
@@ -1,31 +1,30 @@
|
||||
group: CRCR Report
|
||||
# The report waits for the rest of the build (see crcr-report.sh), so gate it on
|
||||
# the long pole rather than letting it wait on an agent. A torch-nightly build
|
||||
# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three
|
||||
# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in
|
||||
# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours.
|
||||
# Only image-build-cpu: this step must always run.
|
||||
#
|
||||
# Depending on it means Buildkite does not schedule this step until the build is
|
||||
# nearly over, so the agent polls briefly instead of idling ~13h. It is a
|
||||
# scheduling hint, not the correctness mechanism: the script still waits for
|
||||
# every job, so if this stops being the slowest step the report is late to start
|
||||
# but still correct.
|
||||
# It was briefly gated on the build's long pole (lm-eval-large-models-4xh100)
|
||||
# so the agent would not idle. That backfired on the first nightly to use it,
|
||||
# build 91191: the long pole OOMed, hung, and was cancelled -- and a cancelled
|
||||
# dependency propagates past `allow_dependency_failure`, so the step went
|
||||
# `waiting_failed` and that nightly got no CRCR row at all. Nothing in this
|
||||
# pipeline sets timeout_in_minutes (the pipeline default and org maximum are
|
||||
# both unset), so a hung job has no terminal state to reach either: gating on
|
||||
# any single job can suppress the report indefinitely.
|
||||
#
|
||||
# The bound now lives in the script, measured from the build's created_at, so
|
||||
# the report lands within CRCR_MAX_BUILD_AGE_S of the build starting whatever
|
||||
# else happens.
|
||||
depends_on:
|
||||
- image-build-cpu
|
||||
- lm-eval-large-models-4xh100
|
||||
steps:
|
||||
- label: ":satellite: Report nightly results to CRCR"
|
||||
key: crcr-report
|
||||
if: build.env("TORCH_NIGHTLY") == "1"
|
||||
soft_fail: true
|
||||
allow_dependency_failure: true
|
||||
# depends_on defers the start to near the end of the build, so in the normal
|
||||
# case the script polls for minutes. This only has to cover the case where
|
||||
# the long pole finishes early and something else is still running, so it is
|
||||
# sized against the whole build rather than the expected wait. The script's
|
||||
# own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial
|
||||
# report; this is the backstop that stops the job outliving the build.
|
||||
timeout_in_minutes: 840
|
||||
# The script polls from the start of the build now, so this covers the whole
|
||||
# wait. It sits above the script's own build-age deadline (7h) so the script
|
||||
# sends its partial report rather than being killed with nothing sent.
|
||||
timeout_in_minutes: 480
|
||||
device: cpu-small
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/crcr-report.sh
|
||||
|
||||
Reference in New Issue
Block a user