[CI] Bound the CRCR report by build age, and stop gating it on a job (#59066)
This commit is contained in:
@@ -79,12 +79,22 @@ if [[ -z "${OIDC_TOKEN}" ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
BUILD_JSON="$(mktemp)"
|
BUILD_JSON="$(mktemp)"
|
||||||
trap 'rm -f "${BUILD_JSON}"' EXIT
|
BUILD_JSON_STAGING="$(mktemp)"
|
||||||
|
trap 'rm -f "${BUILD_JSON}" "${BUILD_JSON_STAGING}"' EXIT
|
||||||
BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}"
|
BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}"
|
||||||
|
|
||||||
|
# Staged so a failed request cannot clobber the last good snapshot: curl -o
|
||||||
|
# writes the error body too, and the deadline path reports from whatever
|
||||||
|
# snapshot it has.
|
||||||
fetch_build() {
|
fetch_build() {
|
||||||
curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
|
local code
|
||||||
-H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}"
|
code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON_STAGING}" \
|
||||||
|
-H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}")"
|
||||||
|
if [[ "${code}" == "200" ]]; then
|
||||||
|
mv -f "${BUILD_JSON_STAGING}" "${BUILD_JSON}"
|
||||||
|
BUILD_JSON_STAGING="$(mktemp)"
|
||||||
|
fi
|
||||||
|
echo "${code}"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Count jobs that have not reached a terminal state, ignoring this job: the
|
# Count jobs that have not reached a terminal state, ignoring this job: the
|
||||||
@@ -113,22 +123,78 @@ PY
|
|||||||
# finished and none of the eventual 152 hard failures visible, and HUD recorded
|
# finished and none of the eventual 152 hard failures visible, and HUD recorded
|
||||||
# that nightly as green.
|
# that nightly as green.
|
||||||
#
|
#
|
||||||
# Polling rather than a full `depends_on` barrier: Buildkite groups carry no
|
# Polling rather than a `depends_on` barrier: Buildkite groups carry no key, so
|
||||||
# key, so a barrier means naming all ~250 step keys, which silently rots when a
|
# a barrier means naming all ~250 step keys, which silently rots when a step is
|
||||||
# step is added and fails the pipeline upload when one is removed or excluded
|
# added and fails the pipeline upload when one is removed or excluded from this
|
||||||
# from this lane (AMD, retired A100). The pipeline does gate this step on the
|
# lane (AMD, retired A100). Gating on a single step was tried and is worse: a
|
||||||
# long pole so it is scheduled near the end of the build -- that keeps the agent
|
# dependency that fails or is cancelled takes the report with it despite
|
||||||
# from idling for hours, but it is only a hint, and this loop is what actually
|
# allow_dependency_failure (builds 91191 and 91461 both lost their report that
|
||||||
# guarantees the snapshot is complete.
|
# way), so this step depends only on the image it runs in.
|
||||||
POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}"
|
# 15 min: the wait now spans the whole build, so a 60s poll made ~420 requests
|
||||||
# Under the step's own timeout, so a stuck build still gets a partial report
|
# against an API shared with the rest of CI to learn something that changes on
|
||||||
# rather than the job being killed with nothing sent.
|
# the order of minutes. The cost is up to one interval of extra latency after
|
||||||
WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} ))
|
# the last job lands.
|
||||||
|
POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-900}"
|
||||||
|
|
||||||
|
# The deadline is measured from when the *build* started, not from when this
|
||||||
|
# script did, so "report no later than N hours into the build" holds however
|
||||||
|
# late this step is scheduled. Anchoring it to script start made the bound
|
||||||
|
# meaningless whenever the step was itself delayed.
|
||||||
|
#
|
||||||
|
# Resolved after the first fetch, from the build's own created_at.
|
||||||
|
MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-25200}"
|
||||||
|
WAIT_DEADLINE=""
|
||||||
|
# Used until created_at is known. Without it, an API outage on the very first
|
||||||
|
# request leaves the deadline unresolved and the loop spins to the step timeout.
|
||||||
|
FALLBACK_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S ))
|
||||||
|
|
||||||
|
build_deadline() {
|
||||||
|
python3 - "${BUILD_JSON}" "${MAX_BUILD_AGE_S}" <<'PY'
|
||||||
|
import datetime, json, sys
|
||||||
|
build = json.load(open(sys.argv[1]))
|
||||||
|
created = build.get("created_at")
|
||||||
|
if not created:
|
||||||
|
raise SystemExit(1)
|
||||||
|
started = datetime.datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||||
|
print(int(started.timestamp()) + int(sys.argv[2]))
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
while :; do
|
while :; do
|
||||||
http_code="$(fetch_build)"
|
http_code="$(fetch_build)"
|
||||||
if [[ "${http_code}" != "200" ]]; then
|
if [[ "${http_code}" != "200" ]]; then
|
||||||
echo "buildkite API returned ${http_code} -- skipping report"
|
# A transient 429/5xx must not cost the whole report: treating one
|
||||||
exit 0
|
# failure as fatal would drop a nightly for a blip. Keep polling on the
|
||||||
|
# last good snapshot; only give up if the deadline passes having never
|
||||||
|
# fetched one.
|
||||||
|
echo "buildkite API returned ${http_code}"
|
||||||
|
if [[ ! -s "${BUILD_JSON}" ]]; then
|
||||||
|
if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then
|
||||||
|
echo "no build data was ever fetched -- nothing to report"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
sleep "${POLL_INTERVAL_S}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then
|
||||||
|
echo "deadline reached while the API is unavailable --" \
|
||||||
|
"reporting from the last good snapshot"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep "${POLL_INTERVAL_S}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ -z "${WAIT_DEADLINE}" ]]; then
|
||||||
|
WAIT_DEADLINE="$(build_deadline)" || WAIT_DEADLINE=""
|
||||||
|
if [[ -z "${WAIT_DEADLINE}" ]]; then
|
||||||
|
# No created_at to anchor to; fall back to script start so the loop
|
||||||
|
# still has a bound rather than running until the step times out.
|
||||||
|
WAIT_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S ))
|
||||||
|
echo "build created_at unavailable -- deadline measured from now"
|
||||||
|
fi
|
||||||
|
echo "reporting deadline: $(date -u -d "@${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null \
|
||||||
|
|| date -u -r "${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ')" \
|
||||||
|
"(build age limit ${MAX_BUILD_AGE_S}s)"
|
||||||
fi
|
fi
|
||||||
remaining="$(outstanding_jobs)" || remaining=""
|
remaining="$(outstanding_jobs)" || remaining=""
|
||||||
if [[ -z "${remaining}" ]]; then
|
if [[ -z "${remaining}" ]]; then
|
||||||
@@ -140,7 +206,7 @@ while :; do
|
|||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
if (( $(date +%s) >= WAIT_DEADLINE )); then
|
if (( $(date +%s) >= WAIT_DEADLINE )); then
|
||||||
echo "still ${remaining} job(s) running at the wait deadline --" \
|
echo "still ${remaining} job(s) running at the build-age deadline --" \
|
||||||
"reporting a partial view"
|
"reporting a partial view"
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -1,31 +1,30 @@
|
|||||||
group: CRCR Report
|
group: CRCR Report
|
||||||
# The report waits for the rest of the build (see crcr-report.sh), so gate it on
|
# Only image-build-cpu: this step must always run.
|
||||||
# the long pole rather than letting it wait on an agent. A torch-nightly build
|
|
||||||
# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three
|
|
||||||
# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in
|
|
||||||
# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours.
|
|
||||||
#
|
#
|
||||||
# Depending on it means Buildkite does not schedule this step until the build is
|
# It was briefly gated on the build's long pole (lm-eval-large-models-4xh100)
|
||||||
# nearly over, so the agent polls briefly instead of idling ~13h. It is a
|
# so the agent would not idle. That backfired on the first nightly to use it,
|
||||||
# scheduling hint, not the correctness mechanism: the script still waits for
|
# build 91191: the long pole OOMed, hung, and was cancelled -- and a cancelled
|
||||||
# every job, so if this stops being the slowest step the report is late to start
|
# dependency propagates past `allow_dependency_failure`, so the step went
|
||||||
# but still correct.
|
# `waiting_failed` and that nightly got no CRCR row at all. Nothing in this
|
||||||
|
# pipeline sets timeout_in_minutes (the pipeline default and org maximum are
|
||||||
|
# both unset), so a hung job has no terminal state to reach either: gating on
|
||||||
|
# any single job can suppress the report indefinitely.
|
||||||
|
#
|
||||||
|
# The bound now lives in the script, measured from the build's created_at, so
|
||||||
|
# the report lands within CRCR_MAX_BUILD_AGE_S of the build starting whatever
|
||||||
|
# else happens.
|
||||||
depends_on:
|
depends_on:
|
||||||
- image-build-cpu
|
- image-build-cpu
|
||||||
- lm-eval-large-models-4xh100
|
|
||||||
steps:
|
steps:
|
||||||
- label: ":satellite: Report nightly results to CRCR"
|
- label: ":satellite: Report nightly results to CRCR"
|
||||||
key: crcr-report
|
key: crcr-report
|
||||||
if: build.env("TORCH_NIGHTLY") == "1"
|
if: build.env("TORCH_NIGHTLY") == "1"
|
||||||
soft_fail: true
|
soft_fail: true
|
||||||
allow_dependency_failure: true
|
allow_dependency_failure: true
|
||||||
# depends_on defers the start to near the end of the build, so in the normal
|
# The script polls from the start of the build now, so this covers the whole
|
||||||
# case the script polls for minutes. This only has to cover the case where
|
# wait. It sits above the script's own build-age deadline (7h) so the script
|
||||||
# the long pole finishes early and something else is still running, so it is
|
# sends its partial report rather than being killed with nothing sent.
|
||||||
# sized against the whole build rather than the expected wait. The script's
|
timeout_in_minutes: 480
|
||||||
# own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial
|
|
||||||
# report; this is the backstop that stops the job outliving the build.
|
|
||||||
timeout_in_minutes: 840
|
|
||||||
device: cpu-small
|
device: cpu-small
|
||||||
commands:
|
commands:
|
||||||
- bash /vllm-workspace/.buildkite/scripts/crcr-report.sh
|
- bash /vllm-workspace/.buildkite/scripts/crcr-report.sh
|
||||||
|
|||||||
Reference in New Issue
Block a user