diff --git a/.buildkite/scripts/crcr-report.sh b/.buildkite/scripts/crcr-report.sh index 0c114f4653..a1c5610a51 100644 --- a/.buildkite/scripts/crcr-report.sh +++ b/.buildkite/scripts/crcr-report.sh @@ -227,7 +227,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # five-minute lifetime and the loop above can run to the build-age deadline. # Build 91827 waited 6h48m and then took a 401 on all 360 callbacks from a # token that had expired hours earlier. -OIDC_TOKEN="$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)" +# Posting is paced under the relay's rate limit, which takes several minutes +# for a full nightly, so ask for a longer lifetime than the default. Fall back +# to the default if this agent rejects --lifetime. +OIDC_TOKEN="$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" \ + --lifetime 1800 2>/dev/null \ + || buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)" if [[ -z "${OIDC_TOKEN}" ]]; then echo "could not mint a Buildkite OIDC token after the wait -- skipping report" exit 0 diff --git a/.buildkite/scripts/crcr_report.py b/.buildkite/scripts/crcr_report.py index 52f0649f01..4fab92a2c5 100644 --- a/.buildkite/scripts/crcr_report.py +++ b/.buildkite/scripts/crcr_report.py @@ -24,7 +24,9 @@ from __future__ import annotations import argparse import json +import os import sys +import time import urllib.error import urllib.request @@ -43,6 +45,14 @@ _CONCLUSION = { } +# The relay rate-limits callbacks per repo over a sliding 60 s window +# (RATE_LIMIT_PER_MIN in pytorch/test-infra), and a nightly has far more jobs +# than that. Pace below the limit, and back off on a 429 for about a minute in +# total so the last retry lands in a new window. +_POST_INTERVAL_S = float(os.environ.get("CRCR_POST_INTERVAL_S", "0.65")) +_RATE_LIMIT_BACKOFF_S = (2, 4, 8, 16, 32) + + def post(url: str, token: str, body: dict, timeout: int = 30) -> tuple[int, str]: req = urllib.request.Request( url, @@ -113,6 +123,12 @@ def main() -> int: ok += 1 continue status, detail = post(args.callback_url, args.oidc_token, body) + for delay in _RATE_LIMIT_BACKOFF_S: + if status != 429: + break + time.sleep(delay) + status, detail = post(args.callback_url, args.oidc_token, body) + time.sleep(_POST_INTERVAL_S) if 200 <= status < 300: ok += 1 else: