[CI] Pace the CRCR nightly report under the relay rate limit (#61269)

Signed-off-by: Andrey Talman <atalman@fb.com>

.buildkite/scripts/crcr-report.sh

@@ -227,7 +227,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # five-minute lifetime and the loop above can run to the build-age deadline.
 # Build 91827 waited 6h48m and then took a 401 on all 360 callbacks from a
 # token that had expired hours earlier.
-OIDC_TOKEN="$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)"
+# Posting is paced under the relay's rate limit, which takes several minutes
+# for a full nightly, so ask for a longer lifetime than the default. Fall back
+# to the default if this agent rejects --lifetime.
+OIDC_TOKEN="$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" \
+    --lifetime 1800 2>/dev/null \
+    || buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)"
 if [[ -z "${OIDC_TOKEN}" ]]; then
     echo "could not mint a Buildkite OIDC token after the wait -- skipping report"
     exit 0

.buildkite/scripts/crcr_report.py

@@ -24,7 +24,9 @@ from __future__ import annotations
 
 import argparse
 import json
+import os
 import sys
+import time
 import urllib.error
 import urllib.request
 
@@ -43,6 +45,14 @@ _CONCLUSION = {
 }
 
 
+# The relay rate-limits callbacks per repo over a sliding 60 s window
+# (RATE_LIMIT_PER_MIN in pytorch/test-infra), and a nightly has far more jobs
+# than that. Pace below the limit, and back off on a 429 for about a minute in
+# total so the last retry lands in a new window.
+_POST_INTERVAL_S = float(os.environ.get("CRCR_POST_INTERVAL_S", "0.65"))
+_RATE_LIMIT_BACKOFF_S = (2, 4, 8, 16, 32)
+
+
 def post(url: str, token: str, body: dict, timeout: int = 30) -> tuple[int, str]:
     req = urllib.request.Request(
         url,
@@ -113,6 +123,12 @@ def main() -> int:
             ok += 1
             continue
         status, detail = post(args.callback_url, args.oidc_token, body)
+        for delay in _RATE_LIMIT_BACKOFF_S:
+            if status != 429:
+                break
+            time.sleep(delay)
+            status, detail = post(args.callback_url, args.oidc_token, body)
+        time.sleep(_POST_INTERVAL_S)
         if 200 <= status < 300:
             ok += 1
         else: