[CI] Report torch-nightly results to PyTorch CRCR (#51830)

Signed-off-by: Andrey Talman <atalman@fb.com>

.buildkite/scripts/crcr-report.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Report this build's per-job results to the PyTorch Cross-Repo CI Relay (CRCR).
+#
+# Only runs in the torch-nightly lane on main. CRCR's nightly path is a
+# self-report: unlike PR-triggered callbacks there is no upstream dispatch to
+# correlate with, so the relay accepts a single "completed" callback per job and
+# forwards it to HUD (hud.pytorch.org/crcr).
+#
+# Reporting is best-effort. A relay outage, an expired mapping or a missing
+# token must never fail the nightly build, so every failure path here exits 0
+# after logging. The step is also marked soft_fail in the pipeline.
+
+set -uo pipefail
+
+# --- Gating -----------------------------------------------------------------
+# Matches the image_build.sh convention: the lane is selected inside the script
+# rather than in pipeline YAML.
+if [[ "${TORCH_NIGHTLY:-0}" != "1" ]]; then
+    echo "TORCH_NIGHTLY != 1 -- not the nightly lane, nothing to report"
+    exit 0
+fi
+
+# Defence in depth. The relay decides what a Buildkite pipeline may claim via
+# ci_providers.yml, and a pipeline that builds fork PRs should constrain
+# build_branch there. Refusing to even mint a token off main means a fork PR
+# cannot report as vllm-project/vllm even if that mapping is ever relaxed.
+if [[ "${BUILDKITE_BRANCH:-}" != "main" ]]; then
+    echo "branch '${BUILDKITE_BRANCH:-}' is not main -- refusing to report"
+    exit 0
+fi
+
+CALLBACK_URL="${CRCR_CALLBACK_URL:-}"
+if [[ -z "${CALLBACK_URL}" ]]; then
+    echo "CRCR_CALLBACK_URL unset -- skipping report"
+    exit 0
+fi
+
+# read_builds only. Needed because a job cannot see its siblings' outcomes:
+# the agent exposes only its own step, so the job list comes from the REST API.
+BK_TOKEN="${BUILDKITE_API_TOKEN:-}"
+if [[ -z "${BK_TOKEN}" ]]; then
+    echo "BUILDKITE_API_TOKEN unset -- skipping report"
+    exit 0
+fi
+
+AUDIENCE="pytorch-cross-repo-ci-relay"
+OIDC_TOKEN="$(buildkite-agent oidc request-token --audience "${AUDIENCE}" 2>/dev/null)"
+if [[ -z "${OIDC_TOKEN}" ]]; then
+    echo "could not mint a Buildkite OIDC token -- skipping report"
+    exit 0
+fi
+
+BUILD_JSON="$(mktemp)"
+trap 'rm -f "${BUILD_JSON}"' EXIT
+http_code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
+    -H "Authorization: Bearer ${BK_TOKEN}" \
+    "https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}")"
+if [[ "${http_code}" != "200" ]]; then
+    echo "buildkite API returned ${http_code} -- skipping report"
+    exit 0
+fi
+
+# One callback per job, matching HUD's per-job crcr_workflow_job schema.
+# delivery_id is synthetic: the nightly path has no upstream dispatch to borrow
+# one from, so build+job is used as the idempotency key.
+python3 .buildkite/scripts/crcr_report.py \
+    --build-json "${BUILD_JSON}" \
+    --callback-url "${CALLBACK_URL}" \
+    --oidc-token "${OIDC_TOKEN}" \
+  || echo "crcr report failed -- continuing (reporting is best-effort)"
+
+exit 0

.buildkite/scripts/crcr_report.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Post one CRCR nightly callback per Buildkite job.
+
+Separated from crcr-report.sh because the payload needs real JSON encoding: job
+labels routinely contain quotes, colons and emoji, and hand-building JSON in
+shell corrupts them silently.
+
+CRCR's nightly contract (aws/lambda/cross_repo_ci_relay/callback/callback_handler.py):
+
+    {"delivery_id": str,
+     "event_type": "nightly",
+     "workflow": {"status": "completed",   # nightly rejects any other status
+                  "conclusion": str,
+                  "run_id": int,
+                  "run_attempt": int,
+                  "name": str,
+                  "job_name": str}}
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import urllib.error
+import urllib.request
+
+# Buildkite job states -> the conclusion vocabulary HUD already stores for
+# GitHub Actions callbacks, so nightly rows aggregate with the rest.
+_CONCLUSION = {
+    "passed": "success",
+    "failed": "failure",
+    "broken": "failure",
+    "canceled": "cancelled",
+    "canceling": "cancelled",
+    "timed_out": "timed_out",
+    "timing_out": "timed_out",
+    "skipped": "skipped",
+    "not_run": "skipped",
+}
+
+
+def post(url: str, token: str, body: dict, timeout: int = 30) -> tuple[int, str]:
+    req = urllib.request.Request(
+        url,
+        data=json.dumps(body).encode(),
+        method="POST",
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+        },
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.status, resp.read().decode("utf-8", "replace")[:200]
+    except urllib.error.HTTPError as exc:
+        return exc.code, exc.read().decode("utf-8", "replace")[:200]
+    except Exception as exc:  # network, DNS, TLS
+        return 0, f"{type(exc).__name__}: {exc}"
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--build-json", required=True)
+    p.add_argument("--callback-url", required=True)
+    p.add_argument("--oidc-token", required=True)
+    p.add_argument("--dry-run", action="store_true")
+    args = p.parse_args()
+
+    with open(args.build_json) as f:
+        build = json.load(f)
+    build_number = build.get("number")
+    commit = build.get("commit", "")
+
+    # Only script jobs are real work. Buildkite also returns waiter/trigger/
+    # manual pseudo-jobs, which would otherwise show up in HUD as phantom
+    # zero-duration entries.
+    jobs = [
+        j
+        for j in build.get("jobs") or []
+        if j.get("type") == "script" and j.get("state") in _CONCLUSION
+    ]
+    if not jobs:
+        print("no terminal script jobs in this build; nothing to report")
+        return 0
+
+    ok = failed = 0
+    for job in jobs:
+        name = job.get("name") or job.get("command") or "<unnamed>"
+        body = {
+            # Stable per (build, job) so a re-run of this reporting step is
+            # idempotent from HUD's point of view rather than double-counting.
+            "delivery_id": f"buildkite-{build_number}-{job['id']}",
+            "event_type": "nightly",
+            "pytorch_head_sha": commit,
+            "workflow": {
+                "status": "completed",
+                "conclusion": _CONCLUSION[job["state"]],
+                "run_id": int(build_number),
+                "run_attempt": 1,
+                "name": build.get("pipeline", {}).get("slug", "ci"),
+                "job_name": name,
+                "url": job.get("web_url", ""),
+                "started_at": job.get("started_at"),
+                "completed_at": job.get("finished_at"),
+            },
+        }
+        if args.dry_run:
+            print(f"[dry-run] {_CONCLUSION[job['state']]:<10} {name[:70]}")
+            ok += 1
+            continue
+        status, detail = post(args.callback_url, args.oidc_token, body)
+        if 200 <= status < 300:
+            ok += 1
+        else:
+            failed += 1
+            # Printed once per job on purpose: a 403 means the pipeline mapping
+            # is missing, and a 200 {"status":"ignored"} means the repo is not
+            # allowlisted -- two failures that otherwise look identical.
+            print(f"  {status} for {name[:60]}: {detail}", file=sys.stderr)
+
+    print(f"reported {ok} job(s), {failed} failed")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.buildkite/test_areas/crcr_report.yaml

@@ -0,0 +1,37 @@
+group: CRCR Report
+# Runs only in the torch-nightly lane; the script exits 0 immediately otherwise,
+# following the same in-script gating as image_build.sh.
+#
+# ORDERING (needs confirmation against the pipeline generator): this must be the
+# last step to run, because it reads final job states from the Buildkite REST
+# API and silently skips any job still in a non-terminal state. On build 83159
+# four jobs were still `scheduled` when the build was already 320 jobs deep, so
+# running early under-reports rather than erroring.
+#
+# depends_on across every group is brittle to maintain. If the generator emits a
+# `wait` between groups, placing this group last is enough. If not, the more
+# robust option is an agent pre-exit hook or a follow-up scheduled step, since
+# `CONTINUE_ON_FAILURE=1` means failures must not stop the report either way.
+steps:
+  - label: ":satellite: Report nightly results to CRCR"
+    key: crcr-report
+    # Belt and braces with the script's own TORCH_NIGHTLY check: this keeps the
+    # step out of non-nightly builds entirely, while the script gate still holds
+    # if the generator ever drops this key.
+    if: build.env("TORCH_NIGHTLY") == "1"
+    # Reporting is observability, never a gate: a relay outage, an expired
+    # pipeline mapping or a missing secret must not turn the nightly red.
+    soft_fail: true
+    allow_dependency_failure: true
+    timeout_in_minutes: 15
+    # Reporting is a couple of REST calls and a curl: no GPU needed, so keep it
+    # off the accelerator agents the test groups queue for.
+    device: cpu-small
+    # No source_file_dependencies: this step is lane-gated, not change-gated,
+    # so it must not be skipped by the file-based step selection.
+    commands:
+      - bash .buildkite/scripts/crcr-report.sh
+    # CRCR_CALLBACK_URL and BUILDKITE_API_TOKEN are deliberately not declared
+    # here. Buildkite interpolates a step-level env block at pipeline-upload
+    # time, so declaring them would substitute empty strings and shadow the
+    # values the agent already has from the pipeline/schedule environment.