diff --git a/.buildkite/scripts/crcr-report.sh b/.buildkite/scripts/crcr-report.sh new file mode 100755 index 0000000000..613fdd46f9 --- /dev/null +++ b/.buildkite/scripts/crcr-report.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Report this build's per-job results to the PyTorch Cross-Repo CI Relay (CRCR). +# +# Only runs in the torch-nightly lane on main. CRCR's nightly path is a +# self-report: unlike PR-triggered callbacks there is no upstream dispatch to +# correlate with, so the relay accepts a single "completed" callback per job and +# forwards it to HUD (hud.pytorch.org/crcr). +# +# Reporting is best-effort. A relay outage, an expired mapping or a missing +# token must never fail the nightly build, so every failure path here exits 0 +# after logging. The step is also marked soft_fail in the pipeline. + +set -uo pipefail + +# --- Gating ----------------------------------------------------------------- +# Matches the image_build.sh convention: the lane is selected inside the script +# rather than in pipeline YAML. +if [[ "${TORCH_NIGHTLY:-0}" != "1" ]]; then + echo "TORCH_NIGHTLY != 1 -- not the nightly lane, nothing to report" + exit 0 +fi + +# Defence in depth. The relay decides what a Buildkite pipeline may claim via +# ci_providers.yml, and a pipeline that builds fork PRs should constrain +# build_branch there. Refusing to even mint a token off main means a fork PR +# cannot report as vllm-project/vllm even if that mapping is ever relaxed. +if [[ "${BUILDKITE_BRANCH:-}" != "main" ]]; then + echo "branch '${BUILDKITE_BRANCH:-}' is not main -- refusing to report" + exit 0 +fi + +CALLBACK_URL="${CRCR_CALLBACK_URL:-}" +if [[ -z "${CALLBACK_URL}" ]]; then + echo "CRCR_CALLBACK_URL unset -- skipping report" + exit 0 +fi + +# read_builds only. Needed because a job cannot see its siblings' outcomes: +# the agent exposes only its own step, so the job list comes from the REST API. +BK_TOKEN="${BUILDKITE_API_TOKEN:-}" +if [[ -z "${BK_TOKEN}" ]]; then + echo "BUILDKITE_API_TOKEN unset -- skipping report" + exit 0 +fi + +AUDIENCE="pytorch-cross-repo-ci-relay" +OIDC_TOKEN="$(buildkite-agent oidc request-token --audience "${AUDIENCE}" 2>/dev/null)" +if [[ -z "${OIDC_TOKEN}" ]]; then + echo "could not mint a Buildkite OIDC token -- skipping report" + exit 0 +fi + +BUILD_JSON="$(mktemp)" +trap 'rm -f "${BUILD_JSON}"' EXIT +http_code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \ + -H "Authorization: Bearer ${BK_TOKEN}" \ + "https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}")" +if [[ "${http_code}" != "200" ]]; then + echo "buildkite API returned ${http_code} -- skipping report" + exit 0 +fi + +# One callback per job, matching HUD's per-job crcr_workflow_job schema. +# delivery_id is synthetic: the nightly path has no upstream dispatch to borrow +# one from, so build+job is used as the idempotency key. +python3 .buildkite/scripts/crcr_report.py \ + --build-json "${BUILD_JSON}" \ + --callback-url "${CALLBACK_URL}" \ + --oidc-token "${OIDC_TOKEN}" \ + || echo "crcr report failed -- continuing (reporting is best-effort)" + +exit 0 diff --git a/.buildkite/scripts/crcr_report.py b/.buildkite/scripts/crcr_report.py new file mode 100644 index 0000000000..52f0649f01 --- /dev/null +++ b/.buildkite/scripts/crcr_report.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Post one CRCR nightly callback per Buildkite job. + +Separated from crcr-report.sh because the payload needs real JSON encoding: job +labels routinely contain quotes, colons and emoji, and hand-building JSON in +shell corrupts them silently. + +CRCR's nightly contract (aws/lambda/cross_repo_ci_relay/callback/callback_handler.py): + + {"delivery_id": str, + "event_type": "nightly", + "workflow": {"status": "completed", # nightly rejects any other status + "conclusion": str, + "run_id": int, + "run_attempt": int, + "name": str, + "job_name": str}} +""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.error +import urllib.request + +# Buildkite job states -> the conclusion vocabulary HUD already stores for +# GitHub Actions callbacks, so nightly rows aggregate with the rest. +_CONCLUSION = { + "passed": "success", + "failed": "failure", + "broken": "failure", + "canceled": "cancelled", + "canceling": "cancelled", + "timed_out": "timed_out", + "timing_out": "timed_out", + "skipped": "skipped", + "not_run": "skipped", +} + + +def post(url: str, token: str, body: dict, timeout: int = 30) -> tuple[int, str]: + req = urllib.request.Request( + url, + data=json.dumps(body).encode(), + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, resp.read().decode("utf-8", "replace")[:200] + except urllib.error.HTTPError as exc: + return exc.code, exc.read().decode("utf-8", "replace")[:200] + except Exception as exc: # network, DNS, TLS + return 0, f"{type(exc).__name__}: {exc}" + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--build-json", required=True) + p.add_argument("--callback-url", required=True) + p.add_argument("--oidc-token", required=True) + p.add_argument("--dry-run", action="store_true") + args = p.parse_args() + + with open(args.build_json) as f: + build = json.load(f) + build_number = build.get("number") + commit = build.get("commit", "") + + # Only script jobs are real work. Buildkite also returns waiter/trigger/ + # manual pseudo-jobs, which would otherwise show up in HUD as phantom + # zero-duration entries. + jobs = [ + j + for j in build.get("jobs") or [] + if j.get("type") == "script" and j.get("state") in _CONCLUSION + ] + if not jobs: + print("no terminal script jobs in this build; nothing to report") + return 0 + + ok = failed = 0 + for job in jobs: + name = job.get("name") or job.get("command") or "" + body = { + # Stable per (build, job) so a re-run of this reporting step is + # idempotent from HUD's point of view rather than double-counting. + "delivery_id": f"buildkite-{build_number}-{job['id']}", + "event_type": "nightly", + "pytorch_head_sha": commit, + "workflow": { + "status": "completed", + "conclusion": _CONCLUSION[job["state"]], + "run_id": int(build_number), + "run_attempt": 1, + "name": build.get("pipeline", {}).get("slug", "ci"), + "job_name": name, + "url": job.get("web_url", ""), + "started_at": job.get("started_at"), + "completed_at": job.get("finished_at"), + }, + } + if args.dry_run: + print(f"[dry-run] {_CONCLUSION[job['state']]:<10} {name[:70]}") + ok += 1 + continue + status, detail = post(args.callback_url, args.oidc_token, body) + if 200 <= status < 300: + ok += 1 + else: + failed += 1 + # Printed once per job on purpose: a 403 means the pipeline mapping + # is missing, and a 200 {"status":"ignored"} means the repo is not + # allowlisted -- two failures that otherwise look identical. + print(f" {status} for {name[:60]}: {detail}", file=sys.stderr) + + print(f"reported {ok} job(s), {failed} failed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.buildkite/test_areas/crcr_report.yaml b/.buildkite/test_areas/crcr_report.yaml new file mode 100644 index 0000000000..142a3b3fed --- /dev/null +++ b/.buildkite/test_areas/crcr_report.yaml @@ -0,0 +1,37 @@ +group: CRCR Report +# Runs only in the torch-nightly lane; the script exits 0 immediately otherwise, +# following the same in-script gating as image_build.sh. +# +# ORDERING (needs confirmation against the pipeline generator): this must be the +# last step to run, because it reads final job states from the Buildkite REST +# API and silently skips any job still in a non-terminal state. On build 83159 +# four jobs were still `scheduled` when the build was already 320 jobs deep, so +# running early under-reports rather than erroring. +# +# depends_on across every group is brittle to maintain. If the generator emits a +# `wait` between groups, placing this group last is enough. If not, the more +# robust option is an agent pre-exit hook or a follow-up scheduled step, since +# `CONTINUE_ON_FAILURE=1` means failures must not stop the report either way. +steps: + - label: ":satellite: Report nightly results to CRCR" + key: crcr-report + # Belt and braces with the script's own TORCH_NIGHTLY check: this keeps the + # step out of non-nightly builds entirely, while the script gate still holds + # if the generator ever drops this key. + if: build.env("TORCH_NIGHTLY") == "1" + # Reporting is observability, never a gate: a relay outage, an expired + # pipeline mapping or a missing secret must not turn the nightly red. + soft_fail: true + allow_dependency_failure: true + timeout_in_minutes: 15 + # Reporting is a couple of REST calls and a curl: no GPU needed, so keep it + # off the accelerator agents the test groups queue for. + device: cpu-small + # No source_file_dependencies: this step is lane-gated, not change-gated, + # so it must not be skipped by the file-based step selection. + commands: + - bash .buildkite/scripts/crcr-report.sh + # CRCR_CALLBACK_URL and BUILDKITE_API_TOKEN are deliberately not declared + # here. Buildkite interpolates a step-level env block at pipeline-upload + # time, so declaring them would substitute empty strings and shadow the + # values the agent already has from the pipeline/schedule environment.