From c9e62f13c2b52619d8bc99f0c76cf04e77bff3c0 Mon Sep 17 00:00:00 2001 From: josh-schell-amd Date: Mon, 12 Oct 2026 13:04:37 -0500 Subject: [PATCH] [CI][ROCm] Add AITER nightly overlay scripts (#60883) Signed-off-by: josh schell Co-authored-by: Claude Sonnet 5.5 Co-authored-by: Micah Williamson --- .../scripts/rocm/aiter-nightly-overlay.sh | 112 ++++++++ .../rocm/select_aiter_nightly_wheel.py | 260 ++++++++++++++++++ 2 files changed, 372 insertions(+) create mode 100755 .buildkite/scripts/rocm/aiter-nightly-overlay.sh create mode 100755 .buildkite/scripts/rocm/select_aiter_nightly_wheel.py diff --git a/.buildkite/scripts/rocm/aiter-nightly-overlay.sh b/.buildkite/scripts/rocm/aiter-nightly-overlay.sh new file mode 100755 index 0000000000..528247964b --- /dev/null +++ b/.buildkite/scripts/rocm/aiter-nightly-overlay.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Install the AITER nightly wheel over this build's ci_base. +# +# Runs in CI when AITER_NIGHTLY=1, once the stock ci_base it builds on is +# published. In order: +# 1. Select the newest nightly wheel that fits the stock ci_base +# (select_aiter_nightly_wheel.py, beside this script, run inside that image) +# 2. Build a new image: pip install that wheel over the stock ci_base +# 3. Push the new image as this build's ci_base-build tag, replacing the stock +# one, so later steps and native jobs use it; record its digest +# +# Buildkite meta-data, a key-value store shared by every job in this build: +# reads rocm-ci-base-image the stock ci_base +# r/w aiter-nightly-wheel-url the wheel step 1 chose; a retry reads it +# back and installs that same wheel +# writes aiter-nightly-ci-base the new image as tag@digest; the vLLM +# image build uses it as its ci_base +# +# Exit status: +# 10 no wheel fits this ci_base +# 11 the wheel does not install +# Anything else is unexpected and passed through as is. +# Buildkite retries this step once on exit status 1 or 2, which it treats as +# transient Docker/registry failures (amd_infra_retry in the template). +# 10 and 11 sit outside that range: they are real AITER failures, and a retry +# would only fail the same way. + +set -euo pipefail + +SELECT_WHEEL="$(dirname "${BASH_SOURCE[0]}")/select_aiter_nightly_wheel.py" +BUILD_TAG="rocm/vllm-dev:ci_base-build-${BUILDKITE_BUILD_ID:?BUILDKITE_BUILD_ID is required}" + +# Scratch space for the build context and captured errors. The trap deletes it +# however the script exits: success, fail, or set -e. +scratch="$(mktemp -d)" +trap 'rm -rf "${scratch}"' EXIT +errors="${scratch}/errors" + +fail() { + local status="$1" title="$2" detail="$3" + echo "+++ :x: AITER nightly: ${title}" + echo "${detail}" >&2 + # shellcheck disable=SC2016 # the backticks are a literal markdown fence + printf ':x: **AITER nightly: %s**\n\n```\n%s\n```\n' "${title}" "${detail}" \ + | buildkite-agent annotate --style error --context aiter-nightly + exit "${status}" +} + +# Set by ensure-ci-base-amd; this step's depends_on guarantees it is there. +ci_base="$(buildkite-agent meta-data get rocm-ci-base-image)" +# Set below on the first attempt. Meta-data is per build, so a retry reuses +# that wheel, not a newer nightly. +pinned="$(buildkite-agent meta-data get aiter-nightly-wheel-url --default "")" + +echo "--- :docker: Pulling ${ci_base}" +docker pull "${ci_base}" + +echo "--- :mag: Selecting the AITER nightly for ${ci_base}" +set +e +out="$(docker run --rm -i --entrypoint python3 "${ci_base}" \ + -P - ${pinned:+--wheel-url "${pinned}"} \ + < "${SELECT_WHEEL}" 2> "${errors}")" +status=$? +set -e +cat "${errors}" >&2 +case "${status}" in + 0) ;; + 10) fail 10 "no wheel for this image" "${out}" ;; + *) fail "${status}" "could not select a wheel" "$(tail -n 20 "${errors}")" ;; +esac +# The first line is the wheel's URL; the rest, the annotation naming it. +wheel_url="$(head -n 1 <<< "${out}")" +note="$(tail -n +2 <<< "${out}")" +buildkite-agent meta-data set aiter-nightly-wheel-url "${wheel_url}" + +cat > "${scratch}/Dockerfile" <<'EOF' +ARG CI_BASE_IMAGE +FROM ${CI_BASE_IMAGE} +ARG AITER_WHEEL_URL +RUN python3 -m pip install --no-cache-dir "$AITER_WHEEL_URL" +EOF + +echo "--- :docker: Installing ${wheel_url##*/}" +if ! docker buildx build \ + --builder default \ + --provenance=false \ + --progress plain \ + --build-arg "CI_BASE_IMAGE=${ci_base}" \ + --build-arg "AITER_WHEEL_URL=${wheel_url}" \ + -t "${BUILD_TAG}" \ + "${scratch}"; then + fail 11 "the wheel does not install" \ + "Installing ${wheel_url##*/} over ${ci_base} failed; see this step's log." +fi + +docker push "${BUILD_TAG}" +# Handed off as tag@digest: build-test-image.sh rejects a ci_base that is not +# digest-pinned. +digest="$(docker buildx imagetools inspect "${BUILD_TAG}" | awk '$1 == "Digest:" { print $2; exit }')" +if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "Could not resolve the pushed digest of ${BUILD_TAG}" >&2 + exit 1 +fi +# The vLLM image build uses this as its ci_base. +buildkite-agent meta-data set aiter-nightly-ci-base "${BUILD_TAG}@${digest}" +# The annotation starts with :warning: when the wheel is stale. +style=info +if [[ "${note}" == :warning:* ]]; then + style=warning +fi +echo "${note}" | buildkite-agent annotate --style "${style}" --context aiter-nightly +echo "--- :white_check_mark: ${BUILD_TAG}@${digest}" diff --git a/.buildkite/scripts/rocm/select_aiter_nightly_wheel.py b/.buildkite/scripts/rocm/select_aiter_nightly_wheel.py new file mode 100755 index 0000000000..af3de54f60 --- /dev/null +++ b/.buildkite/scripts/rocm/select_aiter_nightly_wheel.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Select the AITER nightly wheel for a vLLM ROCm image. + +aiter-nightly-overlay.sh runs this inside the stock ci_base, so it reads that +image's ROCm, torch and Python. It prints the wheel's URL, then the Buildkite +annotation naming it. +""" + +import argparse +import datetime +import importlib.metadata +import os +import sys +import urllib.request +from dataclasses import dataclass +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import unquote, urldefrag, urljoin + +import regex as re + +INDEX_URL = "https://rocm.frameworks-nightlies.amd.com/whl-multi-arch/amd-aiter/" +AITER_COMMIT_URL = "https://github.com/ROCm/aiter/commit/" +STALE_AFTER = datetime.timedelta(days=1) + +# Example: +# amd_aiter-0.1.25+rocm7.2.3.torch2.12.ae984f5.d20261008-cp312-cp312-linux_x86_64.whl +# is AITER main at commit ae984f5, built 2026-10-08, for ROCm 7.2.3, torch 2.12 +# and Python 3.12. 0.1.25 is a guess at the next release, not a real one. +WHEEL_NAME = re.compile( + r"^amd_aiter-(?P[^+-]+)\+(?P[^-]+)" + r"-(?Pcp\d+)-(?P=python)-linux_x86_64\.whl$" +) +ROCM_FIELD = re.compile(r"(?:^|\.)rocm(\d+(?:\.\d+)*)(?:a\d+)?(?=\.|$)") +TORCH_FIELD = re.compile(r"(?:^|\.)torch(\d+\.\d+)(?=\.|$)") +COMMIT_AND_DATE_FIELD = re.compile(r"(?:^|\.)([0-9a-f]{7,40})\.d(\d{8})(?=\.|$)") + + +@dataclass(frozen=True) +class Wheel: + """An AITER nightly wheel, read from its filename (see WHEEL_NAME).""" + + filename: str + url: str + version: str + base_version: str + rocm: str + torch: str + commit: str + build_date: datetime.date + python: str + + +class _LinkParser(HTMLParser): + """Collects every link on the index page, which lists one per wheel.""" + + def __init__(self): + super().__init__() + self.hrefs: list[str] = [] + + def handle_starttag(self, tag, attrs): + if tag == "a": + self.hrefs.extend( + value for name, value in attrs if name == "href" and value + ) + + +def wheel_from_url(url: str) -> Wheel | None: + """The AITER nightly wheel an absolute URL points at, if it is one.""" + filename = unquote(urldefrag(url).url.rsplit("/", 1)[-1]) + match = WHEEL_NAME.match(filename) + if not match: + return None + local = match["local"] + rocm, torch, build = ( + field.search(local) + for field in (ROCM_FIELD, TORCH_FIELD, COMMIT_AND_DATE_FIELD) + ) + if not (rocm and torch and build): + return None + return Wheel( + filename=filename, + url=url, + version=f"{match['base_version']}+{local}", + base_version=match["base_version"], + rocm=rocm[1], + torch=torch[1], + commit=build[1], + build_date=datetime.datetime.strptime(build[2], "%Y%m%d").date(), + python=match["python"], + ) + + +def parse_index(html: str, index_url: str) -> list[Wheel]: + """Every AITER nightly wheel linked from the index page, a plain HTML list + of links, one per file (the format pip reads).""" + parser = _LinkParser() + parser.feed(html) + wheels = (wheel_from_url(urljoin(index_url, href)) for href in parser.hrefs) + return [wheel for wheel in wheels if wheel] + + +def rocm_version() -> str: + """This image's ROCm version; ROCm 10+ ships as the `rocm` pip SDK.""" + try: + return importlib.metadata.version("rocm") + except importlib.metadata.PackageNotFoundError: + # No pip SDK: fall back to older ROCm releases, which install to ROCM_PATH. + rocm_path = Path(os.environ.get("ROCM_PATH", "/opt/rocm")) + return (rocm_path / ".info" / "version").read_text() + + +def rocm_release(version: str) -> str: + """`7.2.3-49` or `10.0.0a20261001` -> `7.2.3` or `10.0.0`, as wheel names + spell it.""" + match = re.match(r"\d+(?:\.\d+)*", version.strip()) + if not match: + raise ValueError(f"Unrecognised ROCm version: {version!r}") + return match.group(0) + + +def torch_release(version: str) -> str: + """`2.12.0a0+git6bbd260` -> `2.12`, as wheel names spell it.""" + match = re.match(r"\d+\.\d+", version) + if not match: + raise ValueError(f"Unrecognised torch version: {version!r}") + return match.group(0) + + +def python_tag(version_info: tuple[int, int]) -> str: + """(3, 12) -> `cp312`, as wheel names spell it.""" + return f"cp{version_info[0]}{version_info[1]}" + + +def _base_version_key(wheel: Wheel) -> tuple[int, ...]: + """`0.1.25` -> (0, 1, 25), so versions sort as numbers: 0.1.10 > 0.1.9.""" + return tuple(int(part) for part in re.findall(r"\d+", wheel.base_version)) + + +def _label(rocm: str, torch: str, python: str) -> str: + """`rocm7.2.3/torch2.12/cp312`, for messages.""" + return f"rocm{rocm}/torch{torch}/{python}" + + +def select_wheel(wheels: list[Wheel], rocm: str, torch: str, python: str) -> Wheel: + """Newest build for this ROCm, torch and Python, by date: version labels lag. + + Two builds with the same date and version: whichever the index lists first. + """ + target = (rocm, torch, python) + matching = [w for w in wheels if (w.rocm, w.torch, w.python) == target] + if not matching: + available = sorted({_label(w.rocm, w.torch, w.python) for w in wheels}) + raise LookupError( + f"No AITER nightly for {_label(*target)}. " + f"Available: {', '.join(available) or 'none'}" + ) + return max(matching, key=lambda w: (w.build_date, _base_version_key(w))) + + +def fetch(url: str) -> str: + """GET url as text. A failure exits 1, so Buildkite retries the step.""" + with urllib.request.urlopen(url, timeout=60) as response: + return response.read().decode() + + +def annotation(wheel: Wheel, today: datetime.date) -> str: + """The Buildkite annotation naming the wheel this build tests. + + Starts with :warning: if the wheel is stale; the overlay script styles it so. + """ + age = today - wheel.build_date + stale = ( + f":warning: **AITER nightly is {age.days} days old**: " + f"no newer build has been published for " + f"{_label(wheel.rocm, wheel.torch, wheel.python)}.\n\n" + if age > STALE_AFTER + else "" + ) + return stale + ( + f":crescent_moon: **AITER nightly**: main at " + f"[`{wheel.commit}`]({AITER_COMMIT_URL}{wheel.commit}), " + f"built {wheel.build_date.isoformat()} for torch {wheel.torch} " + f"(`{wheel.version}`)" + ) + + +class NoWheel(Exception): + """No wheel fits this image's ROCm, torch and Python. Exit 10.""" + + +def _image_wheel(wheel_url: str) -> Wheel: + """The wheel at wheel_url, if it fits this image's ROCm, torch and Python.""" + wheel = wheel_from_url(wheel_url) + if not wheel: + raise NoWheel(f"Not an AITER nightly wheel: {wheel_url}") + return _for_image([wheel]) + + +def _image() -> tuple[str, str, str]: + """This image's ROCm, torch and Python, as wheel names spell them.""" + try: + rocm = rocm_release(rocm_version()) + except (OSError, ValueError) as error: + raise NoWheel(f"Cannot read this image's ROCm version: {error}") from error + try: + torch = torch_release(importlib.metadata.version("torch")) + except (importlib.metadata.PackageNotFoundError, ValueError) as error: + raise NoWheel(f"Cannot read this image's torch version: {error}") from error + return rocm, torch, python_tag(sys.version_info[:2]) + + +def _for_image(wheels: list[Wheel]) -> Wheel: + """The newest of wheels for the image this runs in.""" + try: + return select_wheel(wheels, *_image()) + except LookupError as error: + raise NoWheel(str(error)) from error + + +def select(index_url: str, wheel_url: str = "") -> Wheel: + """The wheel to install: wheel_url on a retry, else the newest. + + However old the newest is, it is tested; the annotation shows its build date. + """ + if wheel_url: + return _image_wheel(wheel_url) + return _for_image(parse_index(fetch(index_url), index_url)) + + +def _today() -> datetime.date: + """Today in UTC; a function so tests can set the date.""" + return datetime.datetime.now(datetime.UTC).date() + + +# Not 1 or 2, which Buildkite retries; see aiter-nightly-overlay.sh. +EXIT_NO_WHEEL = 10 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--index-url", default=INDEX_URL) + parser.add_argument( + "--wheel-url", default="", help="a retry's earlier choice; skips the index" + ) + args = parser.parse_args(argv) + try: + wheel = select(args.index_url, args.wheel_url) + except NoWheel as error: + print(error) + return EXIT_NO_WHEEL + print(wheel.url) + print(annotation(wheel, _today())) + return 0 + + +if __name__ == "__main__": + sys.exit(main())