[CI][ROCm] Add AITER nightly overlay scripts (#60883)

Signed-off-by: josh schell <josh.schell@amd.com>
Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-authored-by: Micah Williamson <micah.williamson@amd.com>
This commit is contained in:
josh-schell-amd
2026-10-12 13:04:37 -05:00
committed by GitHub
co-authored by Claude Sonnet 5.5 Micah Williamson
parent e1a5086256
commit c9e62f13c2
2 changed files with 372 additions and 0 deletions
+112
View File
@@ -0,0 +1,112 @@
#!/bin/bash
# Install the AITER nightly wheel over this build's ci_base.
#
# Runs in CI when AITER_NIGHTLY=1, once the stock ci_base it builds on is
# published. In order:
# 1. Select the newest nightly wheel that fits the stock ci_base
# (select_aiter_nightly_wheel.py, beside this script, run inside that image)
# 2. Build a new image: pip install that wheel over the stock ci_base
# 3. Push the new image as this build's ci_base-build tag, replacing the stock
# one, so later steps and native jobs use it; record its digest
#
# Buildkite meta-data, a key-value store shared by every job in this build:
# reads rocm-ci-base-image the stock ci_base
# r/w aiter-nightly-wheel-url the wheel step 1 chose; a retry reads it
# back and installs that same wheel
# writes aiter-nightly-ci-base the new image as tag@digest; the vLLM
# image build uses it as its ci_base
#
# Exit status:
# 10 no wheel fits this ci_base
# 11 the wheel does not install
# Anything else is unexpected and passed through as is.
# Buildkite retries this step once on exit status 1 or 2, which it treats as
# transient Docker/registry failures (amd_infra_retry in the template).
# 10 and 11 sit outside that range: they are real AITER failures, and a retry
# would only fail the same way.
set -euo pipefail
SELECT_WHEEL="$(dirname "${BASH_SOURCE[0]}")/select_aiter_nightly_wheel.py"
BUILD_TAG="rocm/vllm-dev:ci_base-build-${BUILDKITE_BUILD_ID:?BUILDKITE_BUILD_ID is required}"
# Scratch space for the build context and captured errors. The trap deletes it
# however the script exits: success, fail, or set -e.
scratch="$(mktemp -d)"
trap 'rm -rf "${scratch}"' EXIT
errors="${scratch}/errors"
fail() {
local status="$1" title="$2" detail="$3"
echo "+++ :x: AITER nightly: ${title}"
echo "${detail}" >&2
# shellcheck disable=SC2016 # the backticks are a literal markdown fence
printf ':x: **AITER nightly: %s**\n\n```\n%s\n```\n' "${title}" "${detail}" \
| buildkite-agent annotate --style error --context aiter-nightly
exit "${status}"
}
# Set by ensure-ci-base-amd; this step's depends_on guarantees it is there.
ci_base="$(buildkite-agent meta-data get rocm-ci-base-image)"
# Set below on the first attempt. Meta-data is per build, so a retry reuses
# that wheel, not a newer nightly.
pinned="$(buildkite-agent meta-data get aiter-nightly-wheel-url --default "")"
echo "--- :docker: Pulling ${ci_base}"
docker pull "${ci_base}"
echo "--- :mag: Selecting the AITER nightly for ${ci_base}"
set +e
out="$(docker run --rm -i --entrypoint python3 "${ci_base}" \
-P - ${pinned:+--wheel-url "${pinned}"} \
< "${SELECT_WHEEL}" 2> "${errors}")"
status=$?
set -e
cat "${errors}" >&2
case "${status}" in
0) ;;
10) fail 10 "no wheel for this image" "${out}" ;;
*) fail "${status}" "could not select a wheel" "$(tail -n 20 "${errors}")" ;;
esac
# The first line is the wheel's URL; the rest, the annotation naming it.
wheel_url="$(head -n 1 <<< "${out}")"
note="$(tail -n +2 <<< "${out}")"
buildkite-agent meta-data set aiter-nightly-wheel-url "${wheel_url}"
cat > "${scratch}/Dockerfile" <<'EOF'
ARG CI_BASE_IMAGE
FROM ${CI_BASE_IMAGE}
ARG AITER_WHEEL_URL
RUN python3 -m pip install --no-cache-dir "$AITER_WHEEL_URL"
EOF
echo "--- :docker: Installing ${wheel_url##*/}"
if ! docker buildx build \
--builder default \
--provenance=false \
--progress plain \
--build-arg "CI_BASE_IMAGE=${ci_base}" \
--build-arg "AITER_WHEEL_URL=${wheel_url}" \
-t "${BUILD_TAG}" \
"${scratch}"; then
fail 11 "the wheel does not install" \
"Installing ${wheel_url##*/} over ${ci_base} failed; see this step's log."
fi
docker push "${BUILD_TAG}"
# Handed off as tag@digest: build-test-image.sh rejects a ci_base that is not
# digest-pinned.
digest="$(docker buildx imagetools inspect "${BUILD_TAG}" | awk '$1 == "Digest:" { print $2; exit }')"
if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
echo "Could not resolve the pushed digest of ${BUILD_TAG}" >&2
exit 1
fi
# The vLLM image build uses this as its ci_base.
buildkite-agent meta-data set aiter-nightly-ci-base "${BUILD_TAG}@${digest}"
# The annotation starts with :warning: when the wheel is stale.
style=info
if [[ "${note}" == :warning:* ]]; then
style=warning
fi
echo "${note}" | buildkite-agent annotate --style "${style}" --context aiter-nightly
echo "--- :white_check_mark: ${BUILD_TAG}@${digest}"
+260
View File
@@ -0,0 +1,260 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Select the AITER nightly wheel for a vLLM ROCm image.
aiter-nightly-overlay.sh runs this inside the stock ci_base, so it reads that
image's ROCm, torch and Python. It prints the wheel's URL, then the Buildkite
annotation naming it.
"""
import argparse
import datetime
import importlib.metadata
import os
import sys
import urllib.request
from dataclasses import dataclass
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import unquote, urldefrag, urljoin
import regex as re
INDEX_URL = "https://rocm.frameworks-nightlies.amd.com/whl-multi-arch/amd-aiter/"
AITER_COMMIT_URL = "https://github.com/ROCm/aiter/commit/"
STALE_AFTER = datetime.timedelta(days=1)
# Example:
# amd_aiter-0.1.25+rocm7.2.3.torch2.12.ae984f5.d20261008-cp312-cp312-linux_x86_64.whl
# is AITER main at commit ae984f5, built 2026-10-08, for ROCm 7.2.3, torch 2.12
# and Python 3.12. 0.1.25 is a guess at the next release, not a real one.
WHEEL_NAME = re.compile(
r"^amd_aiter-(?P<base_version>[^+-]+)\+(?P<local>[^-]+)"
r"-(?P<python>cp\d+)-(?P=python)-linux_x86_64\.whl$"
)
ROCM_FIELD = re.compile(r"(?:^|\.)rocm(\d+(?:\.\d+)*)(?:a\d+)?(?=\.|$)")
TORCH_FIELD = re.compile(r"(?:^|\.)torch(\d+\.\d+)(?=\.|$)")
COMMIT_AND_DATE_FIELD = re.compile(r"(?:^|\.)([0-9a-f]{7,40})\.d(\d{8})(?=\.|$)")
@dataclass(frozen=True)
class Wheel:
"""An AITER nightly wheel, read from its filename (see WHEEL_NAME)."""
filename: str
url: str
version: str
base_version: str
rocm: str
torch: str
commit: str
build_date: datetime.date
python: str
class _LinkParser(HTMLParser):
"""Collects every link on the index page, which lists one per wheel."""
def __init__(self):
super().__init__()
self.hrefs: list[str] = []
def handle_starttag(self, tag, attrs):
if tag == "a":
self.hrefs.extend(
value for name, value in attrs if name == "href" and value
)
def wheel_from_url(url: str) -> Wheel | None:
"""The AITER nightly wheel an absolute URL points at, if it is one."""
filename = unquote(urldefrag(url).url.rsplit("/", 1)[-1])
match = WHEEL_NAME.match(filename)
if not match:
return None
local = match["local"]
rocm, torch, build = (
field.search(local)
for field in (ROCM_FIELD, TORCH_FIELD, COMMIT_AND_DATE_FIELD)
)
if not (rocm and torch and build):
return None
return Wheel(
filename=filename,
url=url,
version=f"{match['base_version']}+{local}",
base_version=match["base_version"],
rocm=rocm[1],
torch=torch[1],
commit=build[1],
build_date=datetime.datetime.strptime(build[2], "%Y%m%d").date(),
python=match["python"],
)
def parse_index(html: str, index_url: str) -> list[Wheel]:
"""Every AITER nightly wheel linked from the index page, a plain HTML list
of links, one per file (the format pip reads)."""
parser = _LinkParser()
parser.feed(html)
wheels = (wheel_from_url(urljoin(index_url, href)) for href in parser.hrefs)
return [wheel for wheel in wheels if wheel]
def rocm_version() -> str:
"""This image's ROCm version; ROCm 10+ ships as the `rocm` pip SDK."""
try:
return importlib.metadata.version("rocm")
except importlib.metadata.PackageNotFoundError:
# No pip SDK: fall back to older ROCm releases, which install to ROCM_PATH.
rocm_path = Path(os.environ.get("ROCM_PATH", "/opt/rocm"))
return (rocm_path / ".info" / "version").read_text()
def rocm_release(version: str) -> str:
"""`7.2.3-49` or `10.0.0a20261001` -> `7.2.3` or `10.0.0`, as wheel names
spell it."""
match = re.match(r"\d+(?:\.\d+)*", version.strip())
if not match:
raise ValueError(f"Unrecognised ROCm version: {version!r}")
return match.group(0)
def torch_release(version: str) -> str:
"""`2.12.0a0+git6bbd260` -> `2.12`, as wheel names spell it."""
match = re.match(r"\d+\.\d+", version)
if not match:
raise ValueError(f"Unrecognised torch version: {version!r}")
return match.group(0)
def python_tag(version_info: tuple[int, int]) -> str:
"""(3, 12) -> `cp312`, as wheel names spell it."""
return f"cp{version_info[0]}{version_info[1]}"
def _base_version_key(wheel: Wheel) -> tuple[int, ...]:
"""`0.1.25` -> (0, 1, 25), so versions sort as numbers: 0.1.10 > 0.1.9."""
return tuple(int(part) for part in re.findall(r"\d+", wheel.base_version))
def _label(rocm: str, torch: str, python: str) -> str:
"""`rocm7.2.3/torch2.12/cp312`, for messages."""
return f"rocm{rocm}/torch{torch}/{python}"
def select_wheel(wheels: list[Wheel], rocm: str, torch: str, python: str) -> Wheel:
"""Newest build for this ROCm, torch and Python, by date: version labels lag.
Two builds with the same date and version: whichever the index lists first.
"""
target = (rocm, torch, python)
matching = [w for w in wheels if (w.rocm, w.torch, w.python) == target]
if not matching:
available = sorted({_label(w.rocm, w.torch, w.python) for w in wheels})
raise LookupError(
f"No AITER nightly for {_label(*target)}. "
f"Available: {', '.join(available) or 'none'}"
)
return max(matching, key=lambda w: (w.build_date, _base_version_key(w)))
def fetch(url: str) -> str:
"""GET url as text. A failure exits 1, so Buildkite retries the step."""
with urllib.request.urlopen(url, timeout=60) as response:
return response.read().decode()
def annotation(wheel: Wheel, today: datetime.date) -> str:
"""The Buildkite annotation naming the wheel this build tests.
Starts with :warning: if the wheel is stale; the overlay script styles it so.
"""
age = today - wheel.build_date
stale = (
f":warning: **AITER nightly is {age.days} days old**: "
f"no newer build has been published for "
f"{_label(wheel.rocm, wheel.torch, wheel.python)}.\n\n"
if age > STALE_AFTER
else ""
)
return stale + (
f":crescent_moon: **AITER nightly**: main at "
f"[`{wheel.commit}`]({AITER_COMMIT_URL}{wheel.commit}), "
f"built {wheel.build_date.isoformat()} for torch {wheel.torch} "
f"(`{wheel.version}`)"
)
class NoWheel(Exception):
"""No wheel fits this image's ROCm, torch and Python. Exit 10."""
def _image_wheel(wheel_url: str) -> Wheel:
"""The wheel at wheel_url, if it fits this image's ROCm, torch and Python."""
wheel = wheel_from_url(wheel_url)
if not wheel:
raise NoWheel(f"Not an AITER nightly wheel: {wheel_url}")
return _for_image([wheel])
def _image() -> tuple[str, str, str]:
"""This image's ROCm, torch and Python, as wheel names spell them."""
try:
rocm = rocm_release(rocm_version())
except (OSError, ValueError) as error:
raise NoWheel(f"Cannot read this image's ROCm version: {error}") from error
try:
torch = torch_release(importlib.metadata.version("torch"))
except (importlib.metadata.PackageNotFoundError, ValueError) as error:
raise NoWheel(f"Cannot read this image's torch version: {error}") from error
return rocm, torch, python_tag(sys.version_info[:2])
def _for_image(wheels: list[Wheel]) -> Wheel:
"""The newest of wheels for the image this runs in."""
try:
return select_wheel(wheels, *_image())
except LookupError as error:
raise NoWheel(str(error)) from error
def select(index_url: str, wheel_url: str = "") -> Wheel:
"""The wheel to install: wheel_url on a retry, else the newest.
However old the newest is, it is tested; the annotation shows its build date.
"""
if wheel_url:
return _image_wheel(wheel_url)
return _for_image(parse_index(fetch(index_url), index_url))
def _today() -> datetime.date:
"""Today in UTC; a function so tests can set the date."""
return datetime.datetime.now(datetime.UTC).date()
# Not 1 or 2, which Buildkite retries; see aiter-nightly-overlay.sh.
EXIT_NO_WHEEL = 10
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--index-url", default=INDEX_URL)
parser.add_argument(
"--wheel-url", default="", help="a retry's earlier choice; skips the index"
)
args = parser.parse_args(argv)
try:
wheel = select(args.index_url, args.wheel_url)
except NoWheel as error:
print(error)
return EXIT_NO_WHEEL
print(wheel.url)
print(annotation(wheel, _today()))
return 0
if __name__ == "__main__":
sys.exit(main())