[CI][ROCm] Add AITER nightly overlay scripts (#60883)
Signed-off-by: josh schell <josh.schell@amd.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: Micah Williamson <micah.williamson@amd.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
Micah Williamson
parent
e1a5086256
commit
c9e62f13c2
+112
@@ -0,0 +1,112 @@
|
||||
#!/bin/bash
|
||||
# Install the AITER nightly wheel over this build's ci_base.
|
||||
#
|
||||
# Runs in CI when AITER_NIGHTLY=1, once the stock ci_base it builds on is
|
||||
# published. In order:
|
||||
# 1. Select the newest nightly wheel that fits the stock ci_base
|
||||
# (select_aiter_nightly_wheel.py, beside this script, run inside that image)
|
||||
# 2. Build a new image: pip install that wheel over the stock ci_base
|
||||
# 3. Push the new image as this build's ci_base-build tag, replacing the stock
|
||||
# one, so later steps and native jobs use it; record its digest
|
||||
#
|
||||
# Buildkite meta-data, a key-value store shared by every job in this build:
|
||||
# reads rocm-ci-base-image the stock ci_base
|
||||
# r/w aiter-nightly-wheel-url the wheel step 1 chose; a retry reads it
|
||||
# back and installs that same wheel
|
||||
# writes aiter-nightly-ci-base the new image as tag@digest; the vLLM
|
||||
# image build uses it as its ci_base
|
||||
#
|
||||
# Exit status:
|
||||
# 10 no wheel fits this ci_base
|
||||
# 11 the wheel does not install
|
||||
# Anything else is unexpected and passed through as is.
|
||||
# Buildkite retries this step once on exit status 1 or 2, which it treats as
|
||||
# transient Docker/registry failures (amd_infra_retry in the template).
|
||||
# 10 and 11 sit outside that range: they are real AITER failures, and a retry
|
||||
# would only fail the same way.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SELECT_WHEEL="$(dirname "${BASH_SOURCE[0]}")/select_aiter_nightly_wheel.py"
|
||||
BUILD_TAG="rocm/vllm-dev:ci_base-build-${BUILDKITE_BUILD_ID:?BUILDKITE_BUILD_ID is required}"
|
||||
|
||||
# Scratch space for the build context and captured errors. The trap deletes it
|
||||
# however the script exits: success, fail, or set -e.
|
||||
scratch="$(mktemp -d)"
|
||||
trap 'rm -rf "${scratch}"' EXIT
|
||||
errors="${scratch}/errors"
|
||||
|
||||
fail() {
|
||||
local status="$1" title="$2" detail="$3"
|
||||
echo "+++ :x: AITER nightly: ${title}"
|
||||
echo "${detail}" >&2
|
||||
# shellcheck disable=SC2016 # the backticks are a literal markdown fence
|
||||
printf ':x: **AITER nightly: %s**\n\n```\n%s\n```\n' "${title}" "${detail}" \
|
||||
| buildkite-agent annotate --style error --context aiter-nightly
|
||||
exit "${status}"
|
||||
}
|
||||
|
||||
# Set by ensure-ci-base-amd; this step's depends_on guarantees it is there.
|
||||
ci_base="$(buildkite-agent meta-data get rocm-ci-base-image)"
|
||||
# Set below on the first attempt. Meta-data is per build, so a retry reuses
|
||||
# that wheel, not a newer nightly.
|
||||
pinned="$(buildkite-agent meta-data get aiter-nightly-wheel-url --default "")"
|
||||
|
||||
echo "--- :docker: Pulling ${ci_base}"
|
||||
docker pull "${ci_base}"
|
||||
|
||||
echo "--- :mag: Selecting the AITER nightly for ${ci_base}"
|
||||
set +e
|
||||
out="$(docker run --rm -i --entrypoint python3 "${ci_base}" \
|
||||
-P - ${pinned:+--wheel-url "${pinned}"} \
|
||||
< "${SELECT_WHEEL}" 2> "${errors}")"
|
||||
status=$?
|
||||
set -e
|
||||
cat "${errors}" >&2
|
||||
case "${status}" in
|
||||
0) ;;
|
||||
10) fail 10 "no wheel for this image" "${out}" ;;
|
||||
*) fail "${status}" "could not select a wheel" "$(tail -n 20 "${errors}")" ;;
|
||||
esac
|
||||
# The first line is the wheel's URL; the rest, the annotation naming it.
|
||||
wheel_url="$(head -n 1 <<< "${out}")"
|
||||
note="$(tail -n +2 <<< "${out}")"
|
||||
buildkite-agent meta-data set aiter-nightly-wheel-url "${wheel_url}"
|
||||
|
||||
cat > "${scratch}/Dockerfile" <<'EOF'
|
||||
ARG CI_BASE_IMAGE
|
||||
FROM ${CI_BASE_IMAGE}
|
||||
ARG AITER_WHEEL_URL
|
||||
RUN python3 -m pip install --no-cache-dir "$AITER_WHEEL_URL"
|
||||
EOF
|
||||
|
||||
echo "--- :docker: Installing ${wheel_url##*/}"
|
||||
if ! docker buildx build \
|
||||
--builder default \
|
||||
--provenance=false \
|
||||
--progress plain \
|
||||
--build-arg "CI_BASE_IMAGE=${ci_base}" \
|
||||
--build-arg "AITER_WHEEL_URL=${wheel_url}" \
|
||||
-t "${BUILD_TAG}" \
|
||||
"${scratch}"; then
|
||||
fail 11 "the wheel does not install" \
|
||||
"Installing ${wheel_url##*/} over ${ci_base} failed; see this step's log."
|
||||
fi
|
||||
|
||||
docker push "${BUILD_TAG}"
|
||||
# Handed off as tag@digest: build-test-image.sh rejects a ci_base that is not
|
||||
# digest-pinned.
|
||||
digest="$(docker buildx imagetools inspect "${BUILD_TAG}" | awk '$1 == "Digest:" { print $2; exit }')"
|
||||
if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
|
||||
echo "Could not resolve the pushed digest of ${BUILD_TAG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
# The vLLM image build uses this as its ci_base.
|
||||
buildkite-agent meta-data set aiter-nightly-ci-base "${BUILD_TAG}@${digest}"
|
||||
# The annotation starts with :warning: when the wheel is stale.
|
||||
style=info
|
||||
if [[ "${note}" == :warning:* ]]; then
|
||||
style=warning
|
||||
fi
|
||||
echo "${note}" | buildkite-agent annotate --style "${style}" --context aiter-nightly
|
||||
echo "--- :white_check_mark: ${BUILD_TAG}@${digest}"
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Select the AITER nightly wheel for a vLLM ROCm image.
|
||||
|
||||
aiter-nightly-overlay.sh runs this inside the stock ci_base, so it reads that
|
||||
image's ROCm, torch and Python. It prints the wheel's URL, then the Buildkite
|
||||
annotation naming it.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import importlib.metadata
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urldefrag, urljoin
|
||||
|
||||
import regex as re
|
||||
|
||||
INDEX_URL = "https://rocm.frameworks-nightlies.amd.com/whl-multi-arch/amd-aiter/"
|
||||
AITER_COMMIT_URL = "https://github.com/ROCm/aiter/commit/"
|
||||
STALE_AFTER = datetime.timedelta(days=1)
|
||||
|
||||
# Example:
|
||||
# amd_aiter-0.1.25+rocm7.2.3.torch2.12.ae984f5.d20261008-cp312-cp312-linux_x86_64.whl
|
||||
# is AITER main at commit ae984f5, built 2026-10-08, for ROCm 7.2.3, torch 2.12
|
||||
# and Python 3.12. 0.1.25 is a guess at the next release, not a real one.
|
||||
WHEEL_NAME = re.compile(
|
||||
r"^amd_aiter-(?P<base_version>[^+-]+)\+(?P<local>[^-]+)"
|
||||
r"-(?P<python>cp\d+)-(?P=python)-linux_x86_64\.whl$"
|
||||
)
|
||||
ROCM_FIELD = re.compile(r"(?:^|\.)rocm(\d+(?:\.\d+)*)(?:a\d+)?(?=\.|$)")
|
||||
TORCH_FIELD = re.compile(r"(?:^|\.)torch(\d+\.\d+)(?=\.|$)")
|
||||
COMMIT_AND_DATE_FIELD = re.compile(r"(?:^|\.)([0-9a-f]{7,40})\.d(\d{8})(?=\.|$)")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Wheel:
|
||||
"""An AITER nightly wheel, read from its filename (see WHEEL_NAME)."""
|
||||
|
||||
filename: str
|
||||
url: str
|
||||
version: str
|
||||
base_version: str
|
||||
rocm: str
|
||||
torch: str
|
||||
commit: str
|
||||
build_date: datetime.date
|
||||
python: str
|
||||
|
||||
|
||||
class _LinkParser(HTMLParser):
|
||||
"""Collects every link on the index page, which lists one per wheel."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.hrefs: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "a":
|
||||
self.hrefs.extend(
|
||||
value for name, value in attrs if name == "href" and value
|
||||
)
|
||||
|
||||
|
||||
def wheel_from_url(url: str) -> Wheel | None:
|
||||
"""The AITER nightly wheel an absolute URL points at, if it is one."""
|
||||
filename = unquote(urldefrag(url).url.rsplit("/", 1)[-1])
|
||||
match = WHEEL_NAME.match(filename)
|
||||
if not match:
|
||||
return None
|
||||
local = match["local"]
|
||||
rocm, torch, build = (
|
||||
field.search(local)
|
||||
for field in (ROCM_FIELD, TORCH_FIELD, COMMIT_AND_DATE_FIELD)
|
||||
)
|
||||
if not (rocm and torch and build):
|
||||
return None
|
||||
return Wheel(
|
||||
filename=filename,
|
||||
url=url,
|
||||
version=f"{match['base_version']}+{local}",
|
||||
base_version=match["base_version"],
|
||||
rocm=rocm[1],
|
||||
torch=torch[1],
|
||||
commit=build[1],
|
||||
build_date=datetime.datetime.strptime(build[2], "%Y%m%d").date(),
|
||||
python=match["python"],
|
||||
)
|
||||
|
||||
|
||||
def parse_index(html: str, index_url: str) -> list[Wheel]:
|
||||
"""Every AITER nightly wheel linked from the index page, a plain HTML list
|
||||
of links, one per file (the format pip reads)."""
|
||||
parser = _LinkParser()
|
||||
parser.feed(html)
|
||||
wheels = (wheel_from_url(urljoin(index_url, href)) for href in parser.hrefs)
|
||||
return [wheel for wheel in wheels if wheel]
|
||||
|
||||
|
||||
def rocm_version() -> str:
|
||||
"""This image's ROCm version; ROCm 10+ ships as the `rocm` pip SDK."""
|
||||
try:
|
||||
return importlib.metadata.version("rocm")
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
# No pip SDK: fall back to older ROCm releases, which install to ROCM_PATH.
|
||||
rocm_path = Path(os.environ.get("ROCM_PATH", "/opt/rocm"))
|
||||
return (rocm_path / ".info" / "version").read_text()
|
||||
|
||||
|
||||
def rocm_release(version: str) -> str:
|
||||
"""`7.2.3-49` or `10.0.0a20261001` -> `7.2.3` or `10.0.0`, as wheel names
|
||||
spell it."""
|
||||
match = re.match(r"\d+(?:\.\d+)*", version.strip())
|
||||
if not match:
|
||||
raise ValueError(f"Unrecognised ROCm version: {version!r}")
|
||||
return match.group(0)
|
||||
|
||||
|
||||
def torch_release(version: str) -> str:
|
||||
"""`2.12.0a0+git6bbd260` -> `2.12`, as wheel names spell it."""
|
||||
match = re.match(r"\d+\.\d+", version)
|
||||
if not match:
|
||||
raise ValueError(f"Unrecognised torch version: {version!r}")
|
||||
return match.group(0)
|
||||
|
||||
|
||||
def python_tag(version_info: tuple[int, int]) -> str:
|
||||
"""(3, 12) -> `cp312`, as wheel names spell it."""
|
||||
return f"cp{version_info[0]}{version_info[1]}"
|
||||
|
||||
|
||||
def _base_version_key(wheel: Wheel) -> tuple[int, ...]:
|
||||
"""`0.1.25` -> (0, 1, 25), so versions sort as numbers: 0.1.10 > 0.1.9."""
|
||||
return tuple(int(part) for part in re.findall(r"\d+", wheel.base_version))
|
||||
|
||||
|
||||
def _label(rocm: str, torch: str, python: str) -> str:
|
||||
"""`rocm7.2.3/torch2.12/cp312`, for messages."""
|
||||
return f"rocm{rocm}/torch{torch}/{python}"
|
||||
|
||||
|
||||
def select_wheel(wheels: list[Wheel], rocm: str, torch: str, python: str) -> Wheel:
|
||||
"""Newest build for this ROCm, torch and Python, by date: version labels lag.
|
||||
|
||||
Two builds with the same date and version: whichever the index lists first.
|
||||
"""
|
||||
target = (rocm, torch, python)
|
||||
matching = [w for w in wheels if (w.rocm, w.torch, w.python) == target]
|
||||
if not matching:
|
||||
available = sorted({_label(w.rocm, w.torch, w.python) for w in wheels})
|
||||
raise LookupError(
|
||||
f"No AITER nightly for {_label(*target)}. "
|
||||
f"Available: {', '.join(available) or 'none'}"
|
||||
)
|
||||
return max(matching, key=lambda w: (w.build_date, _base_version_key(w)))
|
||||
|
||||
|
||||
def fetch(url: str) -> str:
|
||||
"""GET url as text. A failure exits 1, so Buildkite retries the step."""
|
||||
with urllib.request.urlopen(url, timeout=60) as response:
|
||||
return response.read().decode()
|
||||
|
||||
|
||||
def annotation(wheel: Wheel, today: datetime.date) -> str:
|
||||
"""The Buildkite annotation naming the wheel this build tests.
|
||||
|
||||
Starts with :warning: if the wheel is stale; the overlay script styles it so.
|
||||
"""
|
||||
age = today - wheel.build_date
|
||||
stale = (
|
||||
f":warning: **AITER nightly is {age.days} days old**: "
|
||||
f"no newer build has been published for "
|
||||
f"{_label(wheel.rocm, wheel.torch, wheel.python)}.\n\n"
|
||||
if age > STALE_AFTER
|
||||
else ""
|
||||
)
|
||||
return stale + (
|
||||
f":crescent_moon: **AITER nightly**: main at "
|
||||
f"[`{wheel.commit}`]({AITER_COMMIT_URL}{wheel.commit}), "
|
||||
f"built {wheel.build_date.isoformat()} for torch {wheel.torch} "
|
||||
f"(`{wheel.version}`)"
|
||||
)
|
||||
|
||||
|
||||
class NoWheel(Exception):
|
||||
"""No wheel fits this image's ROCm, torch and Python. Exit 10."""
|
||||
|
||||
|
||||
def _image_wheel(wheel_url: str) -> Wheel:
|
||||
"""The wheel at wheel_url, if it fits this image's ROCm, torch and Python."""
|
||||
wheel = wheel_from_url(wheel_url)
|
||||
if not wheel:
|
||||
raise NoWheel(f"Not an AITER nightly wheel: {wheel_url}")
|
||||
return _for_image([wheel])
|
||||
|
||||
|
||||
def _image() -> tuple[str, str, str]:
|
||||
"""This image's ROCm, torch and Python, as wheel names spell them."""
|
||||
try:
|
||||
rocm = rocm_release(rocm_version())
|
||||
except (OSError, ValueError) as error:
|
||||
raise NoWheel(f"Cannot read this image's ROCm version: {error}") from error
|
||||
try:
|
||||
torch = torch_release(importlib.metadata.version("torch"))
|
||||
except (importlib.metadata.PackageNotFoundError, ValueError) as error:
|
||||
raise NoWheel(f"Cannot read this image's torch version: {error}") from error
|
||||
return rocm, torch, python_tag(sys.version_info[:2])
|
||||
|
||||
|
||||
def _for_image(wheels: list[Wheel]) -> Wheel:
|
||||
"""The newest of wheels for the image this runs in."""
|
||||
try:
|
||||
return select_wheel(wheels, *_image())
|
||||
except LookupError as error:
|
||||
raise NoWheel(str(error)) from error
|
||||
|
||||
|
||||
def select(index_url: str, wheel_url: str = "") -> Wheel:
|
||||
"""The wheel to install: wheel_url on a retry, else the newest.
|
||||
|
||||
However old the newest is, it is tested; the annotation shows its build date.
|
||||
"""
|
||||
if wheel_url:
|
||||
return _image_wheel(wheel_url)
|
||||
return _for_image(parse_index(fetch(index_url), index_url))
|
||||
|
||||
|
||||
def _today() -> datetime.date:
|
||||
"""Today in UTC; a function so tests can set the date."""
|
||||
return datetime.datetime.now(datetime.UTC).date()
|
||||
|
||||
|
||||
# Not 1 or 2, which Buildkite retries; see aiter-nightly-overlay.sh.
|
||||
EXIT_NO_WHEEL = 10
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--index-url", default=INDEX_URL)
|
||||
parser.add_argument(
|
||||
"--wheel-url", default="", help="a retry's earlier choice; skips the index"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
wheel = select(args.index_url, args.wheel_url)
|
||||
except NoWheel as error:
|
||||
print(error)
|
||||
return EXIT_NO_WHEEL
|
||||
print(wheel.url)
|
||||
print(annotation(wheel, _today()))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user