iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[ROCm][CI] Add AMD CI Pull-Request Commands (#52822)

Browse Source 
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-08-18 15:16:25 -07:00
committed by GitHub
parent 5f7a20b316
commit 203926c477
5 changed files with 406 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/new_pr_bot.yml
+2 -2
View File
@@ -80,9 +80,9 @@ jobs:
'',
'\u{1f4ac} Join our developer Slack at https://slack.vllm.ai to discuss your PR in `#pr-reviews`, coordinate on features in `#feat-` channels, or join special interest groups in `#sig-` channels.',
'',
'PRs do not trigger a full CI run by default. Reviewers with write access and configured trusted contributors can comment `/ci run` whenever CI signals are needed.',
'PRs do not trigger a full CI run by default. Reviewers with write access and configured trusted contributors can comment `/ci run` for upstream CI or `/amd-ci run` for AMD CI only whenever CI signals are needed.',
'',
'Once the PR is approved or has the `ready` label, the PR author can also use `/ci run`, `/ci retry`, or `/ci cancel`. New commits do not start CI automatically.',
'Once the PR is approved or has the `ready` label, the PR author can also use the corresponding `/ci run`, `/ci retry`, and `/ci cancel` commands, or their `/amd-ci` variants. New commits do not start upstream CI automatically.',
'',
'If you have any questions, please reach out to us on Slack at https://slack.vllm.ai.',
'',
.github/workflows/run-ci-command.yml
+7 -1
View File
@@ -21,7 +21,12 @@ jobs:
github.event.comment.body == '/ci run all' ||
github.event.comment.body == '/ci run nightly' ||
github.event.comment.body == '/ci retry' ||
github.event.comment.body == '/ci cancel')
github.event.comment.body == '/ci cancel' ||
github.event.comment.body == '/amd-ci run' ||
github.event.comment.body == '/amd-ci run all' ||
github.event.comment.body == '/amd-ci run nightly' ||
github.event.comment.body == '/amd-ci retry' ||
github.event.comment.body == '/amd-ci cancel')
runs-on: [self-hosted, linux, x64, vllm-runners]
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -37,6 +42,7 @@ jobs:
.github/workflows/scripts/run_ci_command.py
env:
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
BUILDKITE_AMD_PIPELINE: amd-ci
BUILDKITE_ORGANIZATION: vllm
BUILDKITE_PIPELINE: ci
CI_TRUSTED_USERS: ${{ vars.CI_TRUSTED_USERS }}
.github/workflows/scripts/run_ci_command.py
+172 -35
View File
@@ -17,11 +17,40 @@ COMMAND_RUN_CI_ALL = "/ci run all"
COMMAND_RUN_CI_NIGHTLY = "/ci run nightly"
COMMAND_RETRY_FAILED = "/ci retry"
COMMAND_CANCEL_CI = "/ci cancel"
COMMAND_RUN_AMD_CI = "/amd-ci run"
COMMAND_RUN_AMD_CI_ALL = "/amd-ci run all"
COMMAND_RUN_AMD_CI_NIGHTLY = "/amd-ci run nightly"
COMMAND_RETRY_AMD_FAILED = "/amd-ci retry"
COMMAND_CANCEL_AMD_CI = "/amd-ci cancel"
RUN_CI_COMMAND_ENV = {
COMMAND_RUN_CI: {},
COMMAND_RUN_CI_ALL: {"RUN_ALL": "1"},
COMMAND_RUN_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"},
COMMAND_RUN_AMD_CI: {},
COMMAND_RUN_AMD_CI_ALL: {"RUN_ALL": "1"},
COMMAND_RUN_AMD_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"},
}
UPSTREAM_CI_COMMANDS = frozenset(
{
COMMAND_RUN_CI,
COMMAND_RUN_CI_ALL,
COMMAND_RUN_CI_NIGHTLY,
COMMAND_RETRY_FAILED,
COMMAND_CANCEL_CI,
}
)
AMD_CI_COMMANDS = frozenset(
{
COMMAND_RUN_AMD_CI,
COMMAND_RUN_AMD_CI_ALL,
COMMAND_RUN_AMD_CI_NIGHTLY,
COMMAND_RETRY_AMD_FAILED,
COMMAND_CANCEL_AMD_CI,
}
)
RETRY_COMMANDS = frozenset({COMMAND_RETRY_FAILED, COMMAND_RETRY_AMD_FAILED})
CANCEL_COMMANDS = frozenset({COMMAND_CANCEL_CI, COMMAND_CANCEL_AMD_CI})
ALL_CI_COMMANDS = UPSTREAM_CI_COMMANDS | AMD_CI_COMMANDS
CI_AUTHORIZED_COMMENT_MARKER = "<!-- vllm-ci-authorized -->"
READY_LABELS = {"ready", "ready-run-all-tests"}
TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
@@ -408,11 +437,48 @@ class BuildkiteClient:
def parse_command(body: str) -> str | None:
if body in {*RUN_CI_COMMAND_ENV, COMMAND_RETRY_FAILED, COMMAND_CANCEL_CI}:
if body in ALL_CI_COMMANDS:
return body
return None
def pipeline_for_command(
command: str,
*,
amd_ci_pipeline: str = "amd-ci",
upstream_ci_pipeline: str = "ci",
) -> str:
if command in AMD_CI_COMMANDS:
return amd_ci_pipeline
if command in UPSTREAM_CI_COMMANDS:
return upstream_ci_pipeline
raise ValueError(f"Unsupported CI command: {command}")
def ci_name_for_command(command: str) -> str:
if command in AMD_CI_COMMANDS:
return "AMD CI"
if command in UPSTREAM_CI_COMMANDS:
return "CI"
raise ValueError(f"Unsupported CI command: {command}")
def run_command_for_command(command: str) -> str:
if command in AMD_CI_COMMANDS:
return COMMAND_RUN_AMD_CI
if command in UPSTREAM_CI_COMMANDS:
return COMMAND_RUN_CI
raise ValueError(f"Unsupported CI command: {command}")
def retry_command_for_command(command: str) -> str:
if command in AMD_CI_COMMANDS:
return COMMAND_RETRY_AMD_FAILED
if command in UPSTREAM_CI_COMMANDS:
return COMMAND_RETRY_FAILED
raise ValueError(f"Unsupported CI command: {command}")
def parse_trusted_users(value: str = "") -> set[str]:
return {
user.casefold() for item in value.split(",") for user in item.split() if user
@@ -432,6 +498,7 @@ def authorize(
actor: str,
permission: str,
pr: Mapping[str, Any],
run_command: str = COMMAND_RUN_CI,
trusted_approval: bool = False,
trusted_users: set[str] | None = None,
) -> tuple[bool, str]:
@@ -454,7 +521,7 @@ def authorize(
return True, "approval from a trusted reviewer"
return (
False,
"A reviewer with write access must run `/ci run`, approve the PR, "
f"A reviewer with write access must run `{run_command}`, approve the PR, "
"or add the `ready` label first.",
)
@@ -499,6 +566,21 @@ def is_active_build(build: Mapping[str, Any]) -> bool:
return bool(build.get("blocked")) or build.get("state") in ACTIVE_BUILD_STATES
def is_comment_triggered_build(build: Mapping[str, Any]) -> bool:
metadata = build.get("meta_data") or {}
return bool(metadata.get("github-comment-id"))
def blocks_new_run(command: str, build: Mapping[str, Any]) -> bool:
if command in AMD_CI_COMMANDS:
return (
is_comment_triggered_build(build)
and build.get("state") in ACTIVE_BUILD_STATES
and build.get("state") != "blocked"
)
return is_active_build(build)
def select_latest_build(
builds: Sequence[dict[str, Any]],
pr_number: int,
@@ -544,17 +626,21 @@ def create_retry_build_payload(
*,
actor: str,
comment_id: int,
command: str = COMMAND_RETRY_FAILED,
pr: Mapping[str, Any],
source_build: Mapping[str, Any],
step_keys: Sequence[str],
) -> dict[str, Any]:
if command not in RETRY_COMMANDS:
raise ValueError(f"Unsupported retry command: {command}")
payload = create_build_payload(
actor=actor,
comment_id=comment_id,
command=run_command_for_command(command),
pr=pr,
)
source_number = str(source_build["number"])
payload["message"] = f"PR #{pr['number']} {COMMAND_RETRY_FAILED} by @{actor}"
payload["message"] = f"PR #{pr['number']} {command} by @{actor}"
payload["env"]["VLLM_CI_ONLY_STEP_KEYS"] = json.dumps(
step_keys, separators=(",", ":")
)
@@ -657,13 +743,15 @@ def notify_authorized(
pr["number"],
(
f"✅ @{author}, CI is now available for this PR.\n\n"
"- `/ci run` starts a CI build.\n"
"- `/ci run` starts upstream CI; `/amd-ci run` starts AMD CI only.\n"
"- `/ci retry` retries failed jobs in the CI build for the current "
"PR head. If the current head has no CI build, it starts a new CI "
"build for the current head containing only jobs that failed in "
"the latest earlier CI build for this PR.\n"
"- `/amd-ci retry` retries failed jobs in AMD CI for the current PR "
"head. Use `/amd-ci run` when the current head has no AMD CI build.\n"
"- `/ci cancel` cancels scheduled or running CI builds for this PR "
"branch.\n\n"
"branch; `/amd-ci cancel` does the same for AMD CI only.\n\n"
f"{CI_AUTHORIZED_COMMENT_MARKER}"
),
)
@@ -712,25 +800,30 @@ def handle_run_ci(
github: GitHubClient,
pr: Mapping[str, Any],
) -> str:
ci_name = ci_name_for_command(command)
duplicate_builds = buildkite.list_builds(
pr["head"]["sha"],
metadata=("github-comment-id", str(comment_id)),
)
duplicate = select_latest_build(duplicate_builds, pr["number"])
if duplicate:
return f"CI was already requested by this comment: {duplicate['web_url']}"
return (
f"{ci_name} was already requested by this comment: {duplicate['web_url']}"
)
current_builds = buildkite.list_builds(pr["head"]["sha"])
active_build = next(
(
build
for build in current_builds
if is_build_for_pr(build, pr["number"]) and is_active_build(build)
if is_build_for_pr(build, pr["number"]) and blocks_new_run(command, build)
),
None,
)
if active_build:
return f"CI is already running for this commit: {active_build['web_url']}"
return (
f"{ci_name} is already running for this commit: {active_build['web_url']}"
)
current_pr = github.get_pr(pr["number"])
if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
@@ -748,7 +841,7 @@ def handle_run_ci(
)
)
return (
f"Triggered [Buildkite CI #{build['number']}]({build['web_url']}) "
f"Triggered [Buildkite {ci_name} #{build['number']}]({build['web_url']}) "
f"for commit `{current_pr['head']['sha'][:12]}`."
)
@@ -758,15 +851,21 @@ def handle_retry_failed(
actor: str,
buildkite: BuildkiteClient,
comment_id: int,
command: str,
github: GitHubClient,
pr: Mapping[str, Any],
) -> str:
ci_name = ci_name_for_command(command)
run_command = run_command_for_command(command)
retry_command = retry_command_for_command(command)
builds = buildkite.list_builds(pr["head"]["sha"])
build = select_latest_build(builds, pr["number"])
if build:
metadata = build.get("meta_data") or {}
if str(metadata.get("github-comment-id")) == str(comment_id):
return f"CI was already requested by this comment: {build['web_url']}"
return (
f"{ci_name} was already requested by this comment: {build['web_url']}"
)
retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES)
if retried["retried_jobs_count"] == 0:
@@ -776,7 +875,13 @@ def handle_retry_failed(
)
return (
f"Queued {retried['retried_jobs_count']} failed job(s) for retry in "
f"[Buildkite CI #{build['number']}]({build['web_url']})."
f"[Buildkite {ci_name} #{build['number']}]({build['web_url']})."
)
if command == COMMAND_RETRY_AMD_FAILED:
return (
"No AMD CI build exists for the current PR head. "
f"Use `{run_command}` first."
)
previous_builds = buildkite.list_builds(
@@ -790,18 +895,23 @@ def handle_retry_failed(
]
source_build = select_latest_build(previous_builds, pr["number"])
if not source_build:
return "No earlier CI build exists for this PR. Use `/ci run` first."
return (
f"No earlier {ci_name} build exists for this PR. Use `{run_command}` first."
)
if not source_build.get("finished_at") or is_active_build(source_build):
return f"The previous CI build is still running: {source_build['web_url']}"
return (
f"The previous {ci_name} build is still running: {source_build['web_url']}"
)
failed_jobs = buildkite.list_failed_jobs(source_build["number"])
failed_script_jobs = [job for job in failed_jobs if job.get("type") == "script"]
missing_step_keys = [job for job in failed_script_jobs if not job.get("step_key")]
if missing_step_keys:
return (
f"[Buildkite CI #{source_build['number']}]"
f"[Buildkite {ci_name} #{source_build['number']}]"
f"({source_build['web_url']}) has failed jobs without stable step "
"keys, so they cannot be retried on a new commit. Use `/ci run`."
"keys, so they cannot be retried on a new commit. "
f"Use `{run_command}`."
)
failed_step_keys = {str(job["step_key"]) for job in failed_script_jobs}
@@ -812,16 +922,16 @@ def handle_retry_failed(
)
if setup_failures:
return (
f"[Buildkite CI #{source_build['number']}]"
f"[Buildkite {ci_name} #{source_build['number']}]"
f"({source_build['web_url']}) failed during CI setup, so its test "
"failure set is incomplete. Use `/ci run` for the new commit."
f"failure set is incomplete. Use `{run_command}` for the new commit."
)
step_keys = sorted(failed_step_keys)
if not step_keys:
return (
"No failed, timed-out, or expired jobs need retrying in "
f"[Buildkite CI #{source_build['number']}]"
f"[Buildkite {ci_name} #{source_build['number']}]"
f"({source_build['web_url']})."
)
@@ -829,23 +939,24 @@ def handle_retry_failed(
if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
return (
"The PR head changed while processing the command. "
"Comment `/ci retry` again."
f"Comment `{retry_command}` again."
)
retry_build = buildkite.create_build(
create_retry_build_payload(
actor=actor,
comment_id=comment_id,
command=command,
pr=current_pr,
source_build=source_build,
step_keys=step_keys,
)
)
return (
f"Triggered [Buildkite CI #{retry_build['number']}]"
f"Triggered [Buildkite {ci_name} #{retry_build['number']}]"
f"({retry_build['web_url']}) for commit "
f"`{current_pr['head']['sha'][:12]}`, running {len(step_keys)} failed "
f"step(s) from [Buildkite CI #{source_build['number']}]"
f"step(s) from [Buildkite {ci_name} #{source_build['number']}]"
f"({source_build['web_url']})."
)
@@ -853,23 +964,36 @@ def handle_retry_failed(
def handle_cancel_ci(
*,
buildkite: BuildkiteClient,
command: str = COMMAND_CANCEL_CI,
pr: Mapping[str, Any],
) -> str:
ci_name = ci_name_for_command(command)
branch = pr["head"]["ref"]
builds = buildkite.list_builds(
None,
branch=branch,
states=CANCELABLE_BUILD_STATES,
)
branches = [branch]
states = CANCELABLE_BUILD_STATES
if command in AMD_CI_COMMANDS:
head_label = pr["head"].get("label")
if head_label and head_label not in branches:
branches.append(str(head_label))
builds_by_number: dict[int, dict[str, Any]] = {}
for candidate_branch in branches:
for build in buildkite.list_builds(
None,
branch=candidate_branch,
states=states,
):
builds_by_number[build["number"]] = build
cancelable_builds = [
build
for build in builds
if build.get("branch") == branch
for build in builds_by_number.values()
if build.get("branch") in branches
and is_build_for_pr(build, pr["number"])
and build.get("state") in CANCELABLE_BUILD_STATES
and build.get("state") in states
]
if not cancelable_builds:
return f"No cancelable CI build is running for branch `{branch}`."
return f"No cancelable {ci_name} build is running for branch `{branch}`."
for build in cancelable_builds:
buildkite.cancel_build(build["number"])
@@ -879,7 +1003,9 @@ def handle_cancel_ci(
)
count = len(cancelable_builds)
noun = "build" if count == 1 else "builds"
return f"Requested cancellation of {count} CI {noun} for `{branch}`: {links}."
return (
f"Requested cancellation of {count} {ci_name} {noun} for `{branch}`: {links}."
)
def run(
@@ -929,6 +1055,7 @@ def run(
actor=actor,
permission=permission,
pr=pr,
run_command=run_command_for_command(command),
trusted_approval=trusted_approval,
trusted_users=trusted_users,
)
@@ -949,19 +1076,23 @@ def run(
github=github,
pr=pr,
)
elif command == COMMAND_RETRY_FAILED:
elif command in RETRY_COMMANDS:
message = handle_retry_failed(
actor=actor,
buildkite=buildkite,
comment_id=comment_id,
command=command,
github=github,
pr=pr,
)
else:
elif command in CANCEL_COMMANDS:
message = handle_cancel_ci(
buildkite=buildkite,
command=command,
pr=pr,
)
else:
raise ValueError(f"Unsupported CI command: {command}")
add_reaction_safely(github, comment_id, "rocket")
github.add_comment(issue_number, f"✅ {message}")
except Exception:
@@ -1002,13 +1133,19 @@ def main() -> None:
)
return
if not parse_command(event["comment"]["body"]):
command = parse_command(event["comment"]["body"])
if not command:
return
pipeline = pipeline_for_command(
command,
amd_ci_pipeline=os.environ.get("BUILDKITE_AMD_PIPELINE", "amd-ci"),
upstream_ci_pipeline=os.environ.get("BUILDKITE_PIPELINE", "ci"),
)
buildkite = BuildkiteClient(
os.environ.get("BUILDKITE_API_TOKEN", ""),
os.environ.get("BUILDKITE_ORGANIZATION", "vllm"),
os.environ.get("BUILDKITE_PIPELINE", "ci"),
pipeline,
)
run(
event,
.github/workflows/scripts/test_run_ci_command.py
+220 -9
View File
@@ -11,8 +11,13 @@ from unittest.mock import patch
from run_ci_command import (
CANCELABLE_BUILD_STATES,
CI_AUTHORIZED_COMMENT_MARKER,
COMMAND_CANCEL_AMD_CI,
COMMAND_CANCEL_CI,
COMMAND_RETRY_AMD_FAILED,
COMMAND_RETRY_FAILED,
COMMAND_RUN_AMD_CI,
COMMAND_RUN_AMD_CI_ALL,
COMMAND_RUN_AMD_CI_NIGHTLY,
COMMAND_RUN_CI,
COMMAND_RUN_CI_ALL,
COMMAND_RUN_CI_NIGHTLY,
@@ -28,6 +33,7 @@ from run_ci_command import (
notify_authorized,
parse_command,
parse_trusted_users,
pipeline_for_command,
resolve_workflow_run_pr,
run,
select_latest_build,
@@ -137,6 +143,7 @@ class FakeBuildkite:
self.failed_job_lists = failed_job_lists or []
self.job_list_calls: list[int] = []
self.list_calls: list[tuple[str | None, tuple[str, str] | None]] = []
self.list_requests: list[dict[str, Any]] = []
self.retry_calls: list[tuple[int, str]] = []
self.cancel_calls: list[int] = []
@@ -149,6 +156,14 @@ class FakeBuildkite:
states: tuple[str, ...] = (),
) -> list[dict[str, Any]]:
self.list_calls.append((commit, metadata))
self.list_requests.append(
{
"branch": branch,
"commit": commit,
"metadata": metadata,
"states": states,
}
)
return self.build_lists.pop(0)
def create_build(self, body: dict[str, Any]) -> dict[str, Any]:
@@ -291,21 +306,52 @@ class RunCiCommandTest(unittest.TestCase):
self.assertEqual(urlopen.call_count, 1)
def test_only_exact_ci_commands_are_accepted(self) -> None:
self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
self.assertEqual(parse_command(COMMAND_RUN_CI_ALL), COMMAND_RUN_CI_ALL)
self.assertEqual(
parse_command(COMMAND_RUN_CI_NIGHTLY),
commands = (
COMMAND_RUN_CI,
COMMAND_RUN_CI_ALL,
COMMAND_RUN_CI_NIGHTLY,
)
self.assertEqual(
parse_command(COMMAND_RETRY_FAILED),
COMMAND_RETRY_FAILED,
COMMAND_CANCEL_CI,
COMMAND_RUN_AMD_CI,
COMMAND_RUN_AMD_CI_ALL,
COMMAND_RUN_AMD_CI_NIGHTLY,
COMMAND_RETRY_AMD_FAILED,
COMMAND_CANCEL_AMD_CI,
)
self.assertEqual(parse_command(COMMAND_CANCEL_CI), COMMAND_CANCEL_CI)
for command in commands:
with self.subTest(command=command):
self.assertEqual(parse_command(command), command)
self.assertIsNone(parse_command("/ci run please"))
self.assertIsNone(parse_command("/ci run all please"))
self.assertIsNone(parse_command("/ci cancel please"))
self.assertIsNone(parse_command(" /ci run"))
self.assertIsNone(parse_command("/amd-ci run please"))
self.assertIsNone(parse_command("/amd-ci retry "))
self.assertIsNone(parse_command("/AMD-CI run"))
self.assertIsNone(parse_command("/amdci run"))
def test_commands_select_only_their_configured_pipeline(self) -> None:
cases = (
(COMMAND_RUN_CI, "ci"),
(COMMAND_RUN_CI_ALL, "ci"),
(COMMAND_RUN_CI_NIGHTLY, "ci"),
(COMMAND_RETRY_FAILED, "ci"),
(COMMAND_CANCEL_CI, "ci"),
(COMMAND_RUN_AMD_CI, "amd-ci"),
(COMMAND_RUN_AMD_CI_ALL, "amd-ci"),
(COMMAND_RUN_AMD_CI_NIGHTLY, "amd-ci"),
(COMMAND_RETRY_AMD_FAILED, "amd-ci"),
(COMMAND_CANCEL_AMD_CI, "amd-ci"),
)
for command, expected_pipeline in cases:
with self.subTest(command=command):
self.assertEqual(
pipeline_for_command(command),
expected_pipeline,
)
with self.assertRaisesRegex(ValueError, "Unsupported CI command"):
pipeline_for_command("/amd-ci run arbitrary-pipeline")
def test_write_access_authorizes_reviewers_and_authors(self) -> None:
allowed, _ = authorize(
@@ -466,6 +512,45 @@ class RunCiCommandTest(unittest.TestCase):
self.assertTrue(github.comments[0].startswith("✅ "))
self.assertIn("Buildkite CI #123", github.comments[0])
def test_amd_run_ignores_blocked_builds(self) -> None:
for metadata in ({}, {"github-comment-id": "98"}):
with self.subTest(metadata=metadata):
github = FakeGitHub()
blocked_build = {
"blocked": True,
"created_at": "2026-08-18T01:00:00Z",
"meta_data": metadata,
"number": 122,
"pull_request": {"id": 42},
"state": "blocked",
"web_url": "https://buildkite.example/amd-ci/builds/122",
}
buildkite = FakeBuildkite([[], [blocked_build]])
run(make_event(COMMAND_RUN_AMD_CI_ALL), github, buildkite)
self.assertEqual(len(buildkite.created_builds), 1)
self.assertEqual(buildkite.created_builds[0]["env"]["RUN_ALL"], "1")
def test_amd_run_deduplicates_comment_triggered_active_build(self) -> None:
github = FakeGitHub()
command_build = {
"blocked": False,
"created_at": "2026-08-18T01:00:00Z",
"meta_data": {"github-comment-id": "98"},
"number": 122,
"pull_request": {"id": 42},
"source": "api",
"state": "running",
"web_url": "https://buildkite.example/amd-ci/builds/122",
}
buildkite = FakeBuildkite([[], [command_build]])
run(make_event(COMMAND_RUN_AMD_CI_ALL), github, buildkite)
self.assertEqual(buildkite.created_builds, [])
self.assertIn("AMD CI is already running", github.comments[0])
def test_run_all_sets_buildkite_environment(self) -> None:
github = FakeGitHub()
buildkite = FakeBuildkite([[], []])
@@ -488,6 +573,32 @@ class RunCiCommandTest(unittest.TestCase):
self.assertEqual(payload["env"]["RUN_ALL"], "1")
self.assertEqual(payload["env"]["NIGHTLY"], "1")
def test_amd_run_variants_set_buildkite_environment(self) -> None:
cases = (
(COMMAND_RUN_AMD_CI, {}),
(COMMAND_RUN_AMD_CI_ALL, {"RUN_ALL": "1"}),
(
COMMAND_RUN_AMD_CI_NIGHTLY,
{"RUN_ALL": "1", "NIGHTLY": "1"},
),
)
for command, expected_env in cases:
with self.subTest(command=command):
github = FakeGitHub()
buildkite = FakeBuildkite([[], []])
run(make_event(command), github, buildkite)
payload = buildkite.created_builds[0]
self.assertEqual(payload["message"], f"PR #42 {command} by @reviewer")
command_env = {
key: value
for key, value in payload["env"].items()
if key in {"RUN_ALL", "NIGHTLY"}
}
self.assertEqual(command_env, expected_env)
self.assertIn("Buildkite AMD CI #123", github.comments[0])
def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
github = FakeGitHub(
permission="read",
@@ -505,6 +616,20 @@ class RunCiCommandTest(unittest.TestCase):
run(make_event(COMMAND_RUN_CI, "author"), github, buildkite)
self.assertEqual(len(github.comments), 1)
def test_unapproved_author_gets_amd_specific_guidance(self) -> None:
github = FakeGitHub(
permission="read",
pr=make_pr(),
review_decision="REVIEW_REQUIRED",
)
buildkite = FakeBuildkite()
run(make_event(COMMAND_RUN_AMD_CI, "author"), github, buildkite)
self.assertEqual(buildkite.list_calls, [])
self.assertIn("`/amd-ci run`", github.comments[0])
self.assertNotIn("`/ci run`", github.comments[0])
def test_untrusted_approval_cannot_launch_ci(self) -> None:
github = FakeGitHub(
permission="read",
@@ -540,9 +665,12 @@ class RunCiCommandTest(unittest.TestCase):
self.assertEqual(len(github.comments), 1)
comment = github.comments[0]
self.assertTrue(comment.startswith("✅ @author"))
self.assertIn("`/ci run` starts a CI build", comment)
self.assertIn("`/ci run` starts upstream CI", comment)
self.assertIn("`/ci retry` retries failed jobs", comment)
self.assertIn("`/ci cancel` cancels scheduled or running", comment)
self.assertIn("`/amd-ci run` starts AMD CI only", comment)
self.assertIn("`/amd-ci retry` retries failed jobs in AMD CI", comment)
self.assertIn("`/amd-ci cancel` does the same for AMD CI only", comment)
self.assertIn("CI build for the current PR head", comment)
self.assertIn("only jobs that failed in the latest earlier CI build", comment)
self.assertNotIn(COMMAND_RUN_CI_ALL, comment)
@@ -723,6 +851,47 @@ class RunCiCommandTest(unittest.TestCase):
self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)])
self.assertIn("Queued 3 failed job", github.comments[0])
def test_amd_ci_retry_retries_only_the_current_head_build(self) -> None:
github = FakeGitHub(
permission="read",
pr=make_pr(labels=[{"name": "ready"}]),
)
buildkite = FakeBuildkite(
[
[
{
"created_at": "2026-07-28T01:00:00Z",
"number": 321,
"pull_request": {"id": 42},
"state": "failing",
"web_url": "https://buildkite.example/amd-ci/builds/321",
}
]
]
)
run(make_event(COMMAND_RETRY_AMD_FAILED, "author"), github, buildkite)
self.assertEqual(buildkite.retry_calls, [(321, RETRY_STATES)])
self.assertIn("Buildkite AMD CI #321", github.comments[0])
def test_amd_ci_retry_requires_a_build_for_the_current_head(self) -> None:
github = FakeGitHub(
permission="read",
pr=make_pr(labels=[{"name": "ready"}]),
)
buildkite = FakeBuildkite([[]])
run(make_event(COMMAND_RETRY_AMD_FAILED, "author"), github, buildkite)
self.assertEqual(buildkite.list_calls, [("0123456789abcdef", None)])
self.assertEqual(buildkite.retry_calls, [])
self.assertEqual(buildkite.created_builds, [])
self.assertIn(
"No AMD CI build exists for the current PR head", github.comments[0]
)
self.assertIn("Use `/amd-ci run`", github.comments[0])
def test_ci_retry_creates_filtered_build_for_new_head(self) -> None:
github = FakeGitHub(
permission="read",
@@ -926,6 +1095,48 @@ class RunCiCommandTest(unittest.TestCase):
self.assertEqual(buildkite.cancel_calls, [])
self.assertIn("No cancelable CI build is running", github.comments[0])
def test_amd_ci_cancel_handles_command_and_fork_webhook_branches(self) -> None:
pr = make_pr()
pr["head"]["label"] = "contributor:feature"
github = FakeGitHub(pr=pr)
buildkite = FakeBuildkite(
[
[
{
"branch": "feature",
"number": 321,
"pull_request": {"id": 42},
"state": "running",
"web_url": "https://buildkite.example/amd-ci/builds/321",
}
],
[
{
"branch": "contributor:feature",
"number": 322,
"pull_request": {"id": 42},
"state": "failing",
"web_url": "https://buildkite.example/amd-ci/builds/322",
}
],
]
)
run(make_event(COMMAND_CANCEL_AMD_CI), github, buildkite)
self.assertEqual(buildkite.cancel_calls, [321, 322])
self.assertEqual(
[request["branch"] for request in buildkite.list_requests],
["feature", "contributor:feature"],
)
self.assertTrue(
all(
request["states"] == CANCELABLE_BUILD_STATES
for request in buildkite.list_requests
)
)
self.assertIn("cancellation of 2 AMD CI builds", github.comments[0])
def test_buildkite_cancel_uses_cancel_build_endpoint(self) -> None:
transport = FakeTransport({"number": 123, "state": "canceling"})
client = BuildkiteClient(
docs/contributing/README.md
+5 -3
View File
@@ -303,9 +303,11 @@ review process:
clarification or discuss the suggestion.
- Note that not all CI checks will be executed due to limited computational
resources. Reviewers with write access and configured trusted contributors
can comment `/ci run` when CI signals are needed before a PR is ready. After
the PR is approved or has the `ready` label, the PR author can use `/ci run`
or `/ci retry`. New commits do not start CI automatically.
can comment `/ci run` for upstream CI or `/amd-ci run` for AMD CI only when
CI signals are needed before a PR is ready. After the PR is approved or has
the `ready` label, the PR author can use `/ci run`, `/ci retry`, `/ci cancel`,
or the corresponding `/amd-ci` variants. New commits do not start upstream
CI automatically.
### Pull Request Limits and Escalation