diff --git a/.github/workflows/new_pr_bot.yml b/.github/workflows/new_pr_bot.yml index 6e65ccd93a..bd74a761d6 100644 --- a/.github/workflows/new_pr_bot.yml +++ b/.github/workflows/new_pr_bot.yml @@ -80,9 +80,9 @@ jobs: '', '\u{1f4ac} Join our developer Slack at https://slack.vllm.ai to discuss your PR in `#pr-reviews`, coordinate on features in `#feat-` channels, or join special interest groups in `#sig-` channels.', '', - 'PRs do not trigger a full CI run by default. Reviewers with write access and configured trusted contributors can comment `/ci run` whenever CI signals are needed.', + 'PRs do not trigger a full CI run by default. Reviewers with write access and configured trusted contributors can comment `/ci run` for upstream CI or `/amd-ci run` for AMD CI only whenever CI signals are needed.', '', - 'Once the PR is approved or has the `ready` label, the PR author can also use `/ci run`, `/ci retry`, or `/ci cancel`. New commits do not start CI automatically.', + 'Once the PR is approved or has the `ready` label, the PR author can also use the corresponding `/ci run`, `/ci retry`, and `/ci cancel` commands, or their `/amd-ci` variants. New commits do not start upstream CI automatically.', '', 'If you have any questions, please reach out to us on Slack at https://slack.vllm.ai.', '', diff --git a/.github/workflows/run-ci-command.yml b/.github/workflows/run-ci-command.yml index 6d33add747..4df8701a15 100644 --- a/.github/workflows/run-ci-command.yml +++ b/.github/workflows/run-ci-command.yml @@ -21,7 +21,12 @@ jobs: github.event.comment.body == '/ci run all' || github.event.comment.body == '/ci run nightly' || github.event.comment.body == '/ci retry' || - github.event.comment.body == '/ci cancel') + github.event.comment.body == '/ci cancel' || + github.event.comment.body == '/amd-ci run' || + github.event.comment.body == '/amd-ci run all' || + github.event.comment.body == '/amd-ci run nightly' || + github.event.comment.body == '/amd-ci retry' || + github.event.comment.body == '/amd-ci cancel') runs-on: [self-hosted, linux, x64, vllm-runners] steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -37,6 +42,7 @@ jobs: .github/workflows/scripts/run_ci_command.py env: BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }} + BUILDKITE_AMD_PIPELINE: amd-ci BUILDKITE_ORGANIZATION: vllm BUILDKITE_PIPELINE: ci CI_TRUSTED_USERS: ${{ vars.CI_TRUSTED_USERS }} diff --git a/.github/workflows/scripts/run_ci_command.py b/.github/workflows/scripts/run_ci_command.py index 0dfa90cf3f..91c0ae5434 100644 --- a/.github/workflows/scripts/run_ci_command.py +++ b/.github/workflows/scripts/run_ci_command.py @@ -17,11 +17,40 @@ COMMAND_RUN_CI_ALL = "/ci run all" COMMAND_RUN_CI_NIGHTLY = "/ci run nightly" COMMAND_RETRY_FAILED = "/ci retry" COMMAND_CANCEL_CI = "/ci cancel" +COMMAND_RUN_AMD_CI = "/amd-ci run" +COMMAND_RUN_AMD_CI_ALL = "/amd-ci run all" +COMMAND_RUN_AMD_CI_NIGHTLY = "/amd-ci run nightly" +COMMAND_RETRY_AMD_FAILED = "/amd-ci retry" +COMMAND_CANCEL_AMD_CI = "/amd-ci cancel" RUN_CI_COMMAND_ENV = { COMMAND_RUN_CI: {}, COMMAND_RUN_CI_ALL: {"RUN_ALL": "1"}, COMMAND_RUN_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"}, + COMMAND_RUN_AMD_CI: {}, + COMMAND_RUN_AMD_CI_ALL: {"RUN_ALL": "1"}, + COMMAND_RUN_AMD_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"}, } +UPSTREAM_CI_COMMANDS = frozenset( + { + COMMAND_RUN_CI, + COMMAND_RUN_CI_ALL, + COMMAND_RUN_CI_NIGHTLY, + COMMAND_RETRY_FAILED, + COMMAND_CANCEL_CI, + } +) +AMD_CI_COMMANDS = frozenset( + { + COMMAND_RUN_AMD_CI, + COMMAND_RUN_AMD_CI_ALL, + COMMAND_RUN_AMD_CI_NIGHTLY, + COMMAND_RETRY_AMD_FAILED, + COMMAND_CANCEL_AMD_CI, + } +) +RETRY_COMMANDS = frozenset({COMMAND_RETRY_FAILED, COMMAND_RETRY_AMD_FAILED}) +CANCEL_COMMANDS = frozenset({COMMAND_CANCEL_CI, COMMAND_CANCEL_AMD_CI}) +ALL_CI_COMMANDS = UPSTREAM_CI_COMMANDS | AMD_CI_COMMANDS CI_AUTHORIZED_COMMENT_MARKER = "" READY_LABELS = {"ready", "ready-run-all-tests"} TRUSTED_PERMISSIONS = {"admin", "maintain", "write"} @@ -408,11 +437,48 @@ class BuildkiteClient: def parse_command(body: str) -> str | None: - if body in {*RUN_CI_COMMAND_ENV, COMMAND_RETRY_FAILED, COMMAND_CANCEL_CI}: + if body in ALL_CI_COMMANDS: return body return None +def pipeline_for_command( + command: str, + *, + amd_ci_pipeline: str = "amd-ci", + upstream_ci_pipeline: str = "ci", +) -> str: + if command in AMD_CI_COMMANDS: + return amd_ci_pipeline + if command in UPSTREAM_CI_COMMANDS: + return upstream_ci_pipeline + raise ValueError(f"Unsupported CI command: {command}") + + +def ci_name_for_command(command: str) -> str: + if command in AMD_CI_COMMANDS: + return "AMD CI" + if command in UPSTREAM_CI_COMMANDS: + return "CI" + raise ValueError(f"Unsupported CI command: {command}") + + +def run_command_for_command(command: str) -> str: + if command in AMD_CI_COMMANDS: + return COMMAND_RUN_AMD_CI + if command in UPSTREAM_CI_COMMANDS: + return COMMAND_RUN_CI + raise ValueError(f"Unsupported CI command: {command}") + + +def retry_command_for_command(command: str) -> str: + if command in AMD_CI_COMMANDS: + return COMMAND_RETRY_AMD_FAILED + if command in UPSTREAM_CI_COMMANDS: + return COMMAND_RETRY_FAILED + raise ValueError(f"Unsupported CI command: {command}") + + def parse_trusted_users(value: str = "") -> set[str]: return { user.casefold() for item in value.split(",") for user in item.split() if user @@ -432,6 +498,7 @@ def authorize( actor: str, permission: str, pr: Mapping[str, Any], + run_command: str = COMMAND_RUN_CI, trusted_approval: bool = False, trusted_users: set[str] | None = None, ) -> tuple[bool, str]: @@ -454,7 +521,7 @@ def authorize( return True, "approval from a trusted reviewer" return ( False, - "A reviewer with write access must run `/ci run`, approve the PR, " + f"A reviewer with write access must run `{run_command}`, approve the PR, " "or add the `ready` label first.", ) @@ -499,6 +566,21 @@ def is_active_build(build: Mapping[str, Any]) -> bool: return bool(build.get("blocked")) or build.get("state") in ACTIVE_BUILD_STATES +def is_comment_triggered_build(build: Mapping[str, Any]) -> bool: + metadata = build.get("meta_data") or {} + return bool(metadata.get("github-comment-id")) + + +def blocks_new_run(command: str, build: Mapping[str, Any]) -> bool: + if command in AMD_CI_COMMANDS: + return ( + is_comment_triggered_build(build) + and build.get("state") in ACTIVE_BUILD_STATES + and build.get("state") != "blocked" + ) + return is_active_build(build) + + def select_latest_build( builds: Sequence[dict[str, Any]], pr_number: int, @@ -544,17 +626,21 @@ def create_retry_build_payload( *, actor: str, comment_id: int, + command: str = COMMAND_RETRY_FAILED, pr: Mapping[str, Any], source_build: Mapping[str, Any], step_keys: Sequence[str], ) -> dict[str, Any]: + if command not in RETRY_COMMANDS: + raise ValueError(f"Unsupported retry command: {command}") payload = create_build_payload( actor=actor, comment_id=comment_id, + command=run_command_for_command(command), pr=pr, ) source_number = str(source_build["number"]) - payload["message"] = f"PR #{pr['number']} {COMMAND_RETRY_FAILED} by @{actor}" + payload["message"] = f"PR #{pr['number']} {command} by @{actor}" payload["env"]["VLLM_CI_ONLY_STEP_KEYS"] = json.dumps( step_keys, separators=(",", ":") ) @@ -657,13 +743,15 @@ def notify_authorized( pr["number"], ( f"✅ @{author}, CI is now available for this PR.\n\n" - "- `/ci run` starts a CI build.\n" + "- `/ci run` starts upstream CI; `/amd-ci run` starts AMD CI only.\n" "- `/ci retry` retries failed jobs in the CI build for the current " "PR head. If the current head has no CI build, it starts a new CI " "build for the current head containing only jobs that failed in " "the latest earlier CI build for this PR.\n" + "- `/amd-ci retry` retries failed jobs in AMD CI for the current PR " + "head. Use `/amd-ci run` when the current head has no AMD CI build.\n" "- `/ci cancel` cancels scheduled or running CI builds for this PR " - "branch.\n\n" + "branch; `/amd-ci cancel` does the same for AMD CI only.\n\n" f"{CI_AUTHORIZED_COMMENT_MARKER}" ), ) @@ -712,25 +800,30 @@ def handle_run_ci( github: GitHubClient, pr: Mapping[str, Any], ) -> str: + ci_name = ci_name_for_command(command) duplicate_builds = buildkite.list_builds( pr["head"]["sha"], metadata=("github-comment-id", str(comment_id)), ) duplicate = select_latest_build(duplicate_builds, pr["number"]) if duplicate: - return f"CI was already requested by this comment: {duplicate['web_url']}" + return ( + f"{ci_name} was already requested by this comment: {duplicate['web_url']}" + ) current_builds = buildkite.list_builds(pr["head"]["sha"]) active_build = next( ( build for build in current_builds - if is_build_for_pr(build, pr["number"]) and is_active_build(build) + if is_build_for_pr(build, pr["number"]) and blocks_new_run(command, build) ), None, ) if active_build: - return f"CI is already running for this commit: {active_build['web_url']}" + return ( + f"{ci_name} is already running for this commit: {active_build['web_url']}" + ) current_pr = github.get_pr(pr["number"]) if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]: @@ -748,7 +841,7 @@ def handle_run_ci( ) ) return ( - f"Triggered [Buildkite CI #{build['number']}]({build['web_url']}) " + f"Triggered [Buildkite {ci_name} #{build['number']}]({build['web_url']}) " f"for commit `{current_pr['head']['sha'][:12]}`." ) @@ -758,15 +851,21 @@ def handle_retry_failed( actor: str, buildkite: BuildkiteClient, comment_id: int, + command: str, github: GitHubClient, pr: Mapping[str, Any], ) -> str: + ci_name = ci_name_for_command(command) + run_command = run_command_for_command(command) + retry_command = retry_command_for_command(command) builds = buildkite.list_builds(pr["head"]["sha"]) build = select_latest_build(builds, pr["number"]) if build: metadata = build.get("meta_data") or {} if str(metadata.get("github-comment-id")) == str(comment_id): - return f"CI was already requested by this comment: {build['web_url']}" + return ( + f"{ci_name} was already requested by this comment: {build['web_url']}" + ) retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES) if retried["retried_jobs_count"] == 0: @@ -776,7 +875,13 @@ def handle_retry_failed( ) return ( f"Queued {retried['retried_jobs_count']} failed job(s) for retry in " - f"[Buildkite CI #{build['number']}]({build['web_url']})." + f"[Buildkite {ci_name} #{build['number']}]({build['web_url']})." + ) + + if command == COMMAND_RETRY_AMD_FAILED: + return ( + "No AMD CI build exists for the current PR head. " + f"Use `{run_command}` first." ) previous_builds = buildkite.list_builds( @@ -790,18 +895,23 @@ def handle_retry_failed( ] source_build = select_latest_build(previous_builds, pr["number"]) if not source_build: - return "No earlier CI build exists for this PR. Use `/ci run` first." + return ( + f"No earlier {ci_name} build exists for this PR. Use `{run_command}` first." + ) if not source_build.get("finished_at") or is_active_build(source_build): - return f"The previous CI build is still running: {source_build['web_url']}" + return ( + f"The previous {ci_name} build is still running: {source_build['web_url']}" + ) failed_jobs = buildkite.list_failed_jobs(source_build["number"]) failed_script_jobs = [job for job in failed_jobs if job.get("type") == "script"] missing_step_keys = [job for job in failed_script_jobs if not job.get("step_key")] if missing_step_keys: return ( - f"[Buildkite CI #{source_build['number']}]" + f"[Buildkite {ci_name} #{source_build['number']}]" f"({source_build['web_url']}) has failed jobs without stable step " - "keys, so they cannot be retried on a new commit. Use `/ci run`." + "keys, so they cannot be retried on a new commit. " + f"Use `{run_command}`." ) failed_step_keys = {str(job["step_key"]) for job in failed_script_jobs} @@ -812,16 +922,16 @@ def handle_retry_failed( ) if setup_failures: return ( - f"[Buildkite CI #{source_build['number']}]" + f"[Buildkite {ci_name} #{source_build['number']}]" f"({source_build['web_url']}) failed during CI setup, so its test " - "failure set is incomplete. Use `/ci run` for the new commit." + f"failure set is incomplete. Use `{run_command}` for the new commit." ) step_keys = sorted(failed_step_keys) if not step_keys: return ( "No failed, timed-out, or expired jobs need retrying in " - f"[Buildkite CI #{source_build['number']}]" + f"[Buildkite {ci_name} #{source_build['number']}]" f"({source_build['web_url']})." ) @@ -829,23 +939,24 @@ def handle_retry_failed( if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]: return ( "The PR head changed while processing the command. " - "Comment `/ci retry` again." + f"Comment `{retry_command}` again." ) retry_build = buildkite.create_build( create_retry_build_payload( actor=actor, comment_id=comment_id, + command=command, pr=current_pr, source_build=source_build, step_keys=step_keys, ) ) return ( - f"Triggered [Buildkite CI #{retry_build['number']}]" + f"Triggered [Buildkite {ci_name} #{retry_build['number']}]" f"({retry_build['web_url']}) for commit " f"`{current_pr['head']['sha'][:12]}`, running {len(step_keys)} failed " - f"step(s) from [Buildkite CI #{source_build['number']}]" + f"step(s) from [Buildkite {ci_name} #{source_build['number']}]" f"({source_build['web_url']})." ) @@ -853,23 +964,36 @@ def handle_retry_failed( def handle_cancel_ci( *, buildkite: BuildkiteClient, + command: str = COMMAND_CANCEL_CI, pr: Mapping[str, Any], ) -> str: + ci_name = ci_name_for_command(command) branch = pr["head"]["ref"] - builds = buildkite.list_builds( - None, - branch=branch, - states=CANCELABLE_BUILD_STATES, - ) + branches = [branch] + states = CANCELABLE_BUILD_STATES + if command in AMD_CI_COMMANDS: + head_label = pr["head"].get("label") + if head_label and head_label not in branches: + branches.append(str(head_label)) + + builds_by_number: dict[int, dict[str, Any]] = {} + for candidate_branch in branches: + for build in buildkite.list_builds( + None, + branch=candidate_branch, + states=states, + ): + builds_by_number[build["number"]] = build + cancelable_builds = [ build - for build in builds - if build.get("branch") == branch + for build in builds_by_number.values() + if build.get("branch") in branches and is_build_for_pr(build, pr["number"]) - and build.get("state") in CANCELABLE_BUILD_STATES + and build.get("state") in states ] if not cancelable_builds: - return f"No cancelable CI build is running for branch `{branch}`." + return f"No cancelable {ci_name} build is running for branch `{branch}`." for build in cancelable_builds: buildkite.cancel_build(build["number"]) @@ -879,7 +1003,9 @@ def handle_cancel_ci( ) count = len(cancelable_builds) noun = "build" if count == 1 else "builds" - return f"Requested cancellation of {count} CI {noun} for `{branch}`: {links}." + return ( + f"Requested cancellation of {count} {ci_name} {noun} for `{branch}`: {links}." + ) def run( @@ -929,6 +1055,7 @@ def run( actor=actor, permission=permission, pr=pr, + run_command=run_command_for_command(command), trusted_approval=trusted_approval, trusted_users=trusted_users, ) @@ -949,19 +1076,23 @@ def run( github=github, pr=pr, ) - elif command == COMMAND_RETRY_FAILED: + elif command in RETRY_COMMANDS: message = handle_retry_failed( actor=actor, buildkite=buildkite, comment_id=comment_id, + command=command, github=github, pr=pr, ) - else: + elif command in CANCEL_COMMANDS: message = handle_cancel_ci( buildkite=buildkite, + command=command, pr=pr, ) + else: + raise ValueError(f"Unsupported CI command: {command}") add_reaction_safely(github, comment_id, "rocket") github.add_comment(issue_number, f"✅ {message}") except Exception: @@ -1002,13 +1133,19 @@ def main() -> None: ) return - if not parse_command(event["comment"]["body"]): + command = parse_command(event["comment"]["body"]) + if not command: return + pipeline = pipeline_for_command( + command, + amd_ci_pipeline=os.environ.get("BUILDKITE_AMD_PIPELINE", "amd-ci"), + upstream_ci_pipeline=os.environ.get("BUILDKITE_PIPELINE", "ci"), + ) buildkite = BuildkiteClient( os.environ.get("BUILDKITE_API_TOKEN", ""), os.environ.get("BUILDKITE_ORGANIZATION", "vllm"), - os.environ.get("BUILDKITE_PIPELINE", "ci"), + pipeline, ) run( event, diff --git a/.github/workflows/scripts/test_run_ci_command.py b/.github/workflows/scripts/test_run_ci_command.py index f20b22525f..0a3d39eda2 100644 --- a/.github/workflows/scripts/test_run_ci_command.py +++ b/.github/workflows/scripts/test_run_ci_command.py @@ -11,8 +11,13 @@ from unittest.mock import patch from run_ci_command import ( CANCELABLE_BUILD_STATES, CI_AUTHORIZED_COMMENT_MARKER, + COMMAND_CANCEL_AMD_CI, COMMAND_CANCEL_CI, + COMMAND_RETRY_AMD_FAILED, COMMAND_RETRY_FAILED, + COMMAND_RUN_AMD_CI, + COMMAND_RUN_AMD_CI_ALL, + COMMAND_RUN_AMD_CI_NIGHTLY, COMMAND_RUN_CI, COMMAND_RUN_CI_ALL, COMMAND_RUN_CI_NIGHTLY, @@ -28,6 +33,7 @@ from run_ci_command import ( notify_authorized, parse_command, parse_trusted_users, + pipeline_for_command, resolve_workflow_run_pr, run, select_latest_build, @@ -137,6 +143,7 @@ class FakeBuildkite: self.failed_job_lists = failed_job_lists or [] self.job_list_calls: list[int] = [] self.list_calls: list[tuple[str | None, tuple[str, str] | None]] = [] + self.list_requests: list[dict[str, Any]] = [] self.retry_calls: list[tuple[int, str]] = [] self.cancel_calls: list[int] = [] @@ -149,6 +156,14 @@ class FakeBuildkite: states: tuple[str, ...] = (), ) -> list[dict[str, Any]]: self.list_calls.append((commit, metadata)) + self.list_requests.append( + { + "branch": branch, + "commit": commit, + "metadata": metadata, + "states": states, + } + ) return self.build_lists.pop(0) def create_build(self, body: dict[str, Any]) -> dict[str, Any]: @@ -291,21 +306,52 @@ class RunCiCommandTest(unittest.TestCase): self.assertEqual(urlopen.call_count, 1) def test_only_exact_ci_commands_are_accepted(self) -> None: - self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI) - self.assertEqual(parse_command(COMMAND_RUN_CI_ALL), COMMAND_RUN_CI_ALL) - self.assertEqual( - parse_command(COMMAND_RUN_CI_NIGHTLY), + commands = ( + COMMAND_RUN_CI, + COMMAND_RUN_CI_ALL, COMMAND_RUN_CI_NIGHTLY, - ) - self.assertEqual( - parse_command(COMMAND_RETRY_FAILED), COMMAND_RETRY_FAILED, + COMMAND_CANCEL_CI, + COMMAND_RUN_AMD_CI, + COMMAND_RUN_AMD_CI_ALL, + COMMAND_RUN_AMD_CI_NIGHTLY, + COMMAND_RETRY_AMD_FAILED, + COMMAND_CANCEL_AMD_CI, ) - self.assertEqual(parse_command(COMMAND_CANCEL_CI), COMMAND_CANCEL_CI) + for command in commands: + with self.subTest(command=command): + self.assertEqual(parse_command(command), command) + self.assertIsNone(parse_command("/ci run please")) self.assertIsNone(parse_command("/ci run all please")) self.assertIsNone(parse_command("/ci cancel please")) self.assertIsNone(parse_command(" /ci run")) + self.assertIsNone(parse_command("/amd-ci run please")) + self.assertIsNone(parse_command("/amd-ci retry ")) + self.assertIsNone(parse_command("/AMD-CI run")) + self.assertIsNone(parse_command("/amdci run")) + + def test_commands_select_only_their_configured_pipeline(self) -> None: + cases = ( + (COMMAND_RUN_CI, "ci"), + (COMMAND_RUN_CI_ALL, "ci"), + (COMMAND_RUN_CI_NIGHTLY, "ci"), + (COMMAND_RETRY_FAILED, "ci"), + (COMMAND_CANCEL_CI, "ci"), + (COMMAND_RUN_AMD_CI, "amd-ci"), + (COMMAND_RUN_AMD_CI_ALL, "amd-ci"), + (COMMAND_RUN_AMD_CI_NIGHTLY, "amd-ci"), + (COMMAND_RETRY_AMD_FAILED, "amd-ci"), + (COMMAND_CANCEL_AMD_CI, "amd-ci"), + ) + for command, expected_pipeline in cases: + with self.subTest(command=command): + self.assertEqual( + pipeline_for_command(command), + expected_pipeline, + ) + with self.assertRaisesRegex(ValueError, "Unsupported CI command"): + pipeline_for_command("/amd-ci run arbitrary-pipeline") def test_write_access_authorizes_reviewers_and_authors(self) -> None: allowed, _ = authorize( @@ -466,6 +512,45 @@ class RunCiCommandTest(unittest.TestCase): self.assertTrue(github.comments[0].startswith("✅ ")) self.assertIn("Buildkite CI #123", github.comments[0]) + def test_amd_run_ignores_blocked_builds(self) -> None: + for metadata in ({}, {"github-comment-id": "98"}): + with self.subTest(metadata=metadata): + github = FakeGitHub() + blocked_build = { + "blocked": True, + "created_at": "2026-08-18T01:00:00Z", + "meta_data": metadata, + "number": 122, + "pull_request": {"id": 42}, + "state": "blocked", + "web_url": "https://buildkite.example/amd-ci/builds/122", + } + buildkite = FakeBuildkite([[], [blocked_build]]) + + run(make_event(COMMAND_RUN_AMD_CI_ALL), github, buildkite) + + self.assertEqual(len(buildkite.created_builds), 1) + self.assertEqual(buildkite.created_builds[0]["env"]["RUN_ALL"], "1") + + def test_amd_run_deduplicates_comment_triggered_active_build(self) -> None: + github = FakeGitHub() + command_build = { + "blocked": False, + "created_at": "2026-08-18T01:00:00Z", + "meta_data": {"github-comment-id": "98"}, + "number": 122, + "pull_request": {"id": 42}, + "source": "api", + "state": "running", + "web_url": "https://buildkite.example/amd-ci/builds/122", + } + buildkite = FakeBuildkite([[], [command_build]]) + + run(make_event(COMMAND_RUN_AMD_CI_ALL), github, buildkite) + + self.assertEqual(buildkite.created_builds, []) + self.assertIn("AMD CI is already running", github.comments[0]) + def test_run_all_sets_buildkite_environment(self) -> None: github = FakeGitHub() buildkite = FakeBuildkite([[], []]) @@ -488,6 +573,32 @@ class RunCiCommandTest(unittest.TestCase): self.assertEqual(payload["env"]["RUN_ALL"], "1") self.assertEqual(payload["env"]["NIGHTLY"], "1") + def test_amd_run_variants_set_buildkite_environment(self) -> None: + cases = ( + (COMMAND_RUN_AMD_CI, {}), + (COMMAND_RUN_AMD_CI_ALL, {"RUN_ALL": "1"}), + ( + COMMAND_RUN_AMD_CI_NIGHTLY, + {"RUN_ALL": "1", "NIGHTLY": "1"}, + ), + ) + for command, expected_env in cases: + with self.subTest(command=command): + github = FakeGitHub() + buildkite = FakeBuildkite([[], []]) + + run(make_event(command), github, buildkite) + + payload = buildkite.created_builds[0] + self.assertEqual(payload["message"], f"PR #42 {command} by @reviewer") + command_env = { + key: value + for key, value in payload["env"].items() + if key in {"RUN_ALL", "NIGHTLY"} + } + self.assertEqual(command_env, expected_env) + self.assertIn("Buildkite AMD CI #123", github.comments[0]) + def test_unapproved_authors_are_denied_without_buildkite(self) -> None: github = FakeGitHub( permission="read", @@ -505,6 +616,20 @@ class RunCiCommandTest(unittest.TestCase): run(make_event(COMMAND_RUN_CI, "author"), github, buildkite) self.assertEqual(len(github.comments), 1) + def test_unapproved_author_gets_amd_specific_guidance(self) -> None: + github = FakeGitHub( + permission="read", + pr=make_pr(), + review_decision="REVIEW_REQUIRED", + ) + buildkite = FakeBuildkite() + + run(make_event(COMMAND_RUN_AMD_CI, "author"), github, buildkite) + + self.assertEqual(buildkite.list_calls, []) + self.assertIn("`/amd-ci run`", github.comments[0]) + self.assertNotIn("`/ci run`", github.comments[0]) + def test_untrusted_approval_cannot_launch_ci(self) -> None: github = FakeGitHub( permission="read", @@ -540,9 +665,12 @@ class RunCiCommandTest(unittest.TestCase): self.assertEqual(len(github.comments), 1) comment = github.comments[0] self.assertTrue(comment.startswith("✅ @author")) - self.assertIn("`/ci run` starts a CI build", comment) + self.assertIn("`/ci run` starts upstream CI", comment) self.assertIn("`/ci retry` retries failed jobs", comment) self.assertIn("`/ci cancel` cancels scheduled or running", comment) + self.assertIn("`/amd-ci run` starts AMD CI only", comment) + self.assertIn("`/amd-ci retry` retries failed jobs in AMD CI", comment) + self.assertIn("`/amd-ci cancel` does the same for AMD CI only", comment) self.assertIn("CI build for the current PR head", comment) self.assertIn("only jobs that failed in the latest earlier CI build", comment) self.assertNotIn(COMMAND_RUN_CI_ALL, comment) @@ -723,6 +851,47 @@ class RunCiCommandTest(unittest.TestCase): self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)]) self.assertIn("Queued 3 failed job", github.comments[0]) + def test_amd_ci_retry_retries_only_the_current_head_build(self) -> None: + github = FakeGitHub( + permission="read", + pr=make_pr(labels=[{"name": "ready"}]), + ) + buildkite = FakeBuildkite( + [ + [ + { + "created_at": "2026-07-28T01:00:00Z", + "number": 321, + "pull_request": {"id": 42}, + "state": "failing", + "web_url": "https://buildkite.example/amd-ci/builds/321", + } + ] + ] + ) + + run(make_event(COMMAND_RETRY_AMD_FAILED, "author"), github, buildkite) + + self.assertEqual(buildkite.retry_calls, [(321, RETRY_STATES)]) + self.assertIn("Buildkite AMD CI #321", github.comments[0]) + + def test_amd_ci_retry_requires_a_build_for_the_current_head(self) -> None: + github = FakeGitHub( + permission="read", + pr=make_pr(labels=[{"name": "ready"}]), + ) + buildkite = FakeBuildkite([[]]) + + run(make_event(COMMAND_RETRY_AMD_FAILED, "author"), github, buildkite) + + self.assertEqual(buildkite.list_calls, [("0123456789abcdef", None)]) + self.assertEqual(buildkite.retry_calls, []) + self.assertEqual(buildkite.created_builds, []) + self.assertIn( + "No AMD CI build exists for the current PR head", github.comments[0] + ) + self.assertIn("Use `/amd-ci run`", github.comments[0]) + def test_ci_retry_creates_filtered_build_for_new_head(self) -> None: github = FakeGitHub( permission="read", @@ -926,6 +1095,48 @@ class RunCiCommandTest(unittest.TestCase): self.assertEqual(buildkite.cancel_calls, []) self.assertIn("No cancelable CI build is running", github.comments[0]) + def test_amd_ci_cancel_handles_command_and_fork_webhook_branches(self) -> None: + pr = make_pr() + pr["head"]["label"] = "contributor:feature" + github = FakeGitHub(pr=pr) + buildkite = FakeBuildkite( + [ + [ + { + "branch": "feature", + "number": 321, + "pull_request": {"id": 42}, + "state": "running", + "web_url": "https://buildkite.example/amd-ci/builds/321", + } + ], + [ + { + "branch": "contributor:feature", + "number": 322, + "pull_request": {"id": 42}, + "state": "failing", + "web_url": "https://buildkite.example/amd-ci/builds/322", + } + ], + ] + ) + + run(make_event(COMMAND_CANCEL_AMD_CI), github, buildkite) + + self.assertEqual(buildkite.cancel_calls, [321, 322]) + self.assertEqual( + [request["branch"] for request in buildkite.list_requests], + ["feature", "contributor:feature"], + ) + self.assertTrue( + all( + request["states"] == CANCELABLE_BUILD_STATES + for request in buildkite.list_requests + ) + ) + self.assertIn("cancellation of 2 AMD CI builds", github.comments[0]) + def test_buildkite_cancel_uses_cancel_build_endpoint(self) -> None: transport = FakeTransport({"number": 123, "state": "canceling"}) client = BuildkiteClient( diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 25e7ecbdf1..3206094dee 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -303,9 +303,11 @@ review process: clarification or discuss the suggestion. - Note that not all CI checks will be executed due to limited computational resources. Reviewers with write access and configured trusted contributors - can comment `/ci run` when CI signals are needed before a PR is ready. After - the PR is approved or has the `ready` label, the PR author can use `/ci run` - or `/ci retry`. New commits do not start CI automatically. + can comment `/ci run` for upstream CI or `/amd-ci run` for AMD CI only when + CI signals are needed before a PR is ready. After the PR is approved or has + the `ready` label, the PR author can use `/ci run`, `/ci retry`, `/ci cancel`, + or the corresponding `/amd-ci` variants. New commits do not start upstream + CI automatically. ### Pull Request Limits and Escalation