From 43c4bdcae9e589fde99984d1d5d03cbdeecf716b Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Tue, 4 Aug 2026 17:34:27 -0700 Subject: [PATCH] [CI] Add run-all comment commands (#51087) Signed-off-by: khluu Co-authored-by: OpenAI Codex --- .github/workflows/run-ci-command.yml | 2 + .github/workflows/scripts/run_ci_command.py | 41 +++++++++++++----- .../workflows/scripts/test_run_ci_command.py | 43 +++++++++++++++++-- 3 files changed, 72 insertions(+), 14 deletions(-) diff --git a/.github/workflows/run-ci-command.yml b/.github/workflows/run-ci-command.yml index 2c9f36812f..ac449cc680 100644 --- a/.github/workflows/run-ci-command.yml +++ b/.github/workflows/run-ci-command.yml @@ -18,6 +18,8 @@ jobs: if: >- github.event.issue.pull_request && (github.event.comment.body == '/ci run' || + github.event.comment.body == '/ci run all' || + github.event.comment.body == '/ci run nightly' || github.event.comment.body == '/ci retry') runs-on: ubuntu-latest steps: diff --git a/.github/workflows/scripts/run_ci_command.py b/.github/workflows/scripts/run_ci_command.py index 905dacc6b1..3ad3bba770 100644 --- a/.github/workflows/scripts/run_ci_command.py +++ b/.github/workflows/scripts/run_ci_command.py @@ -13,7 +13,14 @@ from collections.abc import Callable, Mapping, Sequence from typing import Any COMMAND_RUN_CI = "/ci run" +COMMAND_RUN_CI_ALL = "/ci run all" +COMMAND_RUN_CI_NIGHTLY = "/ci run nightly" COMMAND_RETRY_FAILED = "/ci retry" +RUN_CI_COMMAND_ENV = { + COMMAND_RUN_CI: {}, + COMMAND_RUN_CI_ALL: {"RUN_ALL": "1"}, + COMMAND_RUN_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"}, +} CI_AUTHORIZED_COMMENT_MARKER = "" READY_LABELS = {"ready", "ready-run-all-tests"} TRUSTED_PERMISSIONS = {"admin", "maintain", "write"} @@ -383,7 +390,7 @@ class BuildkiteClient: def parse_command(body: str) -> str | None: - if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}: + if body in {*RUN_CI_COMMAND_ENV, COMMAND_RETRY_FAILED}: return body return None @@ -486,21 +493,27 @@ def create_build_payload( *, actor: str, comment_id: int, + command: str = COMMAND_RUN_CI, pr: Mapping[str, Any], ) -> dict[str, Any]: + if command not in RUN_CI_COMMAND_ENV: + raise ValueError(f"Unsupported run command: {command}") + + env = { + "VLLM_CI_GITHUB_COMMENT_ID": str(comment_id), + "VLLM_CI_TRIGGERED_BY": actor, + **RUN_CI_COMMAND_ENV[command], + } return { "commit": pr["head"]["sha"], "branch": pr["head"]["ref"], - "message": f"PR #{pr['number']} {COMMAND_RUN_CI} by @{actor}", + "message": f"PR #{pr['number']} {command} by @{actor}", "pull_request_id": pr["number"], "pull_request_base_branch": pr["base"]["ref"], "pull_request_repository": pr["head"]["repo"]["clone_url"], "pull_request_labels": [label["name"] for label in pr["labels"]], "ignore_pipeline_branch_filters": True, - "env": { - "VLLM_CI_GITHUB_COMMENT_ID": str(comment_id), - "VLLM_CI_TRIGGERED_BY": actor, - }, + "env": env, "meta_data": { "github-comment-id": str(comment_id), "github-pr-number": str(pr["number"]), @@ -626,8 +639,12 @@ def notify_authorized( github.add_comment( pr["number"], ( - f"✅ @{author}, CI is now available for this PR. Comment `/ci run` " - "to run full CI or `/ci retry` to retry failed jobs.\n\n" + f"✅ @{author}, CI is now available for this PR.\n\n" + "- `/ci run` starts a CI build.\n" + "- `/ci retry` retries failed jobs in the CI build for the current " + "PR head. If the current head has no CI build, it starts a new CI " + "build for the current head containing only jobs that failed in " + "the latest earlier CI build for this PR.\n\n" f"{CI_AUTHORIZED_COMMENT_MARKER}" ), ) @@ -638,6 +655,7 @@ def handle_run_ci( actor: str, buildkite: BuildkiteClient, comment_id: int, + command: str, github: GitHubClient, pr: Mapping[str, Any], ) -> str: @@ -664,13 +682,15 @@ def handle_run_ci( current_pr = github.get_pr(pr["number"]) if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]: return ( - "The PR head changed while processing the command. Comment `/ci run` again." + "The PR head changed while processing the command. " + f"Comment `{command}` again." ) build = buildkite.create_build( create_build_payload( actor=actor, comment_id=comment_id, + command=command, pr=current_pr, ) ) @@ -835,11 +855,12 @@ def run( return print(f"Authorized @{actor}: {reason}") - if command == COMMAND_RUN_CI: + if command in RUN_CI_COMMAND_ENV: message = handle_run_ci( actor=actor, buildkite=buildkite, comment_id=comment_id, + command=command, github=github, pr=pr, ) diff --git a/.github/workflows/scripts/test_run_ci_command.py b/.github/workflows/scripts/test_run_ci_command.py index 7d583d470f..67fb3f58c6 100644 --- a/.github/workflows/scripts/test_run_ci_command.py +++ b/.github/workflows/scripts/test_run_ci_command.py @@ -12,6 +12,8 @@ from run_ci_command import ( CI_AUTHORIZED_COMMENT_MARKER, COMMAND_RETRY_FAILED, COMMAND_RUN_CI, + COMMAND_RUN_CI_ALL, + COMMAND_RUN_CI_NIGHTLY, RETRY_STATES, BuildkiteClient, HttpTransport, @@ -269,11 +271,17 @@ class RunCiCommandTest(unittest.TestCase): def test_only_exact_ci_commands_are_accepted(self) -> None: self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI) + self.assertEqual(parse_command(COMMAND_RUN_CI_ALL), COMMAND_RUN_CI_ALL) + self.assertEqual( + parse_command(COMMAND_RUN_CI_NIGHTLY), + COMMAND_RUN_CI_NIGHTLY, + ) self.assertEqual( parse_command(COMMAND_RETRY_FAILED), COMMAND_RETRY_FAILED, ) self.assertIsNone(parse_command("/ci run please")) + self.assertIsNone(parse_command("/ci run all please")) self.assertIsNone(parse_command(" /ci run")) def test_write_access_authorizes_reviewers_and_authors(self) -> None: @@ -435,6 +443,28 @@ class RunCiCommandTest(unittest.TestCase): self.assertTrue(github.comments[0].startswith("✅ ")) self.assertIn("Buildkite CI #123", github.comments[0]) + def test_run_all_sets_buildkite_environment(self) -> None: + github = FakeGitHub() + buildkite = FakeBuildkite([[], []]) + + run(make_event(COMMAND_RUN_CI_ALL), github, buildkite) + + payload = buildkite.created_builds[0] + self.assertEqual(payload["message"], "PR #42 /ci run all by @reviewer") + self.assertEqual(payload["env"]["RUN_ALL"], "1") + self.assertNotIn("NIGHTLY", payload["env"]) + + def test_run_nightly_sets_buildkite_environment(self) -> None: + github = FakeGitHub() + buildkite = FakeBuildkite([[], []]) + + run(make_event(COMMAND_RUN_CI_NIGHTLY), github, buildkite) + + payload = buildkite.created_builds[0] + self.assertEqual(payload["message"], "PR #42 /ci run nightly by @reviewer") + self.assertEqual(payload["env"]["RUN_ALL"], "1") + self.assertEqual(payload["env"]["NIGHTLY"], "1") + def test_unapproved_authors_are_denied_without_buildkite(self) -> None: github = FakeGitHub( permission="read", @@ -485,10 +515,15 @@ class RunCiCommandTest(unittest.TestCase): notify_authorized(event, github) self.assertEqual(len(github.comments), 1) - self.assertTrue(github.comments[0].startswith("✅ @author")) - self.assertIn("`/ci run`", github.comments[0]) - self.assertIn("`/ci retry`", github.comments[0]) - self.assertIn(CI_AUTHORIZED_COMMENT_MARKER, github.comments[0]) + comment = github.comments[0] + self.assertTrue(comment.startswith("✅ @author")) + self.assertIn("`/ci run` starts a CI build", comment) + self.assertIn("`/ci retry` retries failed jobs", comment) + self.assertIn("CI build for the current PR head", comment) + self.assertIn("only jobs that failed in the latest earlier CI build", comment) + self.assertNotIn(COMMAND_RUN_CI_ALL, comment) + self.assertNotIn(COMMAND_RUN_CI_NIGHTLY, comment) + self.assertIn(CI_AUTHORIZED_COMMENT_MARKER, comment) def test_ready_label_does_not_notify_after_trusted_approval(self) -> None: pr = make_pr(labels=[{"name": "ready"}])