[CI] Add run-all comment commands (#51087)

Signed-off-by: khluu <khluu000@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Kevin H. Luu
2026-08-04 17:34:27 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent ffee32460d
commit 43c4bdcae9
3 changed files with 72 additions and 14 deletions
+2
View File
@@ -18,6 +18,8 @@ jobs:
if: >- if: >-
github.event.issue.pull_request && github.event.issue.pull_request &&
(github.event.comment.body == '/ci run' || (github.event.comment.body == '/ci run' ||
github.event.comment.body == '/ci run all' ||
github.event.comment.body == '/ci run nightly' ||
github.event.comment.body == '/ci retry') github.event.comment.body == '/ci retry')
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
+31 -10
View File
@@ -13,7 +13,14 @@ from collections.abc import Callable, Mapping, Sequence
from typing import Any from typing import Any
COMMAND_RUN_CI = "/ci run" COMMAND_RUN_CI = "/ci run"
COMMAND_RUN_CI_ALL = "/ci run all"
COMMAND_RUN_CI_NIGHTLY = "/ci run nightly"
COMMAND_RETRY_FAILED = "/ci retry" COMMAND_RETRY_FAILED = "/ci retry"
RUN_CI_COMMAND_ENV = {
COMMAND_RUN_CI: {},
COMMAND_RUN_CI_ALL: {"RUN_ALL": "1"},
COMMAND_RUN_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"},
}
CI_AUTHORIZED_COMMENT_MARKER = "<!-- vllm-ci-authorized -->" CI_AUTHORIZED_COMMENT_MARKER = "<!-- vllm-ci-authorized -->"
READY_LABELS = {"ready", "ready-run-all-tests"} READY_LABELS = {"ready", "ready-run-all-tests"}
TRUSTED_PERMISSIONS = {"admin", "maintain", "write"} TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
@@ -383,7 +390,7 @@ class BuildkiteClient:
def parse_command(body: str) -> str | None: def parse_command(body: str) -> str | None:
if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}: if body in {*RUN_CI_COMMAND_ENV, COMMAND_RETRY_FAILED}:
return body return body
return None return None
@@ -486,21 +493,27 @@ def create_build_payload(
*, *,
actor: str, actor: str,
comment_id: int, comment_id: int,
command: str = COMMAND_RUN_CI,
pr: Mapping[str, Any], pr: Mapping[str, Any],
) -> dict[str, Any]: ) -> dict[str, Any]:
if command not in RUN_CI_COMMAND_ENV:
raise ValueError(f"Unsupported run command: {command}")
env = {
"VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
"VLLM_CI_TRIGGERED_BY": actor,
**RUN_CI_COMMAND_ENV[command],
}
return { return {
"commit": pr["head"]["sha"], "commit": pr["head"]["sha"],
"branch": pr["head"]["ref"], "branch": pr["head"]["ref"],
"message": f"PR #{pr['number']} {COMMAND_RUN_CI} by @{actor}", "message": f"PR #{pr['number']} {command} by @{actor}",
"pull_request_id": pr["number"], "pull_request_id": pr["number"],
"pull_request_base_branch": pr["base"]["ref"], "pull_request_base_branch": pr["base"]["ref"],
"pull_request_repository": pr["head"]["repo"]["clone_url"], "pull_request_repository": pr["head"]["repo"]["clone_url"],
"pull_request_labels": [label["name"] for label in pr["labels"]], "pull_request_labels": [label["name"] for label in pr["labels"]],
"ignore_pipeline_branch_filters": True, "ignore_pipeline_branch_filters": True,
"env": { "env": env,
"VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
"VLLM_CI_TRIGGERED_BY": actor,
},
"meta_data": { "meta_data": {
"github-comment-id": str(comment_id), "github-comment-id": str(comment_id),
"github-pr-number": str(pr["number"]), "github-pr-number": str(pr["number"]),
@@ -626,8 +639,12 @@ def notify_authorized(
github.add_comment( github.add_comment(
pr["number"], pr["number"],
( (
f"✅ @{author}, CI is now available for this PR. Comment `/ci run` " f"✅ @{author}, CI is now available for this PR.\n\n"
"to run full CI or `/ci retry` to retry failed jobs.\n\n" "- `/ci run` starts a CI build.\n"
"- `/ci retry` retries failed jobs in the CI build for the current "
"PR head. If the current head has no CI build, it starts a new CI "
"build for the current head containing only jobs that failed in "
"the latest earlier CI build for this PR.\n\n"
f"{CI_AUTHORIZED_COMMENT_MARKER}" f"{CI_AUTHORIZED_COMMENT_MARKER}"
), ),
) )
@@ -638,6 +655,7 @@ def handle_run_ci(
actor: str, actor: str,
buildkite: BuildkiteClient, buildkite: BuildkiteClient,
comment_id: int, comment_id: int,
command: str,
github: GitHubClient, github: GitHubClient,
pr: Mapping[str, Any], pr: Mapping[str, Any],
) -> str: ) -> str:
@@ -664,13 +682,15 @@ def handle_run_ci(
current_pr = github.get_pr(pr["number"]) current_pr = github.get_pr(pr["number"])
if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]: if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
return ( return (
"The PR head changed while processing the command. Comment `/ci run` again." "The PR head changed while processing the command. "
f"Comment `{command}` again."
) )
build = buildkite.create_build( build = buildkite.create_build(
create_build_payload( create_build_payload(
actor=actor, actor=actor,
comment_id=comment_id, comment_id=comment_id,
command=command,
pr=current_pr, pr=current_pr,
) )
) )
@@ -835,11 +855,12 @@ def run(
return return
print(f"Authorized @{actor}: {reason}") print(f"Authorized @{actor}: {reason}")
if command == COMMAND_RUN_CI: if command in RUN_CI_COMMAND_ENV:
message = handle_run_ci( message = handle_run_ci(
actor=actor, actor=actor,
buildkite=buildkite, buildkite=buildkite,
comment_id=comment_id, comment_id=comment_id,
command=command,
github=github, github=github,
pr=pr, pr=pr,
) )
@@ -12,6 +12,8 @@ from run_ci_command import (
CI_AUTHORIZED_COMMENT_MARKER, CI_AUTHORIZED_COMMENT_MARKER,
COMMAND_RETRY_FAILED, COMMAND_RETRY_FAILED,
COMMAND_RUN_CI, COMMAND_RUN_CI,
COMMAND_RUN_CI_ALL,
COMMAND_RUN_CI_NIGHTLY,
RETRY_STATES, RETRY_STATES,
BuildkiteClient, BuildkiteClient,
HttpTransport, HttpTransport,
@@ -269,11 +271,17 @@ class RunCiCommandTest(unittest.TestCase):
def test_only_exact_ci_commands_are_accepted(self) -> None: def test_only_exact_ci_commands_are_accepted(self) -> None:
self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI) self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
self.assertEqual(parse_command(COMMAND_RUN_CI_ALL), COMMAND_RUN_CI_ALL)
self.assertEqual(
parse_command(COMMAND_RUN_CI_NIGHTLY),
COMMAND_RUN_CI_NIGHTLY,
)
self.assertEqual( self.assertEqual(
parse_command(COMMAND_RETRY_FAILED), parse_command(COMMAND_RETRY_FAILED),
COMMAND_RETRY_FAILED, COMMAND_RETRY_FAILED,
) )
self.assertIsNone(parse_command("/ci run please")) self.assertIsNone(parse_command("/ci run please"))
self.assertIsNone(parse_command("/ci run all please"))
self.assertIsNone(parse_command(" /ci run")) self.assertIsNone(parse_command(" /ci run"))
def test_write_access_authorizes_reviewers_and_authors(self) -> None: def test_write_access_authorizes_reviewers_and_authors(self) -> None:
@@ -435,6 +443,28 @@ class RunCiCommandTest(unittest.TestCase):
self.assertTrue(github.comments[0].startswith("✅ ")) self.assertTrue(github.comments[0].startswith("✅ "))
self.assertIn("Buildkite CI #123", github.comments[0]) self.assertIn("Buildkite CI #123", github.comments[0])
def test_run_all_sets_buildkite_environment(self) -> None:
github = FakeGitHub()
buildkite = FakeBuildkite([[], []])
run(make_event(COMMAND_RUN_CI_ALL), github, buildkite)
payload = buildkite.created_builds[0]
self.assertEqual(payload["message"], "PR #42 /ci run all by @reviewer")
self.assertEqual(payload["env"]["RUN_ALL"], "1")
self.assertNotIn("NIGHTLY", payload["env"])
def test_run_nightly_sets_buildkite_environment(self) -> None:
github = FakeGitHub()
buildkite = FakeBuildkite([[], []])
run(make_event(COMMAND_RUN_CI_NIGHTLY), github, buildkite)
payload = buildkite.created_builds[0]
self.assertEqual(payload["message"], "PR #42 /ci run nightly by @reviewer")
self.assertEqual(payload["env"]["RUN_ALL"], "1")
self.assertEqual(payload["env"]["NIGHTLY"], "1")
def test_unapproved_authors_are_denied_without_buildkite(self) -> None: def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
github = FakeGitHub( github = FakeGitHub(
permission="read", permission="read",
@@ -485,10 +515,15 @@ class RunCiCommandTest(unittest.TestCase):
notify_authorized(event, github) notify_authorized(event, github)
self.assertEqual(len(github.comments), 1) self.assertEqual(len(github.comments), 1)
self.assertTrue(github.comments[0].startswith("✅ @author")) comment = github.comments[0]
self.assertIn("`/ci run`", github.comments[0]) self.assertTrue(comment.startswith("✅ @author"))
self.assertIn("`/ci retry`", github.comments[0]) self.assertIn("`/ci run` starts a CI build", comment)
self.assertIn(CI_AUTHORIZED_COMMENT_MARKER, github.comments[0]) self.assertIn("`/ci retry` retries failed jobs", comment)
self.assertIn("CI build for the current PR head", comment)
self.assertIn("only jobs that failed in the latest earlier CI build", comment)
self.assertNotIn(COMMAND_RUN_CI_ALL, comment)
self.assertNotIn(COMMAND_RUN_CI_NIGHTLY, comment)
self.assertIn(CI_AUTHORIZED_COMMENT_MARKER, comment)
def test_ready_label_does_not_notify_after_trusted_approval(self) -> None: def test_ready_label_does_not_notify_after_trusted_approval(self) -> None:
pr = make_pr(labels=[{"name": "ready"}]) pr = make_pr(labels=[{"name": "ready"}])