[CI] Add run-all comment commands (#51087)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
ffee32460d
commit
43c4bdcae9
@@ -18,6 +18,8 @@ jobs:
|
|||||||
if: >-
|
if: >-
|
||||||
github.event.issue.pull_request &&
|
github.event.issue.pull_request &&
|
||||||
(github.event.comment.body == '/ci run' ||
|
(github.event.comment.body == '/ci run' ||
|
||||||
|
github.event.comment.body == '/ci run all' ||
|
||||||
|
github.event.comment.body == '/ci run nightly' ||
|
||||||
github.event.comment.body == '/ci retry')
|
github.event.comment.body == '/ci retry')
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
|
|||||||
@@ -13,7 +13,14 @@ from collections.abc import Callable, Mapping, Sequence
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
COMMAND_RUN_CI = "/ci run"
|
COMMAND_RUN_CI = "/ci run"
|
||||||
|
COMMAND_RUN_CI_ALL = "/ci run all"
|
||||||
|
COMMAND_RUN_CI_NIGHTLY = "/ci run nightly"
|
||||||
COMMAND_RETRY_FAILED = "/ci retry"
|
COMMAND_RETRY_FAILED = "/ci retry"
|
||||||
|
RUN_CI_COMMAND_ENV = {
|
||||||
|
COMMAND_RUN_CI: {},
|
||||||
|
COMMAND_RUN_CI_ALL: {"RUN_ALL": "1"},
|
||||||
|
COMMAND_RUN_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"},
|
||||||
|
}
|
||||||
CI_AUTHORIZED_COMMENT_MARKER = "<!-- vllm-ci-authorized -->"
|
CI_AUTHORIZED_COMMENT_MARKER = "<!-- vllm-ci-authorized -->"
|
||||||
READY_LABELS = {"ready", "ready-run-all-tests"}
|
READY_LABELS = {"ready", "ready-run-all-tests"}
|
||||||
TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
|
TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
|
||||||
@@ -383,7 +390,7 @@ class BuildkiteClient:
|
|||||||
|
|
||||||
|
|
||||||
def parse_command(body: str) -> str | None:
|
def parse_command(body: str) -> str | None:
|
||||||
if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}:
|
if body in {*RUN_CI_COMMAND_ENV, COMMAND_RETRY_FAILED}:
|
||||||
return body
|
return body
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -486,21 +493,27 @@ def create_build_payload(
|
|||||||
*,
|
*,
|
||||||
actor: str,
|
actor: str,
|
||||||
comment_id: int,
|
comment_id: int,
|
||||||
|
command: str = COMMAND_RUN_CI,
|
||||||
pr: Mapping[str, Any],
|
pr: Mapping[str, Any],
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
|
if command not in RUN_CI_COMMAND_ENV:
|
||||||
|
raise ValueError(f"Unsupported run command: {command}")
|
||||||
|
|
||||||
|
env = {
|
||||||
|
"VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
|
||||||
|
"VLLM_CI_TRIGGERED_BY": actor,
|
||||||
|
**RUN_CI_COMMAND_ENV[command],
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
"commit": pr["head"]["sha"],
|
"commit": pr["head"]["sha"],
|
||||||
"branch": pr["head"]["ref"],
|
"branch": pr["head"]["ref"],
|
||||||
"message": f"PR #{pr['number']} {COMMAND_RUN_CI} by @{actor}",
|
"message": f"PR #{pr['number']} {command} by @{actor}",
|
||||||
"pull_request_id": pr["number"],
|
"pull_request_id": pr["number"],
|
||||||
"pull_request_base_branch": pr["base"]["ref"],
|
"pull_request_base_branch": pr["base"]["ref"],
|
||||||
"pull_request_repository": pr["head"]["repo"]["clone_url"],
|
"pull_request_repository": pr["head"]["repo"]["clone_url"],
|
||||||
"pull_request_labels": [label["name"] for label in pr["labels"]],
|
"pull_request_labels": [label["name"] for label in pr["labels"]],
|
||||||
"ignore_pipeline_branch_filters": True,
|
"ignore_pipeline_branch_filters": True,
|
||||||
"env": {
|
"env": env,
|
||||||
"VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
|
|
||||||
"VLLM_CI_TRIGGERED_BY": actor,
|
|
||||||
},
|
|
||||||
"meta_data": {
|
"meta_data": {
|
||||||
"github-comment-id": str(comment_id),
|
"github-comment-id": str(comment_id),
|
||||||
"github-pr-number": str(pr["number"]),
|
"github-pr-number": str(pr["number"]),
|
||||||
@@ -626,8 +639,12 @@ def notify_authorized(
|
|||||||
github.add_comment(
|
github.add_comment(
|
||||||
pr["number"],
|
pr["number"],
|
||||||
(
|
(
|
||||||
f"✅ @{author}, CI is now available for this PR. Comment `/ci run` "
|
f"✅ @{author}, CI is now available for this PR.\n\n"
|
||||||
"to run full CI or `/ci retry` to retry failed jobs.\n\n"
|
"- `/ci run` starts a CI build.\n"
|
||||||
|
"- `/ci retry` retries failed jobs in the CI build for the current "
|
||||||
|
"PR head. If the current head has no CI build, it starts a new CI "
|
||||||
|
"build for the current head containing only jobs that failed in "
|
||||||
|
"the latest earlier CI build for this PR.\n\n"
|
||||||
f"{CI_AUTHORIZED_COMMENT_MARKER}"
|
f"{CI_AUTHORIZED_COMMENT_MARKER}"
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -638,6 +655,7 @@ def handle_run_ci(
|
|||||||
actor: str,
|
actor: str,
|
||||||
buildkite: BuildkiteClient,
|
buildkite: BuildkiteClient,
|
||||||
comment_id: int,
|
comment_id: int,
|
||||||
|
command: str,
|
||||||
github: GitHubClient,
|
github: GitHubClient,
|
||||||
pr: Mapping[str, Any],
|
pr: Mapping[str, Any],
|
||||||
) -> str:
|
) -> str:
|
||||||
@@ -664,13 +682,15 @@ def handle_run_ci(
|
|||||||
current_pr = github.get_pr(pr["number"])
|
current_pr = github.get_pr(pr["number"])
|
||||||
if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
|
if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
|
||||||
return (
|
return (
|
||||||
"The PR head changed while processing the command. Comment `/ci run` again."
|
"The PR head changed while processing the command. "
|
||||||
|
f"Comment `{command}` again."
|
||||||
)
|
)
|
||||||
|
|
||||||
build = buildkite.create_build(
|
build = buildkite.create_build(
|
||||||
create_build_payload(
|
create_build_payload(
|
||||||
actor=actor,
|
actor=actor,
|
||||||
comment_id=comment_id,
|
comment_id=comment_id,
|
||||||
|
command=command,
|
||||||
pr=current_pr,
|
pr=current_pr,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -835,11 +855,12 @@ def run(
|
|||||||
return
|
return
|
||||||
|
|
||||||
print(f"Authorized @{actor}: {reason}")
|
print(f"Authorized @{actor}: {reason}")
|
||||||
if command == COMMAND_RUN_CI:
|
if command in RUN_CI_COMMAND_ENV:
|
||||||
message = handle_run_ci(
|
message = handle_run_ci(
|
||||||
actor=actor,
|
actor=actor,
|
||||||
buildkite=buildkite,
|
buildkite=buildkite,
|
||||||
comment_id=comment_id,
|
comment_id=comment_id,
|
||||||
|
command=command,
|
||||||
github=github,
|
github=github,
|
||||||
pr=pr,
|
pr=pr,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ from run_ci_command import (
|
|||||||
CI_AUTHORIZED_COMMENT_MARKER,
|
CI_AUTHORIZED_COMMENT_MARKER,
|
||||||
COMMAND_RETRY_FAILED,
|
COMMAND_RETRY_FAILED,
|
||||||
COMMAND_RUN_CI,
|
COMMAND_RUN_CI,
|
||||||
|
COMMAND_RUN_CI_ALL,
|
||||||
|
COMMAND_RUN_CI_NIGHTLY,
|
||||||
RETRY_STATES,
|
RETRY_STATES,
|
||||||
BuildkiteClient,
|
BuildkiteClient,
|
||||||
HttpTransport,
|
HttpTransport,
|
||||||
@@ -269,11 +271,17 @@ class RunCiCommandTest(unittest.TestCase):
|
|||||||
|
|
||||||
def test_only_exact_ci_commands_are_accepted(self) -> None:
|
def test_only_exact_ci_commands_are_accepted(self) -> None:
|
||||||
self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
|
self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
|
||||||
|
self.assertEqual(parse_command(COMMAND_RUN_CI_ALL), COMMAND_RUN_CI_ALL)
|
||||||
|
self.assertEqual(
|
||||||
|
parse_command(COMMAND_RUN_CI_NIGHTLY),
|
||||||
|
COMMAND_RUN_CI_NIGHTLY,
|
||||||
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
parse_command(COMMAND_RETRY_FAILED),
|
parse_command(COMMAND_RETRY_FAILED),
|
||||||
COMMAND_RETRY_FAILED,
|
COMMAND_RETRY_FAILED,
|
||||||
)
|
)
|
||||||
self.assertIsNone(parse_command("/ci run please"))
|
self.assertIsNone(parse_command("/ci run please"))
|
||||||
|
self.assertIsNone(parse_command("/ci run all please"))
|
||||||
self.assertIsNone(parse_command(" /ci run"))
|
self.assertIsNone(parse_command(" /ci run"))
|
||||||
|
|
||||||
def test_write_access_authorizes_reviewers_and_authors(self) -> None:
|
def test_write_access_authorizes_reviewers_and_authors(self) -> None:
|
||||||
@@ -435,6 +443,28 @@ class RunCiCommandTest(unittest.TestCase):
|
|||||||
self.assertTrue(github.comments[0].startswith("✅ "))
|
self.assertTrue(github.comments[0].startswith("✅ "))
|
||||||
self.assertIn("Buildkite CI #123", github.comments[0])
|
self.assertIn("Buildkite CI #123", github.comments[0])
|
||||||
|
|
||||||
|
def test_run_all_sets_buildkite_environment(self) -> None:
|
||||||
|
github = FakeGitHub()
|
||||||
|
buildkite = FakeBuildkite([[], []])
|
||||||
|
|
||||||
|
run(make_event(COMMAND_RUN_CI_ALL), github, buildkite)
|
||||||
|
|
||||||
|
payload = buildkite.created_builds[0]
|
||||||
|
self.assertEqual(payload["message"], "PR #42 /ci run all by @reviewer")
|
||||||
|
self.assertEqual(payload["env"]["RUN_ALL"], "1")
|
||||||
|
self.assertNotIn("NIGHTLY", payload["env"])
|
||||||
|
|
||||||
|
def test_run_nightly_sets_buildkite_environment(self) -> None:
|
||||||
|
github = FakeGitHub()
|
||||||
|
buildkite = FakeBuildkite([[], []])
|
||||||
|
|
||||||
|
run(make_event(COMMAND_RUN_CI_NIGHTLY), github, buildkite)
|
||||||
|
|
||||||
|
payload = buildkite.created_builds[0]
|
||||||
|
self.assertEqual(payload["message"], "PR #42 /ci run nightly by @reviewer")
|
||||||
|
self.assertEqual(payload["env"]["RUN_ALL"], "1")
|
||||||
|
self.assertEqual(payload["env"]["NIGHTLY"], "1")
|
||||||
|
|
||||||
def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
|
def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
|
||||||
github = FakeGitHub(
|
github = FakeGitHub(
|
||||||
permission="read",
|
permission="read",
|
||||||
@@ -485,10 +515,15 @@ class RunCiCommandTest(unittest.TestCase):
|
|||||||
notify_authorized(event, github)
|
notify_authorized(event, github)
|
||||||
|
|
||||||
self.assertEqual(len(github.comments), 1)
|
self.assertEqual(len(github.comments), 1)
|
||||||
self.assertTrue(github.comments[0].startswith("✅ @author"))
|
comment = github.comments[0]
|
||||||
self.assertIn("`/ci run`", github.comments[0])
|
self.assertTrue(comment.startswith("✅ @author"))
|
||||||
self.assertIn("`/ci retry`", github.comments[0])
|
self.assertIn("`/ci run` starts a CI build", comment)
|
||||||
self.assertIn(CI_AUTHORIZED_COMMENT_MARKER, github.comments[0])
|
self.assertIn("`/ci retry` retries failed jobs", comment)
|
||||||
|
self.assertIn("CI build for the current PR head", comment)
|
||||||
|
self.assertIn("only jobs that failed in the latest earlier CI build", comment)
|
||||||
|
self.assertNotIn(COMMAND_RUN_CI_ALL, comment)
|
||||||
|
self.assertNotIn(COMMAND_RUN_CI_NIGHTLY, comment)
|
||||||
|
self.assertIn(CI_AUTHORIZED_COMMENT_MARKER, comment)
|
||||||
|
|
||||||
def test_ready_label_does_not_notify_after_trusted_approval(self) -> None:
|
def test_ready_label_does_not_notify_after_trusted_approval(self) -> None:
|
||||||
pr = make_pr(labels=[{"name": "ready"}])
|
pr = make_pr(labels=[{"name": "ready"}])
|
||||||
|
|||||||
Reference in New Issue
Block a user