[CI] Add comment-based Buildkite triggers (#50132)

Signed-off-by: khluu <khluu000@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Kevin H. Luu
2026-07-28 18:41:48 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 6fbbcf2151
commit 7f4c52f2ba
6 changed files with 989 additions and 5 deletions
+2 -2
View File
@@ -80,9 +80,9 @@ jobs:
'',
'\u{1f4ac} Join our developer Slack at https://slack.vllm.ai to discuss your PR in `#pr-reviews`, coordinate on features in `#feat-` channels, or join special interest groups in `#sig-` channels.',
'',
'PRs do not trigger a full CI run by default. Once the PR is approved and ready to go, your PR reviewer(s) can run CI to test the changes comprehensively before merging.',
'PRs do not trigger a full CI run by default. Reviewers with write access and configured trusted contributors can comment `/ci run` whenever CI signals are needed.',
'',
'To run CI, PR reviewers can either: Add `ready` label to the PR or enable auto-merge.',
'Once the PR is approved or has the `ready` label, the PR author can also use `/ci run` or `/ci retry`. New commits do not start CI automatically.',
'',
'If you have any questions, please reach out to us on Slack at https://slack.vllm.ai.',
'',
+1 -1
View File
@@ -41,7 +41,7 @@ jobs:
if (hasReadyLabel || hasVerifiedLabel || mergedCount >= 4) {
core.info(`Check passed: verified label=${hasVerifiedLabel}, ready label=${hasReadyLabel}, 4+ merged PRs=${mergedCount >= 4}`);
} else {
core.setFailed(`PR must have the 'verified', 'ready', or 'ready-run-all-tests' label (the ready labels also trigger tests) or the author must have at least 4 merged PRs (found ${mergedCount}).`);
core.setFailed(`PR must have the 'verified', 'ready', or 'ready-run-all-tests' label to run pre-commit, or the author must have at least 4 merged PRs (found ${mergedCount}).`);
}
pre-commit:
+40
View File
@@ -0,0 +1,40 @@
name: Run CI from PR comment
on:
issue_comment:
types: [created]
concurrency:
group: run-ci-comment-${{ github.event.issue.number }}
cancel-in-progress: false
permissions:
contents: read
issues: write
pull-requests: read
jobs:
run-ci-command:
if: >-
github.event.issue.pull_request &&
(github.event.comment.body == '/ci run' ||
github.event.comment.body == '/ci retry')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.event.repository.default_branch }}
persist-credentials: false
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
python-version: "3.12"
- name: Authorize and run CI command
run: >-
uv run --no-project --python 3.12
.github/workflows/scripts/run_ci_command.py
env:
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
BUILDKITE_ORGANIZATION: vllm
BUILDKITE_PIPELINE: ci
CI_TRUSTED_USERS: ${{ vars.CI_TRUSTED_USERS }}
GH_TOKEN: ${{ github.token }}
+580
View File
@@ -0,0 +1,580 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
import sys
import urllib.error
import urllib.parse
import urllib.request
from collections.abc import Mapping, Sequence
from typing import Any
COMMAND_RUN_CI = "/ci run"
COMMAND_RETRY_FAILED = "/ci retry"
READY_LABELS = {"ready", "ready-run-all-tests"}
TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
ACTIVE_BUILD_STATES = {
"blocked",
"creating",
"scheduled",
"running",
"failing",
"canceling",
"waiting",
"waiting_failed",
}
RETRY_STATES = "failed,timed_out,expired"
class ApiError(RuntimeError):
def __init__(self, status: int | None, message: str) -> None:
super().__init__(message)
self.status = status
class HttpTransport:
def request(
self,
url: str,
*,
body: Mapping[str, Any] | None = None,
headers: Mapping[str, str] | None = None,
method: str = "GET",
) -> Any:
data = None if body is None else json.dumps(body).encode()
request = urllib.request.Request(
url,
data=data,
headers=dict(headers or {}),
method=method,
)
try:
with urllib.request.urlopen(request, timeout=30) as response:
response_body = response.read().decode()
except urllib.error.HTTPError as error:
response_body = error.read().decode()
message = self._error_message(response_body, error.reason)
raise ApiError(
error.code,
f"API returned {error.code}: {message}",
) from error
except urllib.error.URLError as error:
raise ApiError(None, f"API request failed: {error.reason}") from error
if not response_body:
return None
try:
return json.loads(response_body)
except json.JSONDecodeError as error:
raise ApiError(None, "API returned a non-JSON response.") from error
@staticmethod
def _error_message(response_body: str, fallback: str) -> str:
try:
parsed = json.loads(response_body)
except json.JSONDecodeError:
return fallback
return str(parsed.get("message", fallback))
class GitHubClient:
def __init__(
self,
token: str,
repository: str,
transport: HttpTransport | None = None,
) -> None:
if not token:
raise RuntimeError("GH_TOKEN is not set.")
self.owner, self.repo = repository.split("/", maxsplit=1)
self.transport = transport or HttpTransport()
self.headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
"User-Agent": "vllm-ci-command",
"X-GitHub-Api-Version": "2022-11-28",
}
def _request(
self,
path: str,
*,
body: Mapping[str, Any] | None = None,
method: str = "GET",
) -> Any:
return self.transport.request(
f"https://api.github.com{path}",
body=body,
headers=self.headers,
method=method,
)
def _repo_path(self, suffix: str) -> str:
owner = urllib.parse.quote(self.owner, safe="")
repo = urllib.parse.quote(self.repo, safe="")
return f"/repos/{owner}/{repo}{suffix}"
def _paginate(self, path: str) -> list[dict[str, Any]]:
results: list[dict[str, Any]] = []
separator = "&" if "?" in path else "?"
for page in range(1, 101):
response = self._request(f"{path}{separator}per_page=100&page={page}")
if not isinstance(response, list):
raise ApiError(None, "GitHub API returned an invalid list response.")
results.extend(response)
if len(response) < 100:
return results
raise ApiError(None, "GitHub API pagination exceeded 10,000 results.")
def get_pr(self, number: int) -> dict[str, Any]:
return self._request(self._repo_path(f"/pulls/{number}"))
def get_permission(self, actor: str) -> str:
username = urllib.parse.quote(actor, safe="")
try:
response = self._request(
self._repo_path(f"/collaborators/{username}/permission")
)
except ApiError as error:
if error.status == 404:
return "none"
raise
return str(response["permission"])
def get_review_decision(self, number: int) -> str | None:
query = """
query($owner: String!, $repo: String!, $number: Int!) {
repository(owner: $owner, name: $repo) {
pullRequest(number: $number) {
reviewDecision
}
}
}
"""
response = self._request(
"/graphql",
body={
"query": query,
"variables": {
"number": number,
"owner": self.owner,
"repo": self.repo,
},
},
method="POST",
)
return response["data"]["repository"]["pullRequest"]["reviewDecision"]
def list_reviews(self, number: int) -> list[dict[str, Any]]:
return self._paginate(self._repo_path(f"/pulls/{number}/reviews"))
def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
return self._paginate(
self._repo_path(f"/issues/comments/{comment_id}/reactions")
)
def add_reaction(self, comment_id: int, content: str) -> None:
self._request(
self._repo_path(f"/issues/comments/{comment_id}/reactions"),
body={"content": content},
method="POST",
)
def add_comment(self, issue_number: int, body: str) -> None:
self._request(
self._repo_path(f"/issues/{issue_number}/comments"),
body={"body": body},
method="POST",
)
class BuildkiteClient:
def __init__(
self,
token: str,
organization: str,
pipeline: str,
transport: HttpTransport | None = None,
) -> None:
self.token = token
self.transport = transport or HttpTransport()
organization = urllib.parse.quote(organization, safe="")
pipeline = urllib.parse.quote(pipeline, safe="")
self.base_url = (
"https://api.buildkite.com/v2/organizations/"
f"{organization}/pipelines/{pipeline}/builds"
)
def _request(
self,
*,
body: Mapping[str, Any] | None = None,
method: str = "GET",
path: str = "",
query: Sequence[tuple[str, str]] = (),
) -> Any:
if not self.token:
raise RuntimeError("The BUILDKITE_API_TOKEN repository secret is not set.")
url = f"{self.base_url}{path}"
if query:
url = f"{url}?{urllib.parse.urlencode(query)}"
return self.transport.request(
url,
body=body,
headers={
"Authorization": f"Bearer {self.token}",
"Content-Type": "application/json",
"User-Agent": "vllm-ci-command",
},
method=method,
)
def list_builds(
self,
commit: str,
*,
metadata: tuple[str, str] | None = None,
) -> list[dict[str, Any]]:
query = [
("commit", commit),
("exclude_jobs", "true"),
("exclude_pipeline", "true"),
("per_page", "100"),
]
if metadata:
key, value = metadata
query.append((f"meta_data[{key}]", value))
response = self._request(query=query)
if not isinstance(response, list):
raise ApiError(None, "Buildkite API returned an invalid build list.")
return response
def create_build(self, body: Mapping[str, Any]) -> dict[str, Any]:
return self._request(body=body, method="POST")
def retry_failed_jobs(
self,
build_number: int,
states: str,
) -> dict[str, Any]:
number = urllib.parse.quote(str(build_number), safe="")
return self._request(
body={"states": states},
method="PUT",
path=f"/{number}/retry_failed_jobs",
)
def parse_command(body: str) -> str | None:
if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}:
return body
return None
def parse_trusted_users(value: str = "") -> set[str]:
return {
user.casefold() for item in value.split(",") for user in item.split() if user
}
def has_ready_label(pr: Mapping[str, Any]) -> bool:
return any(label["name"] in READY_LABELS for label in pr["labels"])
def is_trusted_permission(permission: str) -> bool:
return permission in TRUSTED_PERMISSIONS
def authorize(
*,
actor: str,
permission: str,
pr: Mapping[str, Any],
trusted_approval: bool = False,
trusted_users: set[str] | None = None,
) -> tuple[bool, str]:
trusted_users = trusted_users or set()
if is_trusted_permission(permission):
return True, f"repository {permission} permission"
if actor.casefold() in trusted_users:
return True, "configured trusted contributor"
if actor.casefold() != pr["user"]["login"].casefold():
return (
False,
"Only reviewers with write access can run CI before it is "
"delegated to the PR author.",
)
if pr["draft"]:
return False, "PR authors cannot run CI while the PR is a draft."
if has_ready_label(pr):
return True, "ready label"
if trusted_approval:
return True, "approval from a trusted reviewer"
return (
False,
"A reviewer with write access must run `/ci run`, approve the PR, "
"or add the `ready` label first.",
)
def has_trusted_approval(
github: GitHubClient,
number: int,
trusted_users: set[str],
) -> bool:
if github.get_review_decision(number) != "APPROVED":
return False
latest_review_states: dict[str, tuple[str, str]] = {}
for review in github.list_reviews(number):
user = review.get("user") or {}
login = user.get("login")
state = review.get("state")
if login and state in {"APPROVED", "CHANGES_REQUESTED", "DISMISSED"}:
latest_review_states[login.casefold()] = (login, state)
for login, state in latest_review_states.values():
if state != "APPROVED":
continue
if login.casefold() in trusted_users:
return True
if is_trusted_permission(github.get_permission(login)):
return True
return False
def is_build_for_pr(build: Mapping[str, Any], pr_number: int) -> bool:
pull_request = build.get("pull_request")
if isinstance(pull_request, Mapping):
build_pr_number = pull_request.get("id", pull_request.get("number"))
if build_pr_number is not None:
return str(build_pr_number) == str(pr_number)
metadata = build.get("meta_data") or {}
return str(metadata.get("github-pr-number")) == str(pr_number)
def is_active_build(build: Mapping[str, Any]) -> bool:
return bool(build.get("blocked")) or build.get("state") in ACTIVE_BUILD_STATES
def select_latest_build(
builds: Sequence[dict[str, Any]],
pr_number: int,
) -> dict[str, Any] | None:
matching = [build for build in builds if is_build_for_pr(build, pr_number)]
return max(matching, key=lambda build: build.get("created_at", ""), default=None)
def create_build_payload(
*,
actor: str,
comment_id: int,
pr: Mapping[str, Any],
) -> dict[str, Any]:
return {
"commit": pr["head"]["sha"],
"branch": pr["head"]["ref"],
"message": f"PR #{pr['number']} {COMMAND_RUN_CI} by @{actor}",
"pull_request_id": pr["number"],
"pull_request_base_branch": pr["base"]["ref"],
"pull_request_repository": pr["head"]["repo"]["clone_url"],
"pull_request_labels": [label["name"] for label in pr["labels"]],
"env": {
"VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
"VLLM_CI_TRIGGERED_BY": actor,
},
"meta_data": {
"github-comment-id": str(comment_id),
"github-pr-number": str(pr["number"]),
"github-triggered-by": actor,
},
}
def add_reaction_safely(
github: GitHubClient,
comment_id: int,
content: str,
) -> None:
try:
github.add_reaction(comment_id, content)
except Exception as error:
print(f"Could not add {content} reaction: {error}", file=sys.stderr)
def is_already_handled(github: GitHubClient, comment_id: int) -> bool:
return any(
reaction.get("content") in {"rocket", "-1"}
and (reaction.get("user") or {}).get("login") == "github-actions[bot]"
for reaction in github.list_reactions(comment_id)
)
def handle_run_ci(
*,
actor: str,
buildkite: BuildkiteClient,
comment_id: int,
github: GitHubClient,
pr: Mapping[str, Any],
) -> str:
duplicate_builds = buildkite.list_builds(
pr["head"]["sha"],
metadata=("github-comment-id", str(comment_id)),
)
duplicate = select_latest_build(duplicate_builds, pr["number"])
if duplicate:
return f"CI was already requested by this comment: {duplicate['web_url']}"
current_builds = buildkite.list_builds(pr["head"]["sha"])
active_build = next(
(
build
for build in current_builds
if is_build_for_pr(build, pr["number"]) and is_active_build(build)
),
None,
)
if active_build:
return f"CI is already running for this commit: {active_build['web_url']}"
current_pr = github.get_pr(pr["number"])
if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
return (
"The PR head changed while processing the command. Comment `/ci run` again."
)
build = buildkite.create_build(
create_build_payload(
actor=actor,
comment_id=comment_id,
pr=current_pr,
)
)
return (
f"Triggered [Buildkite CI #{build['number']}]({build['web_url']}) "
f"for commit `{current_pr['head']['sha'][:12]}`."
)
def handle_retry_failed(
*,
buildkite: BuildkiteClient,
pr: Mapping[str, Any],
) -> str:
builds = buildkite.list_builds(pr["head"]["sha"])
build = select_latest_build(builds, pr["number"])
if not build:
return "No CI build exists for the current PR commit. Use `/ci run` first."
if not build.get("finished_at") or is_active_build(build):
return f"CI is still running for this commit: {build['web_url']}"
retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES)
if retried["retried_jobs_count"] == 0:
return (
f"No failed, timed-out, or expired jobs need retrying: {build['web_url']}"
)
return (
f"Queued {retried['retried_jobs_count']} failed job(s) for retry in "
f"[Buildkite CI #{build['number']}]({build['web_url']})."
)
def run(
event: Mapping[str, Any],
github: GitHubClient,
buildkite: BuildkiteClient,
trusted_users_value: str = "",
) -> None:
command = parse_command(event["comment"]["body"])
if not command or "pull_request" not in event["issue"]:
return
issue_number = event["issue"]["number"]
comment_id = event["comment"]["id"]
actor = event["comment"]["user"]["login"]
if is_already_handled(github, comment_id):
print(f"Comment {comment_id} was already handled.")
return
add_reaction_safely(github, comment_id, "eyes")
try:
pr = github.get_pr(issue_number)
permission = github.get_permission(actor)
if pr["state"] != "open":
github.add_comment(issue_number, "CI commands require an open PR.")
return
trusted_users = parse_trusted_users(trusted_users_value)
should_check_approval = (
not is_trusted_permission(permission)
and actor.casefold() not in trusted_users
and actor.casefold() == pr["user"]["login"].casefold()
and not pr["draft"]
and not has_ready_label(pr)
)
trusted_approval = should_check_approval and has_trusted_approval(
github,
issue_number,
trusted_users,
)
allowed, reason = authorize(
actor=actor,
permission=permission,
pr=pr,
trusted_approval=trusted_approval,
trusted_users=trusted_users,
)
if not allowed:
add_reaction_safely(github, comment_id, "-1")
github.add_comment(issue_number, f"@{actor}, {reason}")
return
print(f"Authorized @{actor}: {reason}")
if command == COMMAND_RUN_CI:
message = handle_run_ci(
actor=actor,
buildkite=buildkite,
comment_id=comment_id,
github=github,
pr=pr,
)
else:
message = handle_retry_failed(buildkite=buildkite, pr=pr)
add_reaction_safely(github, comment_id, "rocket")
github.add_comment(issue_number, message)
except Exception:
add_reaction_safely(github, comment_id, "confused")
raise
def main() -> None:
event_path = os.environ["GITHUB_EVENT_PATH"]
with open(event_path, encoding="utf-8") as event_file:
event = json.load(event_file)
if not parse_command(event["comment"]["body"]):
return
github = GitHubClient(
os.environ.get("GH_TOKEN", ""),
os.environ["GITHUB_REPOSITORY"],
)
buildkite = BuildkiteClient(
os.environ.get("BUILDKITE_API_TOKEN", ""),
os.environ.get("BUILDKITE_ORGANIZATION", "vllm"),
os.environ.get("BUILDKITE_PIPELINE", "ci"),
)
run(
event,
github,
buildkite,
os.environ.get("CI_TRUSTED_USERS", ""),
)
if __name__ == "__main__":
main()
@@ -0,0 +1,362 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import unittest
from typing import Any
from run_ci_command import (
COMMAND_RETRY_FAILED,
COMMAND_RUN_CI,
RETRY_STATES,
BuildkiteClient,
authorize,
create_build_payload,
has_trusted_approval,
is_active_build,
is_build_for_pr,
parse_command,
parse_trusted_users,
run,
select_latest_build,
)
def make_pr(**overrides: Any) -> dict[str, Any]:
pr = {
"base": {"ref": "main"},
"draft": False,
"head": {
"ref": "feature",
"repo": {"clone_url": "https://github.com/contributor/vllm.git"},
"sha": "0123456789abcdef",
},
"labels": [],
"number": 42,
"state": "open",
"user": {"login": "author"},
}
pr.update(overrides)
return pr
def make_event(command: str, actor: str = "reviewer") -> dict[str, Any]:
return {
"comment": {
"body": command,
"id": 99,
"user": {"login": actor},
},
"issue": {
"number": 42,
"pull_request": {},
},
}
class FakeGitHub:
def __init__(
self,
*,
permission: str = "write",
permissions: dict[str, str] | None = None,
pr: dict[str, Any] | None = None,
review_decision: str = "REVIEW_REQUIRED",
reviews: list[dict[str, Any]] | None = None,
) -> None:
self.comments: list[str] = []
self.permission = permission
self.permissions = permissions or {}
self.pr = pr or make_pr()
self.reactions: list[str] = []
self.review_decision = review_decision
self.reviews = reviews or []
def get_pr(self, number: int) -> dict[str, Any]:
return self.pr
def get_permission(self, actor: str) -> str:
return self.permissions.get(actor, self.permission)
def get_review_decision(self, number: int) -> str:
return self.review_decision
def list_reviews(self, number: int) -> list[dict[str, Any]]:
return self.reviews
def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
return []
def add_reaction(self, comment_id: int, content: str) -> None:
self.reactions.append(content)
def add_comment(self, issue_number: int, body: str) -> None:
self.comments.append(body)
class FakeBuildkite:
def __init__(
self,
build_lists: list[list[dict[str, Any]]] | None = None,
) -> None:
self.build_lists = build_lists or []
self.created_builds: list[dict[str, Any]] = []
self.list_calls: list[tuple[str, tuple[str, str] | None]] = []
self.retry_calls: list[tuple[int, str]] = []
def list_builds(
self,
commit: str,
*,
metadata: tuple[str, str] | None = None,
) -> list[dict[str, Any]]:
self.list_calls.append((commit, metadata))
return self.build_lists.pop(0)
def create_build(self, body: dict[str, Any]) -> dict[str, Any]:
self.created_builds.append(body)
return {
"number": 123,
"web_url": "https://buildkite.example/builds/123",
}
def retry_failed_jobs(
self,
build_number: int,
states: str,
) -> dict[str, Any]:
self.retry_calls.append((build_number, states))
return {"retried_jobs_count": 3}
class FakeTransport:
def __init__(self, response: Any) -> None:
self.calls: list[dict[str, Any]] = []
self.response = response
def request(self, url: str, **kwargs: Any) -> Any:
self.calls.append({"url": url, **kwargs})
return self.response
class RunCiCommandTest(unittest.TestCase):
def test_only_exact_ci_commands_are_accepted(self) -> None:
self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
self.assertEqual(
parse_command(COMMAND_RETRY_FAILED),
COMMAND_RETRY_FAILED,
)
self.assertIsNone(parse_command("/ci run please"))
self.assertIsNone(parse_command(" /ci run"))
def test_write_access_authorizes_reviewers_and_authors(self) -> None:
allowed, _ = authorize(
actor="reviewer",
permission="write",
pr=make_pr(),
)
self.assertTrue(allowed)
def test_configured_trusted_contributors_can_run_ci(self) -> None:
trusted_users = parse_trusted_users("trusted-one, TRUSTED-TWO")
allowed, _ = authorize(
actor="trusted-two",
permission="read",
pr=make_pr(),
trusted_users=trusted_users,
)
self.assertTrue(allowed)
def test_authors_need_an_approval_or_ready_label(self) -> None:
pending, _ = authorize(
actor="author",
permission="read",
pr=make_pr(),
)
approved, _ = authorize(
actor="author",
permission="read",
pr=make_pr(),
trusted_approval=True,
)
ready, _ = authorize(
actor="author",
permission="read",
pr=make_pr(labels=[{"name": "ready"}]),
)
self.assertFalse(pending)
self.assertTrue(approved)
self.assertTrue(ready)
def test_non_author_contributors_without_write_are_denied(self) -> None:
allowed, _ = authorize(
actor="contributor",
permission="read",
pr=make_pr(),
trusted_approval=True,
)
self.assertFalse(allowed)
def test_authors_cannot_use_ready_state_on_draft_prs(self) -> None:
allowed, _ = authorize(
actor="author",
permission="read",
pr=make_pr(draft=True, labels=[{"name": "ready"}]),
trusted_approval=True,
)
self.assertFalse(allowed)
def test_only_trusted_reviewers_can_delegate_through_approval(self) -> None:
approved_review = {
"state": "APPROVED",
"user": {"login": "reviewer"},
}
trusted = FakeGitHub(
permission="read",
permissions={"reviewer": "write"},
review_decision="APPROVED",
reviews=[approved_review],
)
untrusted = FakeGitHub(
permission="read",
review_decision="APPROVED",
reviews=[approved_review],
)
self.assertTrue(has_trusted_approval(trusted, 42, set()))
self.assertFalse(has_trusted_approval(untrusted, 42, set()))
def test_build_matching_is_scoped_to_the_pr(self) -> None:
self.assertTrue(is_build_for_pr({"pull_request": {"id": 42}}, 42))
self.assertFalse(is_build_for_pr({"pull_request": {"id": 43}}, 42))
self.assertTrue(
is_build_for_pr(
{"meta_data": {"github-pr-number": "42"}},
42,
)
)
def test_latest_build_selection_ignores_other_prs(self) -> None:
latest = select_latest_build(
[
{
"created_at": "2026-07-28T02:00:00Z",
"number": 3,
"pull_request": {"id": 43},
},
{
"created_at": "2026-07-28T01:00:00Z",
"number": 2,
"pull_request": {"id": 42},
},
{
"created_at": "2026-07-28T00:00:00Z",
"number": 1,
"pull_request": {"id": 42},
},
],
42,
)
self.assertEqual(latest["number"], 2)
def test_active_build_states_prevent_duplicate_runs(self) -> None:
self.assertTrue(is_active_build({"state": "scheduled"}))
self.assertTrue(is_active_build({"state": "running"}))
self.assertTrue(is_active_build({"state": "waiting"}))
self.assertTrue(is_active_build({"blocked": True, "state": "passed"}))
self.assertFalse(is_active_build({"state": "failed"}))
def test_build_payload_preserves_pr_context(self) -> None:
payload = create_build_payload(
actor="reviewer",
comment_id=99,
pr=make_pr(labels=[{"name": "ready"}, {"name": "v1"}]),
)
self.assertEqual(
payload,
{
"commit": "0123456789abcdef",
"branch": "feature",
"message": "PR #42 /ci run by @reviewer",
"pull_request_id": 42,
"pull_request_base_branch": "main",
"pull_request_repository": ("https://github.com/contributor/vllm.git"),
"pull_request_labels": ["ready", "v1"],
"env": {
"VLLM_CI_GITHUB_COMMENT_ID": "99",
"VLLM_CI_TRIGGERED_BY": "reviewer",
},
"meta_data": {
"github-comment-id": "99",
"github-pr-number": "42",
"github-triggered-by": "reviewer",
},
},
)
def test_ci_run_dispatches_build_with_current_pr_metadata(self) -> None:
github = FakeGitHub()
buildkite = FakeBuildkite([[], []])
run(make_event(COMMAND_RUN_CI), github, buildkite)
self.assertEqual(len(buildkite.created_builds), 1)
self.assertEqual(
buildkite.created_builds[0]["message"],
"PR #42 /ci run by @reviewer",
)
self.assertEqual(github.reactions, ["eyes", "rocket"])
self.assertIn("Buildkite CI #123", github.comments[0])
def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
github = FakeGitHub(
permission="read",
pr=make_pr(),
review_decision="REVIEW_REQUIRED",
)
buildkite = FakeBuildkite()
run(make_event(COMMAND_RUN_CI, "author"), github, buildkite)
self.assertEqual(buildkite.list_calls, [])
self.assertEqual(github.reactions, ["eyes", "-1"])
self.assertIn("approve the PR", github.comments[0])
def test_ci_retry_uses_latest_current_sha_build(self) -> None:
github = FakeGitHub(
permission="read",
pr=make_pr(labels=[{"name": "ready"}]),
)
buildkite = FakeBuildkite(
[
[
{
"created_at": "2026-07-28T01:00:00Z",
"finished_at": "2026-07-28T02:00:00Z",
"number": 123,
"pull_request": {"id": 42},
"state": "failed",
"web_url": "https://buildkite.example/builds/123",
}
]
]
)
run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite)
self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)])
self.assertIn("Queued 3 failed job", github.comments[0])
def test_buildkite_retry_uses_retry_failed_jobs_endpoint(self) -> None:
transport = FakeTransport({"retried_jobs_count": 2})
client = BuildkiteClient(
"secret",
"vllm",
"ci",
transport=transport,
)
client.retry_failed_jobs(123, RETRY_STATES)
call = transport.calls[0]
self.assertEqual(call["method"], "PUT")
self.assertTrue(call["url"].endswith("/123/retry_failed_jobs"))
self.assertEqual(call["body"], {"states": RETRY_STATES})
if __name__ == "__main__":
unittest.main()