iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[CI] Check target branch freshness before starting CI (#56169)

Browse Source 
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
This commit is contained in:
Andreas Karatzas
2026-09-14 14:40:35 -07:00
committed by GitHub
parent d320a21431
commit bdad63c90a
4 changed files with 322 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/scripts/run_ci_command.py
+92 -12
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import http.client
import json
import os
import random
@@ -22,6 +23,7 @@ COMMAND_RUN_AMD_CI_ALL = "/amd-ci run all"
COMMAND_RUN_AMD_CI_NIGHTLY = "/amd-ci run nightly"
COMMAND_RETRY_AMD_FAILED = "/amd-ci retry"
COMMAND_CANCEL_AMD_CI = "/amd-ci cancel"
ALLOW_STALE_SUFFIX = " --allow-stale"
RUN_CI_COMMAND_ENV = {
COMMAND_RUN_CI: {},
COMMAND_RUN_CI_ALL: {"RUN_ALL": "1"},
@@ -30,20 +32,25 @@ RUN_CI_COMMAND_ENV = {
COMMAND_RUN_AMD_CI_ALL: {"RUN_ALL": "1"},
COMMAND_RUN_AMD_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"},
}
ALLOW_STALE_COMMANDS = frozenset(
f"{command}{ALLOW_STALE_SUFFIX}" for command in RUN_CI_COMMAND_ENV
)
RUN_CI_COMMAND_ENV.update(
{
command: RUN_CI_COMMAND_ENV[command.removesuffix(ALLOW_STALE_SUFFIX)]
for command in ALLOW_STALE_COMMANDS
}
)
UPSTREAM_CI_COMMANDS = frozenset(
{
COMMAND_RUN_CI,
COMMAND_RUN_CI_ALL,
COMMAND_RUN_CI_NIGHTLY,
*(command for command in RUN_CI_COMMAND_ENV if command.startswith("/ci ")),
COMMAND_RETRY_FAILED,
COMMAND_CANCEL_CI,
}
)
AMD_CI_COMMANDS = frozenset(
{
COMMAND_RUN_AMD_CI,
COMMAND_RUN_AMD_CI_ALL,
COMMAND_RUN_AMD_CI_NIGHTLY,
*(command for command in RUN_CI_COMMAND_ENV if command.startswith("/amd-ci ")),
COMMAND_RETRY_AMD_FAILED,
COMMAND_CANCEL_AMD_CI,
}
@@ -79,6 +86,10 @@ class ApiError(RuntimeError):
self.status = status
class CiPreparationError(RuntimeError):
pass
def rate_limit_jitter() -> float:
return random.uniform(1, 5)
@@ -137,6 +148,8 @@ class HttpTransport:
None,
f"API request failed: {error.reason}",
) from error
except (OSError, http.client.HTTPException) as error:
raise ApiError(None, f"API request failed: {error}") from error
if not response_body:
return None
@@ -234,6 +247,17 @@ class GitHubClient:
def get_pr(self, number: int) -> dict[str, Any]:
return self._request(self._repo_path(f"/pulls/{number}"))
def get_commits_behind_base(self, base_ref: str, head_sha: str) -> int:
base_ref = urllib.parse.quote(f"refs/heads/{base_ref}", safe="")
head_sha = urllib.parse.quote(head_sha, safe="")
response = self._request(
self._repo_path(f"/compare/{base_ref}...{head_sha}?per_page=1")
)
behind = response.get("behind_by") if isinstance(response, dict) else None
if type(behind) is not int or behind < 0:
raise ApiError(None, "GitHub returned an invalid commits-behind count.")
return behind
def list_pulls_for_commit(self, commit: str) -> list[dict[str, Any]]:
commit = urllib.parse.quote(commit, safe="")
response = self._request(self._repo_path(f"/commits/{commit}/pulls"))
@@ -747,6 +771,10 @@ def notify_authorized(
(
f"✅ @{author}, CI is now available for this PR.\n\n"
"- `/ci run` starts upstream CI; `/amd-ci run` starts AMD CI only.\n"
"- Your branch must contain every commit currently on its upstream "
"target branch. Merge or rebase onto the latest target branch, then "
"rerun the command. Append `--allow-stale` to a run command to test "
"an outdated branch at your own risk.\n"
"- `/ci retry` retries failed jobs in the CI build for the current "
"PR head. If the current head has no CI build, it starts a new CI "
"build for the current head containing only jobs that failed in "
@@ -794,6 +822,56 @@ def resolve_workflow_run_pr(
return find_matching_pr(github.list_pulls_for_commit(head_sha))
def prepare_pr_for_ci(
github: GitHubClient,
pr: Mapping[str, Any],
command: str,
) -> tuple[dict[str, Any], str]:
base_ref = pr["base"]["ref"]
no_action = "No new CI build was started."
try:
behind = github.get_commits_behind_base(base_ref, pr["head"]["sha"])
current_pr = github.get_pr(pr["number"])
except ApiError as error:
raise CiPreparationError(
f"Could not check the PR against upstream `{base_ref}`. "
f"{no_action} Comment `{command}` again. {error}"
) from error
if (
current_pr["state"] != "open"
or current_pr["head"]["sha"] != pr["head"]["sha"]
or current_pr["base"]["ref"] != pr["base"]["ref"]
):
raise CiPreparationError(
f"The PR changed while checking its branch. {no_action} "
f"Comment `{command}` again."
)
commits = "commit" if behind == 1 else "commits"
lag = f"This PR is {behind} {commits} behind upstream `{base_ref}`."
if command in ALLOW_STALE_COMMANDS:
warning = ""
if behind:
run_command = command.removesuffix(ALLOW_STALE_SUFFIX)
warning = (
f"⚠️ {lag} Running CI at your own risk because "
"`--allow-stale` was requested; outdated CI configuration may "
"cause failures. Before merging, merge or rebase onto the latest "
f"`{base_ref}`, then rerun `{run_command}` on the latest PR commit."
)
print(warning)
return current_pr, warning
if behind:
raise CiPreparationError(
f"{lag} Your branch must contain every commit currently on upstream "
f"`{base_ref}`. {no_action} "
f"Merge or rebase onto the latest `{base_ref}`, then rerun `{command}`."
" To test this branch at your own risk, "
f"use `{command}{ALLOW_STALE_SUFFIX}`."
)
return current_pr, ""
def handle_run_ci(
*,
actor: str,
@@ -828,12 +906,7 @@ def handle_run_ci(
f"{ci_name} is already running for this commit: {active_build['web_url']}"
)
current_pr = github.get_pr(pr["number"])
if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
return (
"The PR head changed while processing the command. "
f"Comment `{command}` again."
)
current_pr, warning = prepare_pr_for_ci(github, pr, command)
build = buildkite.create_build(
create_build_payload(
@@ -846,6 +919,7 @@ def handle_run_ci(
return (
f"Triggered [Buildkite {ci_name} #{build['number']}]({build['web_url']}) "
f"for commit `{current_pr['head']['sha'][:12]}`."
+ (f"\n\n{warning}" if warning else "")
)
@@ -1099,6 +1173,12 @@ def run(
raise ValueError(f"Unsupported CI command: {command}")
add_reaction_safely(github, comment_id, "rocket")
github.add_comment(issue_number, f"✅ {message}")
except CiPreparationError as error:
add_reaction_safely(github, comment_id, "-1")
github.add_comment(
issue_number,
f"❌ {error}\n\n{command_comment_marker(comment_id)}",
)
except Exception:
add_reaction_safely(github, comment_id, "confused")
raise
.github/workflows/scripts/test_run_ci_command.py
+196
View File
@@ -22,8 +22,10 @@ from run_ci_command import (
COMMAND_RUN_CI_ALL,
COMMAND_RUN_CI_NIGHTLY,
RETRY_STATES,
RUN_CI_COMMAND_ENV,
ApiError,
BuildkiteClient,
GitHubClient,
HttpTransport,
authorize,
create_build_payload,
@@ -85,6 +87,7 @@ class FakeGitHub:
comments: list[str] | None = None,
pulls_for_commit: list[dict[str, Any]] | None = None,
prs: dict[int, dict[str, Any]] | None = None,
behind: int = 0,
) -> None:
self.comments = comments or []
self.permission = permission
@@ -95,6 +98,12 @@ class FakeGitHub:
self.reviews = reviews or []
self.pulls_for_commit = pulls_for_commit or []
self.prs = prs or {self.pr["number"]: self.pr}
self.behind = behind
self.lag_queries: list[tuple[str, str]] = []
def get_commits_behind_base(self, base_ref: str, head_sha: str) -> int:
self.lag_queries.append((base_ref, head_sha))
return self.behind
def get_pr(self, number: int) -> dict[str, Any]:
try:
@@ -322,6 +331,11 @@ class RunCiCommandTest(unittest.TestCase):
for command in commands:
with self.subTest(command=command):
self.assertEqual(parse_command(command), command)
override = f"{command} --allow-stale"
if " run" in command:
self.assertEqual(parse_command(override), override)
else:
self.assertIsNone(parse_command(override))
self.assertIsNone(parse_command("/ci run please"))
self.assertIsNone(parse_command("/ci run all please"))
@@ -331,6 +345,8 @@ class RunCiCommandTest(unittest.TestCase):
self.assertIsNone(parse_command("/amd-ci retry "))
self.assertIsNone(parse_command("/AMD-CI run"))
self.assertIsNone(parse_command("/amdci run"))
self.assertIsNone(parse_command("/ci run --allow-stale all"))
self.assertIsNone(parse_command("/amd-ci run --allow-stale please"))
def test_commands_select_only_their_configured_pipeline(self) -> None:
cases = (
@@ -530,6 +546,177 @@ class RunCiCommandTest(unittest.TestCase):
self.assertTrue(github.comments[0].startswith("✅ "))
self.assertIn("Buildkite CI #123", github.comments[0])
def test_run_variants_require_current_base_without_changing_commit(self) -> None:
for command in RUN_CI_COMMAND_ENV:
if command.endswith(" --allow-stale"):
continue
for behind in (0, 1, 5):
with self.subTest(command=command, behind=behind):
github = FakeGitHub(
behind=behind, pr=make_pr(base={"ref": "release/1.0"})
)
buildkite = FakeBuildkite([[], []])
run(make_event(command), github, buildkite)
self.assertEqual(
github.lag_queries,
[("release/1.0", github.pr["head"]["sha"])],
)
if behind == 0:
self.assertEqual(
buildkite.created_builds[0]["commit"],
github.pr["head"]["sha"],
)
self.assertIn(
"](https://buildkite.example/builds/123)",
github.comments[0],
)
else:
self.assertEqual(buildkite.created_builds, [])
self.assertTrue(github.comments[0].startswith("❌ "))
self.assertIn(str(behind), github.comments[0])
self.assertIn("upstream `release/1.0`", github.comments[0])
self.assertIn("every commit", github.comments[0])
self.assertIn(
"No new CI build was started.", github.comments[0]
)
self.assertIn(command, github.comments[0])
def test_allow_stale_preserves_author_permissions_and_run_configuration(
self,
) -> None:
cases = (
(COMMAND_RUN_CI, "ci", {}),
(COMMAND_RUN_CI_ALL, "ci", {"RUN_ALL": "1"}),
(COMMAND_RUN_CI_NIGHTLY, "ci", {"RUN_ALL": "1", "NIGHTLY": "1"}),
(COMMAND_RUN_AMD_CI, "amd-ci", {}),
(COMMAND_RUN_AMD_CI_ALL, "amd-ci", {"RUN_ALL": "1"}),
(
COMMAND_RUN_AMD_CI_NIGHTLY,
"amd-ci",
{"RUN_ALL": "1", "NIGHTLY": "1"},
),
)
for original_command, pipeline, expected_env in cases:
command = f"{original_command} --allow-stale"
for ready, behind in ((False, 100), (True, 0), (True, 100)):
with self.subTest(command=command, ready=ready, behind=behind):
github = FakeGitHub(
permission="read",
behind=behind,
pr=make_pr(
base={"ref": "feature/stack"},
labels=[{"name": "ready"}] if ready else [],
),
)
buildkite = FakeBuildkite([[], []])
run(make_event(command, "author"), github, buildkite)
self.assertEqual(pipeline_for_command(command), pipeline)
if not ready:
self.assertEqual(buildkite.created_builds, [])
self.assertEqual(buildkite.list_calls, [])
self.assertEqual(github.lag_queries, [])
self.assertTrue(github.comments[0].startswith("❌ "))
continue
self.assertEqual(len(buildkite.created_builds), 1)
payload = buildkite.created_builds[0]
self.assertEqual(payload["commit"], github.pr["head"]["sha"])
self.assertEqual(
payload["pull_request_base_branch"], "feature/stack"
)
self.assertEqual(payload["message"], f"PR #42 {command} by @author")
self.assertEqual(
payload["env"],
{
"VLLM_CI_GITHUB_COMMENT_ID": "99",
"VLLM_CI_TRIGGERED_BY": "author",
**expected_env,
},
)
self.assertEqual(
github.lag_queries,
[("feature/stack", github.pr["head"]["sha"])],
)
self.assertIn(
"Buildkite AMD CI #123"
if pipeline == "amd-ci"
else "Buildkite CI #123",
github.comments[0],
)
if behind:
self.assertIn("100 commits behind", github.comments[0])
self.assertIn("upstream `feature/stack`", github.comments[0])
self.assertIn("Before merging", github.comments[0])
self.assertIn("latest PR commit", github.comments[0])
self.assertIn(original_command, github.comments[0])
def test_base_lag_query_qualifies_branch_refs_and_validates_count(self) -> None:
for base_ref, encoded_ref in (
("main", "main"),
("release/1.0", "release%2F1.0"),
("feature/stack", "feature%2Fstack"),
):
for count in (0, 5, None, -1, True, "5"):
with self.subTest(base_ref=base_ref, count=count):
transport = FakeTransport(
{} if count is None else {"behind_by": count}
)
github = GitHubClient("token", "vllm-project/vllm", transport)
if type(count) is int and count >= 0:
self.assertEqual(
github.get_commits_behind_base(base_ref, "head-sha"), count
)
else:
with self.assertRaises(ApiError):
github.get_commits_behind_base(base_ref, "head-sha")
self.assertEqual(
transport.calls[0]["url"],
"https://api.github.com/repos/vllm-project/vllm/compare/"
f"refs%2Fheads%2F{encoded_ref}...head-sha?per_page=1",
)
self.assertEqual(transport.calls[0]["method"], "GET")
def test_unknown_lag_or_changed_pr_prevents_runs(self) -> None:
for command in (
COMMAND_RUN_CI,
COMMAND_RUN_AMD_CI,
f"{COMMAND_RUN_CI} --allow-stale",
f"{COMMAND_RUN_AMD_CI} --allow-stale",
):
for fault in ("api", "head", "closed", "base"):
with self.subTest(command=command, fault=fault):
github = FakeGitHub()
buildkite = FakeBuildkite([[], []])
def check(
base_ref: str,
sha: str,
fault: str = fault,
github: FakeGitHub = github,
) -> int:
if fault == "api":
raise ApiError(503, "Unavailable")
changed = make_pr()
if fault == "head":
changed["head"]["sha"] = "another-head"
elif fault == "closed":
changed["state"] = "closed"
else:
changed["base"]["ref"] = "another-base"
github.prs[42] = changed
return 0
with patch.object(
github, "get_commits_behind_base", side_effect=check
):
run(make_event(command), github, buildkite)
self.assertEqual(buildkite.created_builds, [])
self.assertEqual(buildkite.retry_calls, [])
self.assertTrue(github.comments[0].startswith("❌ "))
self.assertNotIn("Triggered", github.comments[0])
def test_amd_run_ignores_blocked_builds(self) -> None:
for metadata in ({}, {"github-comment-id": "98"}):
with self.subTest(metadata=metadata):
@@ -627,6 +814,7 @@ class RunCiCommandTest(unittest.TestCase):
run(make_event(COMMAND_RUN_CI, "author"), github, buildkite)
self.assertEqual(buildkite.list_calls, [])
self.assertEqual(github.lag_queries, [])
self.assertEqual(github.reactions, ["eyes"])
self.assertTrue(github.comments[0].startswith("❌ "))
self.assertIn("approve the PR", github.comments[0])
@@ -848,6 +1036,7 @@ class RunCiCommandTest(unittest.TestCase):
def test_ci_retry_retries_failed_jobs_while_build_is_running(self) -> None:
github = FakeGitHub(
behind=100,
permission="read",
pr=make_pr(labels=[{"name": "ready"}]),
)
@@ -867,10 +1056,12 @@ class RunCiCommandTest(unittest.TestCase):
run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite)
self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)])
self.assertEqual(github.lag_queries, [])
self.assertIn("Queued 3 failed job", github.comments[0])
def test_amd_ci_retry_retries_only_the_current_head_build(self) -> None:
github = FakeGitHub(
behind=100,
permission="read",
pr=make_pr(labels=[{"name": "ready"}]),
)
@@ -891,6 +1082,7 @@ class RunCiCommandTest(unittest.TestCase):
run(make_event(COMMAND_RETRY_AMD_FAILED, "author"), github, buildkite)
self.assertEqual(buildkite.retry_calls, [(321, RETRY_STATES)])
self.assertEqual(github.lag_queries, [])
self.assertIn("Buildkite AMD CI #321", github.comments[0])
def test_amd_ci_retry_requires_a_build_for_the_current_head(self) -> None:
@@ -912,6 +1104,7 @@ class RunCiCommandTest(unittest.TestCase):
def test_ci_retry_creates_filtered_build_for_new_head(self) -> None:
github = FakeGitHub(
behind=100,
permission="read",
pr=make_pr(labels=[{"name": "ready"}]),
)
@@ -953,6 +1146,7 @@ class RunCiCommandTest(unittest.TestCase):
self.assertEqual(len(buildkite.created_builds), 1)
payload = buildkite.created_builds[0]
self.assertEqual(payload["commit"], "0123456789abcdef")
self.assertEqual(github.lag_queries, [])
self.assertEqual(payload["message"], "PR #42 /ci retry by @author")
self.assertEqual(
json.loads(payload["env"]["VLLM_CI_ONLY_STEP_KEYS"]),
@@ -1104,6 +1298,7 @@ class RunCiCommandTest(unittest.TestCase):
run(make_event(COMMAND_CANCEL_CI), github, buildkite)
self.assertEqual(buildkite.cancel_calls, [123, 124, 126])
self.assertEqual(github.lag_queries, [])
self.assertEqual(
[request["branch"] for request in buildkite.list_requests],
["contributor:feature", "feature"],
@@ -1152,6 +1347,7 @@ class RunCiCommandTest(unittest.TestCase):
run(make_event(COMMAND_CANCEL_AMD_CI), github, buildkite)
self.assertEqual(buildkite.cancel_calls, [322, 321])
self.assertEqual(github.lag_queries, [])
self.assertEqual(
[request["branch"] for request in buildkite.list_requests],
["contributor:feature", "feature"],