[Bugfix][CI] Fix the flaky sharded-sampling tests, and the engine teardown need (#58342)

Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
stefankoncarevic
2026-09-24 02:05:11 -07:00
committed by GitHub
co-authored by Andreas Karatzas Andreas Karatzas
parent 23110a0c9e
commit c0c35ddd8e
3 changed files with 132 additions and 37 deletions
+115 -27
View File
@@ -2,26 +2,30 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Batch-sharded sampling must be a drop-in for replicated sampling.
The comparison is tolerance-based, not bitwise: engine boots land in
slightly different kernel/collective states (measured: same-state boots are
bitwise identical across the two modes; cross-state boots differ by up to
~0.25 logprob and can flip a near-tie token). A real sharding bug — rows
routed to the wrong request, misaligned gathers — produces many-nats
divergence and scrambled top-k sets, far outside these bounds."""
The two boots are compared token for token, which is only meaningful because
both schedule the same batches. With the engine in the caller's process every
request is queued before the first step, so the first prefill has the same
shape in both runs. Give the engine its own process and the comparison turns
into a coin toss: prefill starts before the last requests arrive over IPC, so
one boot batches two requests where the other batches one. The batch shape
sets the GEMM reduction order, logprobs shift by up to ~0.25, and near-ties
flip — measured at up to 5 of 16 completions, with sharding disabled on both
sides. A real sharding bug — rows routed to the wrong request, misaligned
gathers — produces many-nats divergence and scrambled top-k sets."""
import pytest
from tests.utils import multi_gpu_test
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.logprobs import Logprob, LogprobsOnePosition
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
# Boot-state numeric noise bound (measured <= 0.25 on identical contexts).
# Misrouted rows move logprobs by nats; anything under this is float noise.
# With matched batches the two boots agreed bitwise over 320 positions, so the
# bound is not expected to be reached.
LOGPROB_TOL = 0.5
# Near-tie greedy/seeded flips from boot-state noise; measured <= 1 per
# engine boot across ~16 completions, plus one of headroom.
MAX_DIVERGENT_PROMPTS = 3
PROMPTS = [
"The capital of France is",
@@ -54,6 +58,10 @@ def _is_sharded_sampling_active(worker) -> bool:
return worker.model_runner.batch_sharder is not None
def _sampling_params() -> list[SamplingParams]:
return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))]
def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
llm = LLM(
model=MODEL,
@@ -75,9 +83,7 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
# intended sampling mode.
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
assert all(mode == (not disable_sharding) for mode in modes), modes
params = [
SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))
]
params = _sampling_params()
# Two waves: the second reuses the request slots freed by the first
# (in whatever order requests finished), covering the cross-rank
# slot-recycling path that request ownership derives from.
@@ -89,13 +95,36 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
cleanup_dist_env_and_memory()
def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
# Allow one boundary entry of the top-k set to swap at a near-tie.
common = set(ref_lps) & set(out_lps)
assert len(common) >= max(len(ref_lps), len(out_lps)) - 1, (
f"{what}: top-k token sets diverge: {sorted(ref_lps)} vs {sorted(out_lps)}"
def _requested_top_k(lps: LogprobsOnePosition, num_logprobs: int) -> set[int]:
"""Exclude the extra sampled/prompt token, including at rank k+1."""
top = {
token_id
for token_id, logprob in lps.items()
if logprob.rank is not None and 1 <= logprob.rank <= num_logprobs
}
assert len(top) == num_logprobs, (
f"expected {num_logprobs} ranked top-k entries, got {len(top)}"
)
for token_id in common:
return top
def _assert_logprob_dicts_close(
ref_lps: LogprobsOnePosition,
out_lps: LogprobsOnePosition,
what: str,
num_logprobs: int,
) -> None:
# Allow one boundary entry of the top-k set to swap at a near-tie. Compare
# the top-k alone: counting the sampled token would spend that allowance
# on the extra entry itself whenever only one side carries one.
ref_top = _requested_top_k(ref_lps, num_logprobs)
out_top = _requested_top_k(out_lps, num_logprobs)
shared_top = ref_top & out_top
assert len(shared_top) >= num_logprobs - 1, (
f"{what}: top-k token sets diverge: {sorted(ref_top)} vs {sorted(out_top)}"
)
# Values are comparable for every shared token, sampled one included.
for token_id in set(ref_lps) & set(out_lps):
diff = abs(ref_lps[token_id].logprob - out_lps[token_id].logprob)
assert diff <= LOGPROB_TOL, (
f"{what}[{token_id}]: logprob diff {diff} "
@@ -103,20 +132,72 @@ def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("num_logprobs", "ref_ranks", "out_ranks"),
[
(2, {0: 1, 1: 2, 2: 3}, {0: 1, 3: 2, 2: 4}),
(1, {0: 1, 1: 2}, {2: 1, 1: 3}),
(2, {0: 1, 1: 2}, {0: 1, 2: 2, 1: 3}),
(2, {0: 1, 1: 2, 2: 4}, {0: 1, 1: 2, 2: 5}),
(0, {0: 2}, {0: 3}),
],
ids=["decode-boundary", "prompt-boundary", "sampled-in-top-k", "gap", "zero"],
)
def test_logprob_comparison_allows_sampled_token_rank_changes(
num_logprobs: int, ref_ranks: dict[int, int], out_ranks: dict[int, int]
):
ref_lps, out_lps = (
{token: Logprob(-2.0 - rank / 100, rank=rank) for token, rank in ranks.items()}
for ranks in (ref_ranks, out_ranks)
)
_assert_logprob_dicts_close(ref_lps, out_lps, "boundary swap", num_logprobs)
@pytest.mark.cpu_test
def test_logprob_comparison_rejects_multiple_top_k_swaps():
ref = {0: Logprob(-2.0, rank=1), 1: Logprob(-2.01, rank=2)}
out = {2: Logprob(-2.0, rank=1), 3: Logprob(-2.01, rank=2)}
with pytest.raises(AssertionError, match="top-k token sets diverge"):
_assert_logprob_dicts_close(ref, out, "misrouted top-k", num_logprobs=2)
@pytest.mark.cpu_test
def test_logprob_comparison_checks_extra_sampled_token_value():
ref = {0: Logprob(-1.0, rank=1), 1: Logprob(-3.0, rank=3)}
out = {0: Logprob(-1.0, rank=1), 1: Logprob(-4.0, rank=3)}
with pytest.raises(AssertionError, match="logprob diff"):
_assert_logprob_dicts_close(ref, out, "sampled token", num_logprobs=1)
@pytest.mark.cpu_test
@pytest.mark.parametrize("rank", [None, 2])
def test_logprob_comparison_rejects_missing_top_k_rank(rank: int | None):
lps = {0: Logprob(-1.0, rank=rank)}
with pytest.raises(AssertionError, match="expected 1 ranked top-k entries"):
_assert_logprob_dicts_close(lps, lps, "missing rank", num_logprobs=1)
@multi_gpu_test(num_gpus=2)
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
"""Generation, logprobs, and prompt logprobs match between batch-sharded
sampling and the replicated fallback, up to measured boot-state noise."""
sampling and the replicated fallback."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
# Queue every request before the first step, so both boots schedule the
# same batches -- see the module docstring.
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
ref_outputs = _generate(monkeypatch, disable_sharding=True)
shard_outputs = _generate(monkeypatch, disable_sharding=False)
assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS)
num_divergent = 0
for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)):
params = _sampling_params()
divergent = []
for i, (ref, out, sampling_params) in enumerate(
zip(ref_outputs, shard_outputs, params + params[::-1])
):
ref_completion = ref.outputs[0]
out_completion = out.outputs[0]
ref_ids = list(ref_completion.token_ids)
@@ -131,21 +212,24 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
break
prefix += 1
if ref_ids != out_ids:
num_divergent += 1
divergent.append((i, prefix))
if ref_completion.logprobs is not None:
assert out_completion.logprobs is not None
assert sampling_params.logprobs is not None
comparable = min(prefix + 1, len(ref_ids), len(out_ids))
for pos in range(comparable):
_assert_logprob_dicts_close(
ref_completion.logprobs[pos],
out_completion.logprobs[pos],
f"prompt {i} logprobs[{pos}]",
sampling_params.logprobs,
)
# The prompt is fixed, so every prompt-logprob position is comparable.
if ref.prompt_logprobs is not None:
assert out.prompt_logprobs is not None
assert sampling_params.prompt_logprobs is not None
for pos, (ref_lps, out_lps) in enumerate(
zip(ref.prompt_logprobs, out.prompt_logprobs)
):
@@ -153,10 +237,14 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
assert ref_lps is None and out_lps is None
continue
_assert_logprob_dicts_close(
ref_lps, out_lps, f"prompt {i} prompt_logprobs[{pos}]"
ref_lps,
out_lps,
f"prompt {i} prompt_logprobs[{pos}]",
sampling_params.prompt_logprobs,
)
assert num_divergent <= MAX_DIVERGENT_PROMPTS, (
f"{num_divergent}/{2 * len(PROMPTS)} prompts diverged: beyond near-tie "
"boot-state noise; sharded sampling is likely misrouting requests"
assert not divergent, (
f"{len(divergent)}/{2 * len(PROMPTS)} completions diverged at "
f"(prompt, position) {divergent}; the sampling modes did not agree "
"token for token with in-process scheduling"
)
@@ -39,6 +39,9 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
m.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
# Queue every request before the first step, so both boots schedule
# the same batches -- see tests/v1/e2e/general/test_sharded_sampling.py.
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
def run(disable_sharding: bool):
llm = LLM(
@@ -79,24 +82,22 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
ref_greedy, ref_seeded = run(disable_sharding=True)
shard_greedy, shard_seeded = run(disable_sharding=False)
# Engine boots are not bitwise deterministic (kernel/collective
# selection shifts logits by ~1 ulp, flipping near-tie tokens), so
# allow a small number of divergent prompts. A sharding bug produces
# wholesale divergence, not isolated near-tie flips.
# A sharding bug shows up as wholesale divergence, not isolated flips.
for name, ref_outputs, shard_outputs in (
("greedy", ref_greedy, shard_greedy),
("seeded", ref_seeded, shard_seeded),
):
num_divergent = 0
divergent = []
for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)):
if list(ref.outputs[0].token_ids) != list(out.outputs[0].token_ids):
num_divergent += 1
divergent.append(i)
print(
f"{name} prompt {i} diverged:\n"
f" replicated: {ref.outputs[0].text!r}\n"
f" sharded: {out.outputs[0].text!r}"
)
assert num_divergent <= 2, (
f"{name}: {num_divergent}/{len(ref_outputs)} prompts diverged, "
"beyond near-tie boot noise"
assert not divergent, (
f"{name}: {len(divergent)}/{len(ref_outputs)} prompts diverged "
f"at {divergent}; the sampling modes did not agree token for "
"token with in-process scheduling"
)
+7 -1
View File
@@ -366,6 +366,11 @@ class InprocClient(EngineCoreClient):
log_stats,
executor_fail_callback=executor_fail_callback,
)
# Release the executor's workers when the client is collected, as
# MPClient does; they hold GPU memory for the life of the process
# otherwise. The callback must not reference self, or self would stay
# reachable and the finalizer would never run.
self._finalizer = weakref.finalize(self, self.engine_core.shutdown)
def get_output(self) -> EngineCoreOutputs:
outputs, model_executed = self.engine_core.step_fn()
@@ -384,7 +389,8 @@ class InprocClient(EngineCoreClient):
self.engine_core.abort_requests(request_ids)
def shutdown(self, timeout: float | None = None) -> None:
self.engine_core.shutdown()
if self._finalizer.detach() is not None:
self.engine_core.shutdown()
def profile(self, is_start: bool = True, profile_prefix: str | None = None) -> None:
self.engine_core.profile(is_start, profile_prefix)