From c0c35ddd8e9c6dadd53a8f92e20e9a03a210da15 Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Thu, 24 Sep 2026 11:05:11 +0200 Subject: [PATCH] [Bugfix][CI] Fix the flaky sharded-sampling tests, and the engine teardown need (#58342) Signed-off-by: Stefan Koncarevic Signed-off-by: Andreas Karatzas Co-authored-by: Andreas Karatzas Co-authored-by: Andreas Karatzas --- tests/v1/e2e/general/test_sharded_sampling.py | 142 ++++++++++++++---- .../e2e/spec_decode/test_sharded_sampling.py | 19 +-- vllm/v1/engine/core_client.py | 8 +- 3 files changed, 132 insertions(+), 37 deletions(-) diff --git a/tests/v1/e2e/general/test_sharded_sampling.py b/tests/v1/e2e/general/test_sharded_sampling.py index a580605fb0..8686186776 100644 --- a/tests/v1/e2e/general/test_sharded_sampling.py +++ b/tests/v1/e2e/general/test_sharded_sampling.py @@ -2,26 +2,30 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Batch-sharded sampling must be a drop-in for replicated sampling. -The comparison is tolerance-based, not bitwise: engine boots land in -slightly different kernel/collective states (measured: same-state boots are -bitwise identical across the two modes; cross-state boots differ by up to -~0.25 logprob and can flip a near-tie token). A real sharding bug — rows -routed to the wrong request, misaligned gathers — produces many-nats -divergence and scrambled top-k sets, far outside these bounds.""" +The two boots are compared token for token, which is only meaningful because +both schedule the same batches. With the engine in the caller's process every +request is queued before the first step, so the first prefill has the same +shape in both runs. Give the engine its own process and the comparison turns +into a coin toss: prefill starts before the last requests arrive over IPC, so +one boot batches two requests where the other batches one. The batch shape +sets the GEMM reduction order, logprobs shift by up to ~0.25, and near-ties +flip — measured at up to 5 of 16 completions, with sharding disabled on both +sides. A real sharding bug — rows routed to the wrong request, misaligned +gathers — produces many-nats divergence and scrambled top-k sets.""" import pytest from tests.utils import multi_gpu_test from vllm import LLM, SamplingParams from vllm.distributed import cleanup_dist_env_and_memory +from vllm.logprobs import Logprob, LogprobsOnePosition MODEL = "meta-llama/Llama-3.2-1B-Instruct" -# Boot-state numeric noise bound (measured <= 0.25 on identical contexts). +# Misrouted rows move logprobs by nats; anything under this is float noise. +# With matched batches the two boots agreed bitwise over 320 positions, so the +# bound is not expected to be reached. LOGPROB_TOL = 0.5 -# Near-tie greedy/seeded flips from boot-state noise; measured <= 1 per -# engine boot across ~16 completions, plus one of headroom. -MAX_DIVERGENT_PROMPTS = 3 PROMPTS = [ "The capital of France is", @@ -54,6 +58,10 @@ def _is_sharded_sampling_active(worker) -> bool: return worker.model_runner.batch_sharder is not None +def _sampling_params() -> list[SamplingParams]: + return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))] + + def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool): llm = LLM( model=MODEL, @@ -75,9 +83,7 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool): # intended sampling mode. modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active) assert all(mode == (not disable_sharding) for mode in modes), modes - params = [ - SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS)) - ] + params = _sampling_params() # Two waves: the second reuses the request slots freed by the first # (in whatever order requests finished), covering the cross-rank # slot-recycling path that request ownership derives from. @@ -89,13 +95,36 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool): cleanup_dist_env_and_memory() -def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None: - # Allow one boundary entry of the top-k set to swap at a near-tie. - common = set(ref_lps) & set(out_lps) - assert len(common) >= max(len(ref_lps), len(out_lps)) - 1, ( - f"{what}: top-k token sets diverge: {sorted(ref_lps)} vs {sorted(out_lps)}" +def _requested_top_k(lps: LogprobsOnePosition, num_logprobs: int) -> set[int]: + """Exclude the extra sampled/prompt token, including at rank k+1.""" + top = { + token_id + for token_id, logprob in lps.items() + if logprob.rank is not None and 1 <= logprob.rank <= num_logprobs + } + assert len(top) == num_logprobs, ( + f"expected {num_logprobs} ranked top-k entries, got {len(top)}" ) - for token_id in common: + return top + + +def _assert_logprob_dicts_close( + ref_lps: LogprobsOnePosition, + out_lps: LogprobsOnePosition, + what: str, + num_logprobs: int, +) -> None: + # Allow one boundary entry of the top-k set to swap at a near-tie. Compare + # the top-k alone: counting the sampled token would spend that allowance + # on the extra entry itself whenever only one side carries one. + ref_top = _requested_top_k(ref_lps, num_logprobs) + out_top = _requested_top_k(out_lps, num_logprobs) + shared_top = ref_top & out_top + assert len(shared_top) >= num_logprobs - 1, ( + f"{what}: top-k token sets diverge: {sorted(ref_top)} vs {sorted(out_top)}" + ) + # Values are comparable for every shared token, sampled one included. + for token_id in set(ref_lps) & set(out_lps): diff = abs(ref_lps[token_id].logprob - out_lps[token_id].logprob) assert diff <= LOGPROB_TOL, ( f"{what}[{token_id}]: logprob diff {diff} " @@ -103,20 +132,72 @@ def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None: ) +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("num_logprobs", "ref_ranks", "out_ranks"), + [ + (2, {0: 1, 1: 2, 2: 3}, {0: 1, 3: 2, 2: 4}), + (1, {0: 1, 1: 2}, {2: 1, 1: 3}), + (2, {0: 1, 1: 2}, {0: 1, 2: 2, 1: 3}), + (2, {0: 1, 1: 2, 2: 4}, {0: 1, 1: 2, 2: 5}), + (0, {0: 2}, {0: 3}), + ], + ids=["decode-boundary", "prompt-boundary", "sampled-in-top-k", "gap", "zero"], +) +def test_logprob_comparison_allows_sampled_token_rank_changes( + num_logprobs: int, ref_ranks: dict[int, int], out_ranks: dict[int, int] +): + ref_lps, out_lps = ( + {token: Logprob(-2.0 - rank / 100, rank=rank) for token, rank in ranks.items()} + for ranks in (ref_ranks, out_ranks) + ) + _assert_logprob_dicts_close(ref_lps, out_lps, "boundary swap", num_logprobs) + + +@pytest.mark.cpu_test +def test_logprob_comparison_rejects_multiple_top_k_swaps(): + ref = {0: Logprob(-2.0, rank=1), 1: Logprob(-2.01, rank=2)} + out = {2: Logprob(-2.0, rank=1), 3: Logprob(-2.01, rank=2)} + with pytest.raises(AssertionError, match="top-k token sets diverge"): + _assert_logprob_dicts_close(ref, out, "misrouted top-k", num_logprobs=2) + + +@pytest.mark.cpu_test +def test_logprob_comparison_checks_extra_sampled_token_value(): + ref = {0: Logprob(-1.0, rank=1), 1: Logprob(-3.0, rank=3)} + out = {0: Logprob(-1.0, rank=1), 1: Logprob(-4.0, rank=3)} + with pytest.raises(AssertionError, match="logprob diff"): + _assert_logprob_dicts_close(ref, out, "sampled token", num_logprobs=1) + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("rank", [None, 2]) +def test_logprob_comparison_rejects_missing_top_k_rank(rank: int | None): + lps = {0: Logprob(-1.0, rank=rank)} + with pytest.raises(AssertionError, match="expected 1 ranked top-k entries"): + _assert_logprob_dicts_close(lps, lps, "missing rank", num_logprobs=1) + + @multi_gpu_test(num_gpus=2) def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch): """Generation, logprobs, and prompt logprobs match between batch-sharded - sampling and the replicated fallback, up to measured boot-state noise.""" + sampling and the replicated fallback.""" monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") # Required for the collective_rpc mode check below. monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + # Queue every request before the first step, so both boots schedule the + # same batches -- see the module docstring. + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") ref_outputs = _generate(monkeypatch, disable_sharding=True) shard_outputs = _generate(monkeypatch, disable_sharding=False) assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS) - num_divergent = 0 - for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)): + params = _sampling_params() + divergent = [] + for i, (ref, out, sampling_params) in enumerate( + zip(ref_outputs, shard_outputs, params + params[::-1]) + ): ref_completion = ref.outputs[0] out_completion = out.outputs[0] ref_ids = list(ref_completion.token_ids) @@ -131,21 +212,24 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch): break prefix += 1 if ref_ids != out_ids: - num_divergent += 1 + divergent.append((i, prefix)) if ref_completion.logprobs is not None: assert out_completion.logprobs is not None + assert sampling_params.logprobs is not None comparable = min(prefix + 1, len(ref_ids), len(out_ids)) for pos in range(comparable): _assert_logprob_dicts_close( ref_completion.logprobs[pos], out_completion.logprobs[pos], f"prompt {i} logprobs[{pos}]", + sampling_params.logprobs, ) # The prompt is fixed, so every prompt-logprob position is comparable. if ref.prompt_logprobs is not None: assert out.prompt_logprobs is not None + assert sampling_params.prompt_logprobs is not None for pos, (ref_lps, out_lps) in enumerate( zip(ref.prompt_logprobs, out.prompt_logprobs) ): @@ -153,10 +237,14 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch): assert ref_lps is None and out_lps is None continue _assert_logprob_dicts_close( - ref_lps, out_lps, f"prompt {i} prompt_logprobs[{pos}]" + ref_lps, + out_lps, + f"prompt {i} prompt_logprobs[{pos}]", + sampling_params.prompt_logprobs, ) - assert num_divergent <= MAX_DIVERGENT_PROMPTS, ( - f"{num_divergent}/{2 * len(PROMPTS)} prompts diverged: beyond near-tie " - "boot-state noise; sharded sampling is likely misrouting requests" + assert not divergent, ( + f"{len(divergent)}/{2 * len(PROMPTS)} completions diverged at " + f"(prompt, position) {divergent}; the sampling modes did not agree " + "token for token with in-process scheduling" ) diff --git a/tests/v1/e2e/spec_decode/test_sharded_sampling.py b/tests/v1/e2e/spec_decode/test_sharded_sampling.py index cc9781ca9f..8f58b21a1a 100644 --- a/tests/v1/e2e/spec_decode/test_sharded_sampling.py +++ b/tests/v1/e2e/spec_decode/test_sharded_sampling.py @@ -39,6 +39,9 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch): m.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") # Required for the collective_rpc mode check below. m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + # Queue every request before the first step, so both boots schedule + # the same batches -- see tests/v1/e2e/general/test_sharded_sampling.py. + m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") def run(disable_sharding: bool): llm = LLM( @@ -79,24 +82,22 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch): ref_greedy, ref_seeded = run(disable_sharding=True) shard_greedy, shard_seeded = run(disable_sharding=False) - # Engine boots are not bitwise deterministic (kernel/collective - # selection shifts logits by ~1 ulp, flipping near-tie tokens), so - # allow a small number of divergent prompts. A sharding bug produces - # wholesale divergence, not isolated near-tie flips. + # A sharding bug shows up as wholesale divergence, not isolated flips. for name, ref_outputs, shard_outputs in ( ("greedy", ref_greedy, shard_greedy), ("seeded", ref_seeded, shard_seeded), ): - num_divergent = 0 + divergent = [] for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)): if list(ref.outputs[0].token_ids) != list(out.outputs[0].token_ids): - num_divergent += 1 + divergent.append(i) print( f"{name} prompt {i} diverged:\n" f" replicated: {ref.outputs[0].text!r}\n" f" sharded: {out.outputs[0].text!r}" ) - assert num_divergent <= 2, ( - f"{name}: {num_divergent}/{len(ref_outputs)} prompts diverged, " - "beyond near-tie boot noise" + assert not divergent, ( + f"{name}: {len(divergent)}/{len(ref_outputs)} prompts diverged " + f"at {divergent}; the sampling modes did not agree token for " + "token with in-process scheduling" ) diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 6ad05e1f8c..5d59bdeaea 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -366,6 +366,11 @@ class InprocClient(EngineCoreClient): log_stats, executor_fail_callback=executor_fail_callback, ) + # Release the executor's workers when the client is collected, as + # MPClient does; they hold GPU memory for the life of the process + # otherwise. The callback must not reference self, or self would stay + # reachable and the finalizer would never run. + self._finalizer = weakref.finalize(self, self.engine_core.shutdown) def get_output(self) -> EngineCoreOutputs: outputs, model_executed = self.engine_core.step_fn() @@ -384,7 +389,8 @@ class InprocClient(EngineCoreClient): self.engine_core.abort_requests(request_ids) def shutdown(self, timeout: float | None = None) -> None: - self.engine_core.shutdown() + if self._finalizer.detach() is not None: + self.engine_core.shutdown() def profile(self, is_start: bool = True, profile_prefix: str | None = None) -> None: self.engine_core.profile(is_start, profile_prefix)