[Bugfix][CI] Fix the flaky sharded-sampling tests, and the engine teardown need (#58342)
Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
Andreas Karatzas
parent
23110a0c9e
commit
c0c35ddd8e
@@ -2,26 +2,30 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Batch-sharded sampling must be a drop-in for replicated sampling.
|
||||
|
||||
The comparison is tolerance-based, not bitwise: engine boots land in
|
||||
slightly different kernel/collective states (measured: same-state boots are
|
||||
bitwise identical across the two modes; cross-state boots differ by up to
|
||||
~0.25 logprob and can flip a near-tie token). A real sharding bug — rows
|
||||
routed to the wrong request, misaligned gathers — produces many-nats
|
||||
divergence and scrambled top-k sets, far outside these bounds."""
|
||||
The two boots are compared token for token, which is only meaningful because
|
||||
both schedule the same batches. With the engine in the caller's process every
|
||||
request is queued before the first step, so the first prefill has the same
|
||||
shape in both runs. Give the engine its own process and the comparison turns
|
||||
into a coin toss: prefill starts before the last requests arrive over IPC, so
|
||||
one boot batches two requests where the other batches one. The batch shape
|
||||
sets the GEMM reduction order, logprobs shift by up to ~0.25, and near-ties
|
||||
flip — measured at up to 5 of 16 completions, with sharding disabled on both
|
||||
sides. A real sharding bug — rows routed to the wrong request, misaligned
|
||||
gathers — produces many-nats divergence and scrambled top-k sets."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.logprobs import Logprob, LogprobsOnePosition
|
||||
|
||||
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
|
||||
# Boot-state numeric noise bound (measured <= 0.25 on identical contexts).
|
||||
# Misrouted rows move logprobs by nats; anything under this is float noise.
|
||||
# With matched batches the two boots agreed bitwise over 320 positions, so the
|
||||
# bound is not expected to be reached.
|
||||
LOGPROB_TOL = 0.5
|
||||
# Near-tie greedy/seeded flips from boot-state noise; measured <= 1 per
|
||||
# engine boot across ~16 completions, plus one of headroom.
|
||||
MAX_DIVERGENT_PROMPTS = 3
|
||||
|
||||
PROMPTS = [
|
||||
"The capital of France is",
|
||||
@@ -54,6 +58,10 @@ def _is_sharded_sampling_active(worker) -> bool:
|
||||
return worker.model_runner.batch_sharder is not None
|
||||
|
||||
|
||||
def _sampling_params() -> list[SamplingParams]:
|
||||
return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))]
|
||||
|
||||
|
||||
def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
@@ -75,9 +83,7 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
|
||||
# intended sampling mode.
|
||||
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
|
||||
assert all(mode == (not disable_sharding) for mode in modes), modes
|
||||
params = [
|
||||
SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))
|
||||
]
|
||||
params = _sampling_params()
|
||||
# Two waves: the second reuses the request slots freed by the first
|
||||
# (in whatever order requests finished), covering the cross-rank
|
||||
# slot-recycling path that request ownership derives from.
|
||||
@@ -89,13 +95,36 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
|
||||
# Allow one boundary entry of the top-k set to swap at a near-tie.
|
||||
common = set(ref_lps) & set(out_lps)
|
||||
assert len(common) >= max(len(ref_lps), len(out_lps)) - 1, (
|
||||
f"{what}: top-k token sets diverge: {sorted(ref_lps)} vs {sorted(out_lps)}"
|
||||
def _requested_top_k(lps: LogprobsOnePosition, num_logprobs: int) -> set[int]:
|
||||
"""Exclude the extra sampled/prompt token, including at rank k+1."""
|
||||
top = {
|
||||
token_id
|
||||
for token_id, logprob in lps.items()
|
||||
if logprob.rank is not None and 1 <= logprob.rank <= num_logprobs
|
||||
}
|
||||
assert len(top) == num_logprobs, (
|
||||
f"expected {num_logprobs} ranked top-k entries, got {len(top)}"
|
||||
)
|
||||
for token_id in common:
|
||||
return top
|
||||
|
||||
|
||||
def _assert_logprob_dicts_close(
|
||||
ref_lps: LogprobsOnePosition,
|
||||
out_lps: LogprobsOnePosition,
|
||||
what: str,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
# Allow one boundary entry of the top-k set to swap at a near-tie. Compare
|
||||
# the top-k alone: counting the sampled token would spend that allowance
|
||||
# on the extra entry itself whenever only one side carries one.
|
||||
ref_top = _requested_top_k(ref_lps, num_logprobs)
|
||||
out_top = _requested_top_k(out_lps, num_logprobs)
|
||||
shared_top = ref_top & out_top
|
||||
assert len(shared_top) >= num_logprobs - 1, (
|
||||
f"{what}: top-k token sets diverge: {sorted(ref_top)} vs {sorted(out_top)}"
|
||||
)
|
||||
# Values are comparable for every shared token, sampled one included.
|
||||
for token_id in set(ref_lps) & set(out_lps):
|
||||
diff = abs(ref_lps[token_id].logprob - out_lps[token_id].logprob)
|
||||
assert diff <= LOGPROB_TOL, (
|
||||
f"{what}[{token_id}]: logprob diff {diff} "
|
||||
@@ -103,20 +132,72 @@ def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("num_logprobs", "ref_ranks", "out_ranks"),
|
||||
[
|
||||
(2, {0: 1, 1: 2, 2: 3}, {0: 1, 3: 2, 2: 4}),
|
||||
(1, {0: 1, 1: 2}, {2: 1, 1: 3}),
|
||||
(2, {0: 1, 1: 2}, {0: 1, 2: 2, 1: 3}),
|
||||
(2, {0: 1, 1: 2, 2: 4}, {0: 1, 1: 2, 2: 5}),
|
||||
(0, {0: 2}, {0: 3}),
|
||||
],
|
||||
ids=["decode-boundary", "prompt-boundary", "sampled-in-top-k", "gap", "zero"],
|
||||
)
|
||||
def test_logprob_comparison_allows_sampled_token_rank_changes(
|
||||
num_logprobs: int, ref_ranks: dict[int, int], out_ranks: dict[int, int]
|
||||
):
|
||||
ref_lps, out_lps = (
|
||||
{token: Logprob(-2.0 - rank / 100, rank=rank) for token, rank in ranks.items()}
|
||||
for ranks in (ref_ranks, out_ranks)
|
||||
)
|
||||
_assert_logprob_dicts_close(ref_lps, out_lps, "boundary swap", num_logprobs)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_logprob_comparison_rejects_multiple_top_k_swaps():
|
||||
ref = {0: Logprob(-2.0, rank=1), 1: Logprob(-2.01, rank=2)}
|
||||
out = {2: Logprob(-2.0, rank=1), 3: Logprob(-2.01, rank=2)}
|
||||
with pytest.raises(AssertionError, match="top-k token sets diverge"):
|
||||
_assert_logprob_dicts_close(ref, out, "misrouted top-k", num_logprobs=2)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_logprob_comparison_checks_extra_sampled_token_value():
|
||||
ref = {0: Logprob(-1.0, rank=1), 1: Logprob(-3.0, rank=3)}
|
||||
out = {0: Logprob(-1.0, rank=1), 1: Logprob(-4.0, rank=3)}
|
||||
with pytest.raises(AssertionError, match="logprob diff"):
|
||||
_assert_logprob_dicts_close(ref, out, "sampled token", num_logprobs=1)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("rank", [None, 2])
|
||||
def test_logprob_comparison_rejects_missing_top_k_rank(rank: int | None):
|
||||
lps = {0: Logprob(-1.0, rank=rank)}
|
||||
with pytest.raises(AssertionError, match="expected 1 ranked top-k entries"):
|
||||
_assert_logprob_dicts_close(lps, lps, "missing rank", num_logprobs=1)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Generation, logprobs, and prompt logprobs match between batch-sharded
|
||||
sampling and the replicated fallback, up to measured boot-state noise."""
|
||||
sampling and the replicated fallback."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
# Required for the collective_rpc mode check below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
# Queue every request before the first step, so both boots schedule the
|
||||
# same batches -- see the module docstring.
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
ref_outputs = _generate(monkeypatch, disable_sharding=True)
|
||||
shard_outputs = _generate(monkeypatch, disable_sharding=False)
|
||||
|
||||
assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS)
|
||||
num_divergent = 0
|
||||
for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)):
|
||||
params = _sampling_params()
|
||||
divergent = []
|
||||
for i, (ref, out, sampling_params) in enumerate(
|
||||
zip(ref_outputs, shard_outputs, params + params[::-1])
|
||||
):
|
||||
ref_completion = ref.outputs[0]
|
||||
out_completion = out.outputs[0]
|
||||
ref_ids = list(ref_completion.token_ids)
|
||||
@@ -131,21 +212,24 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
|
||||
break
|
||||
prefix += 1
|
||||
if ref_ids != out_ids:
|
||||
num_divergent += 1
|
||||
divergent.append((i, prefix))
|
||||
|
||||
if ref_completion.logprobs is not None:
|
||||
assert out_completion.logprobs is not None
|
||||
assert sampling_params.logprobs is not None
|
||||
comparable = min(prefix + 1, len(ref_ids), len(out_ids))
|
||||
for pos in range(comparable):
|
||||
_assert_logprob_dicts_close(
|
||||
ref_completion.logprobs[pos],
|
||||
out_completion.logprobs[pos],
|
||||
f"prompt {i} logprobs[{pos}]",
|
||||
sampling_params.logprobs,
|
||||
)
|
||||
|
||||
# The prompt is fixed, so every prompt-logprob position is comparable.
|
||||
if ref.prompt_logprobs is not None:
|
||||
assert out.prompt_logprobs is not None
|
||||
assert sampling_params.prompt_logprobs is not None
|
||||
for pos, (ref_lps, out_lps) in enumerate(
|
||||
zip(ref.prompt_logprobs, out.prompt_logprobs)
|
||||
):
|
||||
@@ -153,10 +237,14 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
|
||||
assert ref_lps is None and out_lps is None
|
||||
continue
|
||||
_assert_logprob_dicts_close(
|
||||
ref_lps, out_lps, f"prompt {i} prompt_logprobs[{pos}]"
|
||||
ref_lps,
|
||||
out_lps,
|
||||
f"prompt {i} prompt_logprobs[{pos}]",
|
||||
sampling_params.prompt_logprobs,
|
||||
)
|
||||
|
||||
assert num_divergent <= MAX_DIVERGENT_PROMPTS, (
|
||||
f"{num_divergent}/{2 * len(PROMPTS)} prompts diverged: beyond near-tie "
|
||||
"boot-state noise; sharded sampling is likely misrouting requests"
|
||||
assert not divergent, (
|
||||
f"{len(divergent)}/{2 * len(PROMPTS)} completions diverged at "
|
||||
f"(prompt, position) {divergent}; the sampling modes did not agree "
|
||||
"token for token with in-process scheduling"
|
||||
)
|
||||
|
||||
@@ -39,6 +39,9 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
|
||||
m.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
# Required for the collective_rpc mode check below.
|
||||
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
# Queue every request before the first step, so both boots schedule
|
||||
# the same batches -- see tests/v1/e2e/general/test_sharded_sampling.py.
|
||||
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
def run(disable_sharding: bool):
|
||||
llm = LLM(
|
||||
@@ -79,24 +82,22 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
|
||||
ref_greedy, ref_seeded = run(disable_sharding=True)
|
||||
shard_greedy, shard_seeded = run(disable_sharding=False)
|
||||
|
||||
# Engine boots are not bitwise deterministic (kernel/collective
|
||||
# selection shifts logits by ~1 ulp, flipping near-tie tokens), so
|
||||
# allow a small number of divergent prompts. A sharding bug produces
|
||||
# wholesale divergence, not isolated near-tie flips.
|
||||
# A sharding bug shows up as wholesale divergence, not isolated flips.
|
||||
for name, ref_outputs, shard_outputs in (
|
||||
("greedy", ref_greedy, shard_greedy),
|
||||
("seeded", ref_seeded, shard_seeded),
|
||||
):
|
||||
num_divergent = 0
|
||||
divergent = []
|
||||
for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)):
|
||||
if list(ref.outputs[0].token_ids) != list(out.outputs[0].token_ids):
|
||||
num_divergent += 1
|
||||
divergent.append(i)
|
||||
print(
|
||||
f"{name} prompt {i} diverged:\n"
|
||||
f" replicated: {ref.outputs[0].text!r}\n"
|
||||
f" sharded: {out.outputs[0].text!r}"
|
||||
)
|
||||
assert num_divergent <= 2, (
|
||||
f"{name}: {num_divergent}/{len(ref_outputs)} prompts diverged, "
|
||||
"beyond near-tie boot noise"
|
||||
assert not divergent, (
|
||||
f"{name}: {len(divergent)}/{len(ref_outputs)} prompts diverged "
|
||||
f"at {divergent}; the sampling modes did not agree token for "
|
||||
"token with in-process scheduling"
|
||||
)
|
||||
|
||||
@@ -366,6 +366,11 @@ class InprocClient(EngineCoreClient):
|
||||
log_stats,
|
||||
executor_fail_callback=executor_fail_callback,
|
||||
)
|
||||
# Release the executor's workers when the client is collected, as
|
||||
# MPClient does; they hold GPU memory for the life of the process
|
||||
# otherwise. The callback must not reference self, or self would stay
|
||||
# reachable and the finalizer would never run.
|
||||
self._finalizer = weakref.finalize(self, self.engine_core.shutdown)
|
||||
|
||||
def get_output(self) -> EngineCoreOutputs:
|
||||
outputs, model_executed = self.engine_core.step_fn()
|
||||
@@ -384,7 +389,8 @@ class InprocClient(EngineCoreClient):
|
||||
self.engine_core.abort_requests(request_ids)
|
||||
|
||||
def shutdown(self, timeout: float | None = None) -> None:
|
||||
self.engine_core.shutdown()
|
||||
if self._finalizer.detach() is not None:
|
||||
self.engine_core.shutdown()
|
||||
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None) -> None:
|
||||
self.engine_core.profile(is_start, profile_prefix)
|
||||
|
||||
Reference in New Issue
Block a user