[Model Runner V2] batch-sharded sample (#50465)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-08-24 15:49:57 -07:00
committed by GitHub
parent 0d7d5ed0b2
commit d154d90d6c
19 changed files with 1781 additions and 22 deletions
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Batch-sharded sampling must be a drop-in for replicated sampling.
The comparison is tolerance-based, not bitwise: engine boots land in
slightly different kernel/collective states (measured: same-state boots are
bitwise identical across the two modes; cross-state boots differ by up to
~0.25 logprob and can flip a near-tie token). A real sharding bug — rows
routed to the wrong request, misaligned gathers — produces many-nats
divergence and scrambled top-k sets, far outside these bounds."""
import pytest
from tests.utils import multi_gpu_test
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
# Boot-state numeric noise bound (measured <= 0.25 on identical contexts).
LOGPROB_TOL = 0.5
# Near-tie greedy/seeded flips from boot-state noise; measured <= 1 per
# engine boot across ~16 completions, plus one of headroom.
MAX_DIVERGENT_PROMPTS = 3
PROMPTS = [
"The capital of France is",
"The three primary colors are",
"Write a haiku about the ocean.",
"1, 1, 2, 3, 5, 8,",
"The chemical symbol for gold is",
"Once upon a time, in a distant kingdom,",
"Python's GIL stands for",
"The fastest land animal is",
]
# Exercise greedy, seeded random, top-k/top-p, penalties, and logprobs.
SAMPLING_PARAMS = [
SamplingParams(temperature=0.0, max_tokens=32, logprobs=5, prompt_logprobs=1),
SamplingParams(temperature=0.8, seed=42, top_p=0.9, max_tokens=32, logprobs=3),
SamplingParams(temperature=0.7, seed=7, top_k=50, min_p=0.05, max_tokens=32),
SamplingParams(
temperature=0.9,
seed=123,
frequency_penalty=0.5,
repetition_penalty=1.2,
max_tokens=32,
logprobs=2,
),
]
def _is_sharded_sampling_active(worker) -> bool:
return worker.model_runner.batch_sharder is not None
def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
llm = LLM(
model=MODEL,
tensor_parallel_size=2,
enable_batch_sharded_sampling=not disable_sharding,
max_model_len=1024,
# More requests than ranks, so round-robin slot ownership splits the
# batch across both ranks.
max_num_seqs=len(PROMPTS),
enforce_eager=True,
enable_prefix_caching=False,
gpu_memory_utilization=0.7,
# Timing-based kernel selection is the dominant source of cross-boot
# numeric noise; disable it so the comparison bounds stay tight.
kernel_config={"enable_flashinfer_autotune": False},
)
try:
# Guard against a vacuous comparison: verify every worker is in the
# intended sampling mode.
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
assert all(mode == (not disable_sharding) for mode in modes), modes
params = [
SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))
]
# Two waves: the second reuses the request slots freed by the first
# (in whatever order requests finished), covering the cross-rank
# slot-recycling path that request ownership derives from.
wave1 = llm.generate(PROMPTS, params)
wave2 = llm.generate(PROMPTS[::-1], params[::-1])
return wave1 + wave2
finally:
del llm
cleanup_dist_env_and_memory()
def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
# Allow one boundary entry of the top-k set to swap at a near-tie.
common = set(ref_lps) & set(out_lps)
assert len(common) >= max(len(ref_lps), len(out_lps)) - 1, (
f"{what}: top-k token sets diverge: {sorted(ref_lps)} vs {sorted(out_lps)}"
)
for token_id in common:
diff = abs(ref_lps[token_id].logprob - out_lps[token_id].logprob)
assert diff <= LOGPROB_TOL, (
f"{what}[{token_id}]: logprob diff {diff} "
f"({ref_lps[token_id].logprob} vs {out_lps[token_id].logprob})"
)
@multi_gpu_test(num_gpus=2)
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
"""Generation, logprobs, and prompt logprobs match between batch-sharded
sampling and the replicated fallback, up to measured boot-state noise."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
ref_outputs = _generate(monkeypatch, disable_sharding=True)
shard_outputs = _generate(monkeypatch, disable_sharding=False)
assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS)
num_divergent = 0
for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)):
ref_completion = ref.outputs[0]
out_completion = out.outputs[0]
ref_ids = list(ref_completion.token_ids)
out_ids = list(out_completion.token_ids)
# Logits are only comparable at positions whose context (prompt +
# previously sampled tokens) is identical: every position up to and
# including the first divergence.
prefix = 0
while prefix < min(len(ref_ids), len(out_ids)):
if ref_ids[prefix] != out_ids[prefix]:
break
prefix += 1
if ref_ids != out_ids:
num_divergent += 1
if ref_completion.logprobs is not None:
assert out_completion.logprobs is not None
comparable = min(prefix + 1, len(ref_ids), len(out_ids))
for pos in range(comparable):
_assert_logprob_dicts_close(
ref_completion.logprobs[pos],
out_completion.logprobs[pos],
f"prompt {i} logprobs[{pos}]",
)
# The prompt is fixed, so every prompt-logprob position is comparable.
if ref.prompt_logprobs is not None:
assert out.prompt_logprobs is not None
for pos, (ref_lps, out_lps) in enumerate(
zip(ref.prompt_logprobs, out.prompt_logprobs)
):
if ref_lps is None or out_lps is None:
assert ref_lps is None and out_lps is None
continue
_assert_logprob_dicts_close(
ref_lps, out_lps, f"prompt {i} prompt_logprobs[{pos}]"
)
assert num_divergent <= MAX_DIVERGENT_PROMPTS, (
f"{num_divergent}/{2 * len(PROMPTS)} prompts diverged: beyond near-tie "
"boot-state noise; sharded sampling is likely misrouting requests"
)
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Batch-sharded sampling must be a drop-in for replicated sampling under
speculative decoding.
The non-spec arm of this claim lives in tests/v1/e2e/general/
test_sharded_sampling.py. Spec decode is the harder case: draft tokens give
each request a different number of logits rows, so the shard plan's
per-request arithmetic and the rejection sampler are both exercised.
"""
import pytest
import torch
from tests.utils import wait_for_memory_to_settle
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from .utils import _skip_if_insufficient_gpus_for_tp, get_test_prompts, greedy_sampling
def _is_sharded_sampling_active(worker) -> bool:
return worker.model_runner.batch_sharder is not None
def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
"""Batch-sharded sampling must be a bit-exact drop-in for replicated
sampling under MTP spec decoding: the collectives move the same logits
bytes, Gumbel keys derive from (request slot, position, seed), and slot
assignment is rank-deterministic. Both runs here are spec decode with
identical math, so outputs must match exactly."""
tp_size = 2
_skip_if_insufficient_gpus_for_tp(tp_size)
model_name = "Qwen/Qwen3.5-0.8B-Base"
test_prompts = get_test_prompts(mm_enabled=False, num_prompts=20)
with monkeypatch.context() as m:
# Batch-sharded sampling only exists in the V2 model runner.
m.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
def run(disable_sharding: bool):
llm = LLM(
model=model_name,
tensor_parallel_size=tp_size,
enable_batch_sharded_sampling=not disable_sharding,
speculative_config={
"method": "mtp",
"num_speculative_tokens": 1,
"max_model_len": 2048,
},
max_model_len=2048,
limit_mm_per_prompt={"image": 0, "video": 0},
# More requests than ranks, so round-robin slot ownership
# splits the batch across both ranks.
max_num_seqs=len(test_prompts),
# Timing-based kernel selection is the dominant source of
# cross-boot numeric noise; disable for a tight comparison.
kernel_config={"enable_flashinfer_autotune": False},
)
try:
# Guard against a vacuous comparison: verify every worker is
# in the intended sampling mode.
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
assert all(mode == (not disable_sharding) for mode in modes)
greedy = llm.chat(test_prompts, greedy_sampling())
seeded = llm.chat(
test_prompts,
SamplingParams(temperature=1.0, seed=33, max_tokens=10),
)
return greedy, seeded
finally:
del llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
wait_for_memory_to_settle()
ref_greedy, ref_seeded = run(disable_sharding=True)
shard_greedy, shard_seeded = run(disable_sharding=False)
# Engine boots are not bitwise deterministic (kernel/collective
# selection shifts logits by ~1 ulp, flipping near-tie tokens), so
# allow a small number of divergent prompts. A sharding bug produces
# wholesale divergence, not isolated near-tie flips.
for name, ref_outputs, shard_outputs in (
("greedy", ref_greedy, shard_greedy),
("seeded", ref_seeded, shard_seeded),
):
num_divergent = 0
for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)):
if list(ref.outputs[0].token_ids) != list(out.outputs[0].token_ids):
num_divergent += 1
print(
f"{name} prompt {i} diverged:\n"
f" replicated: {ref.outputs[0].text!r}\n"
f" sharded: {out.outputs[0].text!r}"
)
assert num_divergent <= 2, (
f"{name}: {num_divergent}/{len(ref_outputs)} prompts diverged, "
"beyond near-tie boot noise"
)
+613
View File
@@ -0,0 +1,613 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routing-math tests for batch-sharded sampling.
These simulate every TP rank in-process (no torch.distributed): the local
sub-batch, the all-to-all send/recv layout, and the padded result gather must
compose to a bit-exact round trip of the replicated computation.
"""
from types import SimpleNamespace
from unittest import mock
import numpy as np
import pytest
import torch
from vllm.v1.core.sched.output import GrammarOutput
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.sample import batch_shard
from vllm.v1.worker.gpu.sample.batch_shard import (
BatchSharder,
_shard_grammar_output,
)
DEVICE = "cuda"
# The owner-sorted layout is expanded by a Triton kernel, so the routing math
# can only be exercised on device.
requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA"
)
def _np(t: torch.Tensor) -> np.ndarray:
return t.cpu().numpy()
def _make_batch(
rng: np.random.Generator,
num_reqs: int,
max_num_reqs: int,
max_spec: int,
slot_pool: np.ndarray | None = None,
) -> InputBatch:
"""Consistent batch with random unique request slots, 1 + k_i logits
rows per request at the tail of each query segment."""
if slot_pool is None:
slot_pool = np.arange(max_num_reqs)
slots = rng.choice(slot_pool, size=num_reqs, replace=False)
slots = slots.astype(np.int32)
k = rng.integers(0, max_spec + 1, size=num_reqs)
num_logits_per_req = (1 + k).astype(np.int32)
cu = np.zeros(num_reqs + 1, dtype=np.int32)
np.cumsum(num_logits_per_req, out=cu[1:])
query_lens = num_logits_per_req + rng.integers(0, 4, size=num_reqs).astype(np.int32)
qsl = np.zeros(num_reqs + 1, dtype=np.int32)
np.cumsum(query_lens, out=qsl[1:])
num_tokens = int(qsl[-1])
logits_indices_np = np.concatenate(
[
np.arange(qsl[i + 1] - num_logits_per_req[i], qsl[i + 1])
for i in range(num_reqs)
]
).astype(np.int64)
expanded_idx_np = np.repeat(slots, num_logits_per_req)
expanded_pos_np = np.concatenate(
[np.arange(n, dtype=np.int32) for n in num_logits_per_req]
)
seq_lens_np = (query_lens + rng.integers(0, 100, size=num_reqs)).astype(np.int32)
def dev(a: np.ndarray) -> torch.Tensor:
return torch.from_numpy(a).to(DEVICE)
return InputBatch(
req_ids=[f"req-{i}" for i in range(num_reqs)],
num_reqs=num_reqs,
num_reqs_after_padding=num_reqs,
idx_mapping=dev(slots),
idx_mapping_np=slots,
expanded_idx_mapping=dev(expanded_idx_np),
expanded_local_pos=dev(expanded_pos_np),
num_scheduled_tokens=query_lens,
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens,
num_draft_tokens=int(k.sum()),
num_draft_tokens_per_req=k.astype(np.int32),
query_start_loc=dev(qsl),
query_start_loc_np=qsl,
seq_lens=dev(seq_lens_np),
seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens_np.copy()),
dcp_local_seq_lens=None,
num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32),
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
has_prefill=False,
max_seq_len_np=None,
input_ids=torch.zeros(num_tokens, dtype=torch.int32, device=DEVICE),
positions=torch.arange(num_tokens, dtype=torch.int64, device=DEVICE),
is_padding=torch.zeros(num_tokens, dtype=torch.bool, device=DEVICE),
logits_indices=dev(logits_indices_np),
cu_num_logits=dev(cu),
cu_num_logits_np=cu,
has_structured_output_reqs=False,
prompt_lens=None,
)
def _shard_all_ranks(batch: InputBatch, max_num_reqs: int, tp_size: int):
"""Simulate every TP rank by faking get_tp_group() per BatchSharder."""
results = []
for rank in range(tp_size):
fake_group = SimpleNamespace(rank_in_group=rank, world_size=tp_size)
with mock.patch.object(batch_shard, "get_tp_group", return_value=fake_group):
sharder = BatchSharder(
max_num_reqs=max_num_reqs,
max_num_logits_per_req=8,
device=torch.device(DEVICE),
)
local_batch, sorted_logits_indices, _, metadata = sharder.shard_sampler_inputs(
batch, None
)
results.append((local_batch, sorted_logits_indices, metadata))
return results
def _owned_batch_indices(local_batch: InputBatch) -> np.ndarray:
return np.array([int(r.split("-")[1]) for r in local_batch.req_ids], dtype=np.int64)
def _owned_rows(cu: np.ndarray, owned: np.ndarray) -> np.ndarray:
if len(owned) == 0:
return np.array([], dtype=np.int64)
return np.concatenate([np.arange(cu[r], cu[r + 1]) for r in owned])
@requires_cuda
@pytest.mark.parametrize("tp_size", [2, 4, 8])
@pytest.mark.parametrize("seed", [0, 1])
def test_local_batch_partition(tp_size: int, seed: int):
"""Local sub-batches partition the batch by owner, preserve batch order
within an owner, and carry exactly the owned requests' entries of every
per-request and per-row array."""
rng = np.random.default_rng(seed)
max_num_reqs = 64
batch = _make_batch(rng, num_reqs=23, max_num_reqs=max_num_reqs, max_spec=3)
num_logits = int(batch.cu_num_logits_np[-1])
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
cu = batch.cu_num_logits_np
metadata0 = results[0][2]
assert sum(metadata0.num_logits_per_rank) == num_logits
# Gather width = max logits rows of any request (bounds num_sampled).
assert metadata0.max_num_logits_per_req == int(np.diff(cu).max())
all_owned: list[int] = []
for rank, (local, _, metadata) in enumerate(results):
owned = _owned_batch_indices(local)
assert local.num_reqs == len(owned) == metadata.num_local_reqs
assert (batch.idx_mapping_np[owned] % tp_size == rank).all()
# Stable sort: batch order is preserved within an owner.
assert (np.diff(owned) > 0).all()
all_owned.extend(owned.tolist())
rows = _owned_rows(cu, owned)
assert (
metadata.num_local_logits
== len(rows)
== metadata0.num_logits_per_rank[rank]
)
expected_idx = batch.idx_mapping_np[owned]
assert (local.idx_mapping_np == expected_idx).all()
assert _np(local.idx_mapping).tolist() == expected_idx.tolist()
assert (np.diff(local.cu_num_logits_np) == np.diff(cu)[owned]).all()
assert _np(local.cu_num_logits).tolist() == local.cu_num_logits_np.tolist()
assert (
_np(local.expanded_idx_mapping) == _np(batch.expanded_idx_mapping)[rows]
).all()
assert (
_np(local.expanded_local_pos) == _np(batch.expanded_local_pos)[rows]
).all()
assert (_np(local.logits_indices) == _np(batch.logits_indices)[rows]).all()
assert (_np(local.seq_lens) == _np(batch.seq_lens)[owned]).all()
assert batch.num_draft_tokens_per_req is not None
assert local.num_draft_tokens_per_req is not None
assert (
local.num_draft_tokens_per_req == batch.num_draft_tokens_per_req[owned]
).all()
assert local.num_draft_tokens == int(
batch.num_draft_tokens_per_req[owned].sum()
)
# Triton specializes kernels on pointer alignment: every packed-buffer
# view fed to kernels must stay 16-byte aligned across batch shapes,
# or serving JITs new kernel variants mid-collective.
for t in (
local.idx_mapping,
local.expanded_idx_mapping,
local.expanded_local_pos,
local.logits_indices,
local.cu_num_logits,
metadata.gathered_src_indices,
):
assert t.data_ptr() % 16 == 0
assert sorted(all_owned) == list(range(batch.num_reqs))
# sorted_logits_indices is identical on all ranks: the owner-sorted logits row
# positions, i.e. logits_indices permuted rank-major.
sorted_logits_indices = _np(results[0][1])
for _, gi, _ in results:
assert (_np(gi) == sorted_logits_indices).all()
rows_all = np.concatenate(
[_owned_rows(cu, _owned_batch_indices(local)) for local, _, _ in results]
)
assert (
sorted_logits_indices.tolist() == _np(batch.logits_indices)[rows_all].tolist()
)
@requires_cuda
@pytest.mark.parametrize("tp_size", [2, 4])
def test_all_to_all_reassembly_simulated(tp_size: int):
"""Vocab shards computed from owner-sorted hidden states and routed
through the simulated all-to-all reassemble into exactly the full-vocab
logits of each rank's owned rows."""
rng = np.random.default_rng(7)
max_num_reqs = 32
batch = _make_batch(rng, num_reqs=11, max_num_reqs=max_num_reqs, max_spec=4)
vocab = 64 * tp_size
shard_w = vocab // tp_size
token_logits = torch.randn(batch.num_tokens, vocab, device=DEVICE)
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
sorted_logits_indices = results[0][1]
num_logits_per_rank = results[0][2].num_logits_per_rank
bounds = np.zeros(tp_size + 1, dtype=np.int64)
np.cumsum(num_logits_per_rank, out=bounds[1:])
# send_r = rank r's vocab shard of the owner-sorted rows, with no further
# permute; chunk d of send_r goes to rank d.
sends = [
token_logits[sorted_logits_indices][:, r * shard_w : (r + 1) * shard_w]
for r in range(tp_size)
]
for rank, (local, _, metadata) in enumerate(results):
recv = torch.cat(
[sends[src][bounds[rank] : bounds[rank + 1]] for src in range(tp_size)]
)
# Reassemble source-rank-major chunks into full-vocab rows, mirroring
# all_to_all_logits (source-rank order is vocab-shard order).
reassembled = (
recv.view(tp_size, metadata.num_local_logits, shard_w)
.permute(1, 0, 2)
.reshape(metadata.num_local_logits, vocab)
)
expected = token_logits[local.logits_indices]
assert torch.equal(reassembled, expected)
@requires_cuda
def test_shard_no_spec():
"""Batches without draft tokens (one logits row per request) degenerate
cleanly: rows mirror requests."""
rng = np.random.default_rng(11)
tp_size = 4
max_num_reqs = 32
batch = _make_batch(rng, num_reqs=10, max_num_reqs=max_num_reqs, max_spec=0)
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
assert results[0][2].max_num_logits_per_req == 1
for local, _, metadata in results:
assert metadata.num_local_logits == metadata.num_local_reqs == local.num_reqs
owned = _owned_batch_indices(local)
assert _np(local.logits_indices).tolist() == (
_np(batch.logits_indices)[owned].tolist()
)
@pytest.mark.parametrize("tp_size", [2, 4])
def test_shard_grammar_output(tp_size: int):
"""The grammar filter keeps exactly the owned requests' bitmask rows, in
grammar order, and the per-rank pieces partition the global bitmask."""
rng = np.random.default_rng(5)
max_num_reqs = 32
num_reqs = 12
slots = rng.choice(np.arange(max_num_reqs), size=num_reqs, replace=False)
k = rng.integers(0, 4, size=num_reqs)
cu = np.zeros(num_reqs + 1, dtype=np.int32)
np.cumsum(1 + k, out=cu[1:])
owner = slots % tp_size
# Grammar requests: every other request, in a scrambled order.
req_ids = [f"req-{i}" for i in range(num_reqs)]
grammar_idx = list(rng.permutation(np.arange(0, num_reqs, 2)))
grammar_req_ids = [req_ids[i] for i in grammar_idx]
num_grammar_rows = sum(int(cu[i + 1] - cu[i]) for i in grammar_idx)
bitmask = rng.integers(0, 2**31, size=(num_grammar_rows, 4), dtype=np.int32)
input_batch = SimpleNamespace(req_ids=req_ids, cu_num_logits_np=cu)
# Global row offset of each grammar request within the bitmask.
row_offsets = {}
cursor = 0
for i in grammar_idx:
row_offsets[i] = cursor
cursor += int(cu[i + 1] - cu[i])
grammar_output = GrammarOutput(
structured_output_request_ids=grammar_req_ids,
grammar_bitmask=bitmask,
)
kept_total = 0
for rank in range(tp_size):
local_req_ids = [req_ids[i] for i in range(num_reqs) if owner[i] == rank]
local = _shard_grammar_output(grammar_output, input_batch, local_req_ids)
expected_idx = [i for i in grammar_idx if owner[i] == rank]
if not expected_idx:
assert local is None
continue
assert local is not None
assert local.structured_output_request_ids == [req_ids[i] for i in expected_idx]
expected_rows = np.concatenate(
[
np.arange(row_offsets[i], row_offsets[i] + cu[i + 1] - cu[i])
for i in expected_idx
]
)
assert (local.grammar_bitmask == bitmask[expected_rows.astype(int)]).all()
kept_total += local.grammar_bitmask.shape[0]
assert kept_total == num_grammar_rows
@requires_cuda
@pytest.mark.parametrize("tp_size", [2, 4])
def test_gather_src_indices_round_trip(tp_size: int):
"""Padded per-rank result blocks indexed by gathered_src_indices restore
batch order, including when some rank owns zero requests."""
rng = np.random.default_rng(3)
max_num_reqs = 40
all_slots = np.arange(max_num_reqs)
batch = _make_batch(
rng,
num_reqs=17,
max_num_reqs=max_num_reqs,
max_spec=2,
slot_pool=all_slots[all_slots % tp_size != 0], # rank 0 owns nothing
)
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
metadata0 = results[0][2]
assert metadata0.num_local_reqs == 0
# Value of request j is j; each rank packs its owned requests' values into
# a zero-padded (max_num_reqs_per_rank, 1) block, gathered rank-major.
pad = metadata0.max_num_reqs_per_rank
blocks = []
for local, _, _ in results:
owned = _owned_batch_indices(local)
block = torch.zeros(pad, 1, dtype=torch.int64, device=DEVICE)
block[: len(owned), 0] = torch.from_numpy(owned).to(DEVICE)
blocks.append(block)
gathered = torch.cat(blocks)
restored = gathered[metadata0.gathered_src_indices][:, 0]
assert restored.tolist() == list(range(batch.num_reqs))
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("tp_size", [2, 4])
def test_gather_sampler_output_kernels(tp_size: int):
"""The fused pack/unpack kernels round-trip per-rank sampler results
into batch order: token ids zero-filled past the local width, counts
cast back to int32, identical on every rank."""
from dataclasses import replace as dc_replace
from vllm.v1.worker.gpu.sample.output import SamplerOutput
rng = np.random.default_rng(9)
max_num_reqs = 32
batch = _make_batch(rng, num_reqs=13, max_num_reqs=max_num_reqs, max_spec=3)
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
device = torch.device("cuda")
num_reqs = batch.num_reqs
width = results[0][2].max_num_logits_per_req
# Distinctive per-rank local outputs. Even ranks mimic the regular
# sampler (one column, exercising zero-fill); odd ranks mimic the
# rejection sampler's over-allocation (exercising truncation).
local_outputs: list[SamplerOutput | None] = []
expected_ids = torch.zeros(num_reqs, width, dtype=torch.int64)
expected_sampled = torch.zeros(num_reqs, dtype=torch.int32)
expected_rejected = torch.zeros(num_reqs, dtype=torch.int32)
for rank, (local, _, metadata) in enumerate(results):
if metadata.num_local_reqs == 0:
local_outputs.append(None)
continue
owned = torch.from_numpy(_owned_batch_indices(local))
src_width = 1 if rank % 2 == 0 else width + 3
ids = owned[:, None] * 1000 + torch.arange(src_width)[None, :]
num_sampled = (owned % width + 1).to(torch.int32)
num_rejected = (owned % 3).to(torch.int32)
local_outputs.append(
SamplerOutput(
sampled_token_ids=ids.to(device),
logprobs_tensors=None,
num_nans=None,
num_sampled=num_sampled.to(device),
num_rejected=num_rejected.to(device),
)
)
copy_width = min(src_width, width)
expected_ids[owned, :copy_width] = ids[:, :copy_width]
expected_sampled[owned] = num_sampled
expected_rejected[owned] = num_rejected
# Two passes over a mocked all-gather: capture each rank's packed send
# block, then replay with the concatenated blocks as the gathered result.
packed_blocks: dict[int, torch.Tensor] = {}
gathered_full: list[torch.Tensor] = []
def run(rank: int) -> SamplerOutput:
metadata = dc_replace(
results[rank][2],
gathered_src_indices=results[rank][2].gathered_src_indices.to(device),
)
def fake_all_gather(x: torch.Tensor, dim: int = 0) -> torch.Tensor:
packed_blocks[rank] = x.clone()
if gathered_full:
return gathered_full[0]
return torch.zeros(
tp_size * x.shape[0], x.shape[1], dtype=x.dtype, device=x.device
)
with mock.patch.object(
batch_shard, "tensor_model_parallel_all_gather", fake_all_gather
):
return batch_shard.gather_sampler_output(
local_outputs[rank],
metadata,
device,
global_batch=batch,
local_batch=results[rank][0],
)
for rank in range(tp_size):
run(rank)
# Zero the uninitialized padding rows so the comparison is deterministic;
# the real path never reads them (gathered_src_indices skips padding).
for rank, (local, _, metadata) in enumerate(results):
packed_blocks[rank][metadata.num_local_reqs :] = 0
gathered_full.append(torch.cat([packed_blocks[r] for r in range(tp_size)]))
for rank in range(tp_size):
out = run(rank)
assert out.sampled_token_ids.dtype == torch.int64
assert out.num_sampled is not None and out.num_sampled.dtype == torch.int32
assert out.num_rejected is not None
assert torch.equal(out.sampled_token_ids.cpu(), expected_ids)
assert torch.equal(out.num_sampled.cpu(), expected_sampled)
assert torch.equal(out.num_rejected.cpu(), expected_rejected.to(torch.int32))
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_gather_sampler_output_logprobs_and_nans():
"""num_nans is reduced per request and gathered; LogprobsTensors are
packed into fixed per-request blocks, width-aligned across ranks (pad
and truncate paths), and restored to global row order."""
from dataclasses import replace as dc_replace
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.worker.gpu.sample.output import SamplerOutput
tp_size = 2
rng = np.random.default_rng(21)
max_num_reqs = 32
batch = _make_batch(rng, num_reqs=6, max_num_reqs=max_num_reqs, max_spec=3)
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
device = torch.device("cuda")
num_reqs = batch.num_reqs
cu = batch.cu_num_logits_np
num_logits = int(cu[-1])
width = results[0][2].max_num_logits_per_req
num_cols = 3
def cuda_fields(b: InputBatch) -> InputBatch:
return dc_replace(
b,
cu_num_logits=b.cu_num_logits.to(device),
expanded_local_pos=b.expanded_local_pos.to(device),
)
global_cuda = cuda_fields(batch)
local_outputs: list[SamplerOutput | None] = []
local_batches: list[InputBatch] = []
expected_ids = torch.zeros(num_logits, num_cols, dtype=torch.int64)
expected_logprobs = torch.full(
(num_logits, num_cols), float("-inf"), dtype=torch.float32
)
expected_ranks = torch.zeros(num_logits, dtype=torch.int64)
expected_nans = torch.zeros(num_reqs, dtype=torch.int32)
for rank, (local, _, metadata) in enumerate(results):
local_batches.append(cuda_fields(local))
if metadata.num_local_reqs == 0:
local_outputs.append(None)
continue
owned = _owned_batch_indices(local)
rows = torch.from_numpy(_owned_rows(cu, owned))
# Rank 0 sends fewer columns (pad path), rank 1 more (truncate path).
local_cols = num_cols - 1 if rank == 0 else num_cols + 2
ids = rows[:, None] * 10 + torch.arange(local_cols)[None, :]
logprobs = ids.float() * 0.5
ranks_t = rows + 7
nans = (rows % 4).to(torch.int32)
local_outputs.append(
SamplerOutput(
sampled_token_ids=torch.zeros(
metadata.num_local_reqs, width, dtype=torch.int64, device=device
),
logprobs_tensors=LogprobsTensors(
logprob_token_ids=ids.to(device),
logprobs=logprobs.to(device),
selected_token_ranks=ranks_t.to(device),
cu_num_generated_tokens=None,
),
num_nans=nans.to(device),
num_sampled=torch.ones(
metadata.num_local_reqs, dtype=torch.int32, device=device
),
num_rejected=torch.zeros(
metadata.num_local_reqs, dtype=torch.int32, device=device
),
)
)
copy_cols = min(local_cols, num_cols)
expected_ids[rows, :copy_cols] = ids[:, :copy_cols]
expected_logprobs[rows, :copy_cols] = logprobs[:, :copy_cols]
expected_ranks[rows] = ranks_t
for j in owned:
in_req = (rows >= int(cu[j])) & (rows < int(cu[j + 1]))
expected_nans[j] = int(nans[in_req].sum())
recorded: dict[int, list[torch.Tensor]] = {}
gathered_full: list[torch.Tensor] = []
def run(rank: int) -> SamplerOutput:
metadata = dc_replace(
results[rank][2],
gathered_src_indices=results[rank][2].gathered_src_indices.to(device),
)
calls = {"i": 0}
def fake_all_gather(x: torch.Tensor, dim: int = 0) -> torch.Tensor:
i = calls["i"]
calls["i"] += 1
if gathered_full:
return gathered_full[i]
recorded.setdefault(rank, []).append(x.clone())
return torch.zeros(
tp_size * x.shape[0], *x.shape[1:], dtype=x.dtype, device=x.device
)
with mock.patch.object(
batch_shard, "tensor_model_parallel_all_gather", fake_all_gather
):
return batch_shard.gather_sampler_output(
local_outputs[rank],
metadata,
device,
global_batch=global_cuda,
local_batch=local_batches[rank],
gather_num_nans=True,
logprobs_dims=(num_cols - 1, 0),
)
for rank in range(tp_size):
run(rank)
gathered_full.extend(
torch.cat([recorded[r][i] for r in range(tp_size)]) for i in range(3)
)
for rank in range(tp_size):
out = run(rank)
assert out.num_nans is not None
assert torch.equal(out.num_nans.cpu(), expected_nans)
lp = out.logprobs_tensors
assert lp is not None
assert torch.equal(lp.logprob_token_ids.cpu(), expected_ids)
assert torch.equal(lp.logprobs.cpu(), expected_logprobs)
assert torch.equal(lp.selected_token_ranks.cpu(), expected_ranks)
if num_logits != num_reqs:
assert lp.cu_num_generated_tokens == cu.tolist()
else:
assert lp.cu_num_generated_tokens is None
def test_finish_requests_frees_slots_in_sorted_order():
"""Request slots must be freed in the same order on every TP rank:
ownership derives from slot indices, and `finished_req_ids` is a set
whose iteration order is per-process hash-randomized."""
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
removed: list[str] = []
fake_runner = SimpleNamespace(_remove_request=removed.append, pooling_runner=None)
scheduler_output = SimpleNamespace(
finished_req_ids={"req-b", "req-c", "req-a"},
preempted_req_ids={"req-e", "req-d"},
)
GPUModelRunner.finish_requests(fake_runner, scheduler_output)
assert removed == sorted(removed)
assert set(removed) == {"req-a", "req-b", "req-c", "req-d", "req-e"}
@@ -60,6 +60,7 @@ def _make_runner(**overrides: Any) -> Any:
runner.parallel_config = SimpleNamespace(
enable_eplb=True,
enable_elastic_ep=False,
enable_batch_sharded_sampling=False,
eplb_config=SimpleNamespace(log_balancedness=True),
)
runner.vllm_config = SimpleNamespace(